1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2007-2014 Nicira, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/init.h> 9 #include <linux/module.h> 10 #include <linux/if_arp.h> 11 #include <linux/if_vlan.h> 12 #include <linux/in.h> 13 #include <linux/ip.h> 14 #include <linux/jhash.h> 15 #include <linux/delay.h> 16 #include <linux/time.h> 17 #include <linux/etherdevice.h> 18 #include <linux/kernel.h> 19 #include <linux/kthread.h> 20 #include <linux/mutex.h> 21 #include <linux/percpu.h> 22 #include <linux/rcupdate.h> 23 #include <linux/tcp.h> 24 #include <linux/udp.h> 25 #include <linux/ethtool.h> 26 #include <linux/wait.h> 27 #include <asm/div64.h> 28 #include <linux/highmem.h> 29 #include <linux/netfilter_bridge.h> 30 #include <linux/netfilter_ipv4.h> 31 #include <linux/inetdevice.h> 32 #include <linux/list.h> 33 #include <linux/openvswitch.h> 34 #include <linux/rculist.h> 35 #include <linux/dmi.h> 36 #include <net/genetlink.h> 37 #include <net/gso.h> 38 #include <net/net_namespace.h> 39 #include <net/netns/generic.h> 40 #include <net/pkt_cls.h> 41 42 #include "datapath.h" 43 #include "drop.h" 44 #include "flow.h" 45 #include "flow_table.h" 46 #include "flow_netlink.h" 47 #include "meter.h" 48 #include "openvswitch_trace.h" 49 #include "vport-internal_dev.h" 50 #include "vport-netdev.h" 51 52 unsigned int ovs_net_id __read_mostly; 53 54 static struct genl_family dp_packet_genl_family; 55 static struct genl_family dp_flow_genl_family; 56 static struct genl_family dp_datapath_genl_family; 57 58 static const struct nla_policy flow_policy[]; 59 60 static const struct genl_multicast_group ovs_dp_flow_multicast_group = { 61 .name = OVS_FLOW_MCGROUP, 62 }; 63 64 static const struct genl_multicast_group ovs_dp_datapath_multicast_group = { 65 .name = OVS_DATAPATH_MCGROUP, 66 }; 67 68 static const struct genl_multicast_group ovs_dp_vport_multicast_group = { 69 .name = OVS_VPORT_MCGROUP, 70 }; 71 72 /* Check if need to build a reply message. 73 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */ 74 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info, 75 unsigned int group) 76 { 77 return info->nlhdr->nlmsg_flags & NLM_F_ECHO || 78 genl_has_listeners(family, genl_info_net(info), group); 79 } 80 81 static void ovs_notify(struct genl_family *family, 82 struct sk_buff *skb, struct genl_info *info) 83 { 84 genl_notify(family, skb, info, 0, GFP_KERNEL); 85 } 86 87 /** 88 * DOC: Locking: 89 * 90 * All writes e.g. Writes to device state (add/remove datapath, port, set 91 * operations on vports, etc.), Writes to other state (flow table 92 * modifications, set miscellaneous datapath parameters, etc.) are protected 93 * by ovs_lock. 94 * 95 * Reads are protected by RCU. 96 * 97 * There are a few special cases (mostly stats) that have their own 98 * synchronization but they nest under all of above and don't interact with 99 * each other. 100 * 101 * The RTNL lock nests inside ovs_mutex. 102 */ 103 104 static DEFINE_MUTEX(ovs_mutex); 105 106 void ovs_lock(void) 107 { 108 mutex_lock(&ovs_mutex); 109 } 110 111 void ovs_unlock(void) 112 { 113 mutex_unlock(&ovs_mutex); 114 } 115 116 #ifdef CONFIG_LOCKDEP 117 int lockdep_ovsl_is_held(void) 118 { 119 if (debug_locks) 120 return lockdep_is_held(&ovs_mutex); 121 else 122 return 1; 123 } 124 #endif 125 126 static struct vport *new_vport(const struct vport_parms *); 127 static int queue_gso_packets(struct datapath *dp, struct sk_buff *, 128 const struct sw_flow_key *, 129 const struct dp_upcall_info *, 130 uint32_t cutlen); 131 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *, 132 const struct sw_flow_key *, 133 const struct dp_upcall_info *, 134 uint32_t cutlen); 135 136 static void ovs_dp_masks_rebalance(struct work_struct *work); 137 138 static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *); 139 140 /* Must be called with rcu_read_lock or ovs_mutex. */ 141 const char *ovs_dp_name(const struct datapath *dp) 142 { 143 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL); 144 return ovs_vport_name(vport); 145 } 146 147 static int get_dpifindex(const struct datapath *dp) 148 { 149 struct vport *local; 150 int ifindex; 151 152 rcu_read_lock(); 153 154 local = ovs_vport_rcu(dp, OVSP_LOCAL); 155 if (local) 156 ifindex = local->dev->ifindex; 157 else 158 ifindex = 0; 159 160 rcu_read_unlock(); 161 162 return ifindex; 163 } 164 165 static void destroy_dp_rcu(struct rcu_head *rcu) 166 { 167 struct datapath *dp = container_of(rcu, struct datapath, rcu); 168 169 ovs_flow_tbl_destroy(&dp->table); 170 free_percpu(dp->stats_percpu); 171 kfree(dp->ports); 172 ovs_meters_exit(dp); 173 kfree(rcu_dereference_raw(dp->upcall_portids)); 174 kfree(dp); 175 } 176 177 static struct hlist_head *vport_hash_bucket(const struct datapath *dp, 178 u16 port_no) 179 { 180 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)]; 181 } 182 183 /* Called with ovs_mutex or RCU read lock. */ 184 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no) 185 { 186 struct vport *vport; 187 struct hlist_head *head; 188 189 head = vport_hash_bucket(dp, port_no); 190 hlist_for_each_entry_rcu(vport, head, dp_hash_node, 191 lockdep_ovsl_is_held()) { 192 if (vport->port_no == port_no) 193 return vport; 194 } 195 return NULL; 196 } 197 198 /* Called with ovs_mutex. */ 199 static struct vport *new_vport(const struct vport_parms *parms) 200 { 201 struct vport *vport; 202 203 vport = ovs_vport_add(parms); 204 if (!IS_ERR(vport)) { 205 struct datapath *dp = parms->dp; 206 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no); 207 208 hlist_add_head_rcu(&vport->dp_hash_node, head); 209 } 210 return vport; 211 } 212 213 static void ovs_vport_update_upcall_stats(struct sk_buff *skb, 214 const struct dp_upcall_info *upcall_info, 215 bool upcall_result) 216 { 217 struct vport *p = OVS_CB(skb)->input_vport; 218 struct vport_upcall_stats_percpu *stats; 219 220 if (upcall_info->cmd != OVS_PACKET_CMD_MISS && 221 upcall_info->cmd != OVS_PACKET_CMD_ACTION) 222 return; 223 224 stats = this_cpu_ptr(p->upcall_stats); 225 u64_stats_update_begin(&stats->syncp); 226 if (upcall_result) 227 u64_stats_inc(&stats->n_success); 228 else 229 u64_stats_inc(&stats->n_fail); 230 u64_stats_update_end(&stats->syncp); 231 } 232 233 void ovs_dp_detach_port(struct vport *p) 234 { 235 ASSERT_OVSL(); 236 237 /* First drop references to device. */ 238 hlist_del_rcu(&p->dp_hash_node); 239 240 /* Then destroy it. */ 241 ovs_vport_del(p); 242 } 243 244 /* Must be called with rcu_read_lock. */ 245 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) 246 { 247 struct ovs_pcpu_storage *ovs_pcpu = this_cpu_ptr(ovs_pcpu_storage); 248 const struct vport *p = OVS_CB(skb)->input_vport; 249 struct datapath *dp = p->dp; 250 struct sw_flow *flow; 251 struct sw_flow_actions *sf_acts; 252 struct dp_stats_percpu *stats; 253 bool ovs_pcpu_locked = false; 254 u64 *stats_counter; 255 u32 n_mask_hit; 256 u32 n_cache_hit; 257 int error; 258 259 stats = this_cpu_ptr(dp->stats_percpu); 260 261 /* Look up flow. */ 262 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb), 263 &n_mask_hit, &n_cache_hit); 264 if (unlikely(!flow)) { 265 struct dp_upcall_info upcall; 266 267 memset(&upcall, 0, sizeof(upcall)); 268 upcall.cmd = OVS_PACKET_CMD_MISS; 269 270 if (OVS_CB(skb)->upcall_pid) 271 upcall.portid = OVS_CB(skb)->upcall_pid; 272 else if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU) 273 upcall.portid = 274 ovs_dp_get_upcall_portid(dp, smp_processor_id()); 275 else 276 upcall.portid = ovs_vport_find_upcall_portid(p, skb); 277 278 upcall.mru = OVS_CB(skb)->mru; 279 error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX); 280 switch (error) { 281 case 0: 282 case -EAGAIN: 283 case -ERESTARTSYS: 284 case -EINTR: 285 consume_skb(skb); 286 break; 287 default: 288 skb_tx_error(skb); 289 kfree_skb(skb); 290 break; 291 } 292 stats_counter = &stats->n_missed; 293 goto out; 294 } 295 296 ovs_flow_stats_update(flow, key->tp.flags, skb); 297 sf_acts = rcu_dereference(flow->sf_acts); 298 /* This path can be invoked recursively: Use the current task to 299 * identify recursive invocation - the lock must be acquired only once. 300 * Even with disabled bottom halves this can be preempted on PREEMPT_RT. 301 * Limit the locking to RT to avoid assigning `owner' if it can be 302 * avoided. 303 */ 304 if (IS_ENABLED(CONFIG_PREEMPT_RT) && ovs_pcpu->owner != current) { 305 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock); 306 ovs_pcpu->owner = current; 307 ovs_pcpu_locked = true; 308 } 309 310 error = ovs_execute_actions(dp, skb, sf_acts, key); 311 if (unlikely(error)) 312 net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n", 313 ovs_dp_name(dp), error); 314 if (ovs_pcpu_locked) { 315 ovs_pcpu->owner = NULL; 316 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock); 317 } 318 319 stats_counter = &stats->n_hit; 320 321 out: 322 /* Update datapath statistics. */ 323 u64_stats_update_begin(&stats->syncp); 324 (*stats_counter)++; 325 stats->n_mask_hit += n_mask_hit; 326 stats->n_cache_hit += n_cache_hit; 327 u64_stats_update_end(&stats->syncp); 328 } 329 330 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb, 331 const struct sw_flow_key *key, 332 const struct dp_upcall_info *upcall_info, 333 uint32_t cutlen) 334 { 335 struct dp_stats_percpu *stats; 336 int err; 337 338 if (trace_ovs_dp_upcall_enabled()) 339 trace_ovs_dp_upcall(dp, skb, key, upcall_info); 340 341 if (upcall_info->portid == 0) { 342 err = -ENOTCONN; 343 goto err; 344 } 345 346 if (!skb_is_gso(skb)) 347 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen); 348 else 349 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen); 350 351 ovs_vport_update_upcall_stats(skb, upcall_info, !err); 352 if (err) 353 goto err; 354 355 return 0; 356 357 err: 358 stats = this_cpu_ptr(dp->stats_percpu); 359 360 u64_stats_update_begin(&stats->syncp); 361 stats->n_lost++; 362 u64_stats_update_end(&stats->syncp); 363 364 return err; 365 } 366 367 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb, 368 const struct sw_flow_key *key, 369 const struct dp_upcall_info *upcall_info, 370 uint32_t cutlen) 371 { 372 unsigned int gso_type = skb_shinfo(skb)->gso_type; 373 struct sw_flow_key later_key; 374 struct sk_buff *segs, *nskb; 375 int err; 376 377 BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET); 378 segs = __skb_gso_segment(skb, NETIF_F_SG, false); 379 if (IS_ERR(segs)) 380 return PTR_ERR(segs); 381 if (segs == NULL) 382 return -EINVAL; 383 384 if (gso_type & SKB_GSO_UDP) { 385 /* The initial flow key extracted by ovs_flow_key_extract() 386 * in this case is for a first fragment, so we need to 387 * properly mark later fragments. 388 */ 389 later_key = *key; 390 later_key.ip.frag = OVS_FRAG_TYPE_LATER; 391 } 392 393 /* Queue all of the segments. */ 394 skb_list_walk_safe(segs, skb, nskb) { 395 if (gso_type & SKB_GSO_UDP && skb != segs) 396 key = &later_key; 397 398 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen); 399 if (err) 400 break; 401 402 } 403 404 /* Free all of the segments. */ 405 skb_list_walk_safe(segs, skb, nskb) { 406 if (err) 407 kfree_skb(skb); 408 else 409 consume_skb(skb); 410 } 411 return err; 412 } 413 414 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info, 415 unsigned int hdrlen, int actions_attrlen) 416 { 417 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header)) 418 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */ 419 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */ 420 + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */ 421 + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */ 422 423 /* OVS_PACKET_ATTR_USERDATA */ 424 if (upcall_info->userdata) 425 size += NLA_ALIGN(upcall_info->userdata->nla_len); 426 427 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */ 428 if (upcall_info->egress_tun_info) 429 size += nla_total_size(ovs_tun_key_attr_size()); 430 431 /* OVS_PACKET_ATTR_ACTIONS */ 432 if (upcall_info->actions_len) 433 size += nla_total_size(actions_attrlen); 434 435 /* OVS_PACKET_ATTR_MRU */ 436 if (upcall_info->mru) 437 size += nla_total_size(sizeof(upcall_info->mru)); 438 439 return size; 440 } 441 442 static void pad_packet(struct datapath *dp, struct sk_buff *skb) 443 { 444 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) { 445 size_t plen = NLA_ALIGN(skb->len) - skb->len; 446 447 if (plen > 0) 448 skb_put_zero(skb, plen); 449 } 450 } 451 452 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, 453 const struct sw_flow_key *key, 454 const struct dp_upcall_info *upcall_info, 455 uint32_t cutlen) 456 { 457 struct ovs_header *upcall; 458 struct sk_buff *nskb = NULL; 459 struct sk_buff *user_skb = NULL; /* to be queued to userspace */ 460 struct nlattr *nla; 461 size_t msg_size; 462 size_t skb_len; 463 unsigned int hlen; 464 int err, dp_ifindex; 465 u64 hash; 466 467 dp_ifindex = get_dpifindex(dp); 468 if (!dp_ifindex) 469 return -ENODEV; 470 471 if (!skb_frags_readable(skb)) 472 return -EFAULT; 473 474 if (skb_vlan_tag_present(skb)) { 475 nskb = skb_clone(skb, GFP_ATOMIC); 476 if (!nskb) 477 return -ENOMEM; 478 479 nskb = __vlan_hwaccel_push_inside(nskb); 480 if (!nskb) 481 return -ENOMEM; 482 483 skb = nskb; 484 } 485 486 skb_len = min(skb->len, cutlen); 487 if (nla_attr_size(skb_len) > USHRT_MAX) { 488 err = -EFBIG; 489 goto out; 490 } 491 492 /* Complete checksum if needed */ 493 if (skb->ip_summed == CHECKSUM_PARTIAL && 494 (err = skb_csum_hwoffload_help(skb, 0))) 495 goto out; 496 497 /* Older versions of OVS user space enforce alignment of the last 498 * Netlink attribute to NLA_ALIGNTO which would require extensive 499 * padding logic. Only perform zerocopy if padding is not required. 500 */ 501 if (dp->user_features & OVS_DP_F_UNALIGNED) 502 hlen = min(skb_zerocopy_headlen(skb), cutlen); 503 else 504 hlen = skb_len; 505 506 msg_size = upcall_msg_size(upcall_info, hlen, 507 OVS_CB(skb)->acts_origlen); 508 user_skb = genlmsg_new(msg_size, GFP_ATOMIC); 509 if (!user_skb) { 510 err = -ENOMEM; 511 goto out; 512 } 513 514 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family, 515 0, upcall_info->cmd); 516 if (!upcall) { 517 err = -EINVAL; 518 goto out; 519 } 520 upcall->dp_ifindex = dp_ifindex; 521 522 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb); 523 if (err) 524 goto out; 525 526 if (upcall_info->userdata) 527 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA, 528 nla_len(upcall_info->userdata), 529 nla_data(upcall_info->userdata)); 530 531 if (upcall_info->egress_tun_info) { 532 nla = nla_nest_start_noflag(user_skb, 533 OVS_PACKET_ATTR_EGRESS_TUN_KEY); 534 if (!nla) { 535 err = -EMSGSIZE; 536 goto out; 537 } 538 err = ovs_nla_put_tunnel_info(user_skb, 539 upcall_info->egress_tun_info); 540 if (err) 541 goto out; 542 543 nla_nest_end(user_skb, nla); 544 } 545 546 if (upcall_info->actions_len) { 547 nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS); 548 if (!nla) { 549 err = -EMSGSIZE; 550 goto out; 551 } 552 err = ovs_nla_put_actions(upcall_info->actions, 553 upcall_info->actions_len, 554 user_skb); 555 if (!err) 556 nla_nest_end(user_skb, nla); 557 else 558 nla_nest_cancel(user_skb, nla); 559 } 560 561 /* Add OVS_PACKET_ATTR_MRU */ 562 if (upcall_info->mru && 563 nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) { 564 err = -ENOBUFS; 565 goto out; 566 } 567 568 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */ 569 if (skb_len < skb->len && 570 nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) { 571 err = -ENOBUFS; 572 goto out; 573 } 574 575 /* Add OVS_PACKET_ATTR_HASH */ 576 hash = skb_get_hash_raw(skb); 577 if (skb->sw_hash) 578 hash |= OVS_PACKET_HASH_SW_BIT; 579 580 if (skb->l4_hash) 581 hash |= OVS_PACKET_HASH_L4_BIT; 582 583 if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) { 584 err = -ENOBUFS; 585 goto out; 586 } 587 588 /* Only reserve room for attribute header, packet data is added 589 * in skb_zerocopy() */ 590 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) { 591 err = -ENOBUFS; 592 goto out; 593 } 594 nla->nla_len = nla_attr_size(skb_len); 595 596 err = skb_zerocopy(user_skb, skb, skb_len, hlen); 597 if (err) 598 goto out; 599 600 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */ 601 pad_packet(dp, user_skb); 602 603 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len; 604 605 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid); 606 user_skb = NULL; 607 out: 608 consume_skb(user_skb); 609 consume_skb(nskb); 610 611 return err; 612 } 613 614 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info) 615 { 616 struct ovs_header *ovs_header = genl_info_userhdr(info); 617 struct net *net = sock_net(skb->sk); 618 struct nlattr **a = info->attrs; 619 struct sw_flow_actions *acts; 620 struct sk_buff *packet; 621 struct sw_flow *flow; 622 struct sw_flow_actions *sf_acts; 623 struct datapath *dp; 624 struct vport *input_vport; 625 u16 mru = 0; 626 u64 hash; 627 int len; 628 int err; 629 bool log = !a[OVS_PACKET_ATTR_PROBE]; 630 631 err = -EINVAL; 632 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] || 633 !a[OVS_PACKET_ATTR_ACTIONS]) 634 goto err; 635 636 len = nla_len(a[OVS_PACKET_ATTR_PACKET]); 637 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL); 638 err = -ENOMEM; 639 if (!packet) 640 goto err; 641 skb_reserve(packet, NET_IP_ALIGN); 642 643 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len); 644 645 /* Set packet's mru */ 646 if (a[OVS_PACKET_ATTR_MRU]) { 647 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]); 648 packet->ignore_df = 1; 649 } 650 OVS_CB(packet)->mru = mru; 651 OVS_CB(packet)->cutlen = U32_MAX; 652 653 if (a[OVS_PACKET_ATTR_HASH]) { 654 hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]); 655 656 __skb_set_hash(packet, hash & 0xFFFFFFFFULL, 657 !!(hash & OVS_PACKET_HASH_SW_BIT), 658 !!(hash & OVS_PACKET_HASH_L4_BIT)); 659 } 660 661 OVS_CB(packet)->upcall_pid = 662 nla_get_u32_default(a[OVS_PACKET_ATTR_UPCALL_PID], 0); 663 664 /* Build an sw_flow for sending this packet. */ 665 flow = ovs_flow_alloc(); 666 err = PTR_ERR(flow); 667 if (IS_ERR(flow)) 668 goto err_kfree_skb; 669 670 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY], 671 packet, &flow->key, log); 672 if (err) 673 goto err_flow_free; 674 675 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS], 676 &flow->key, &acts, log); 677 if (err) 678 goto err_flow_free; 679 680 rcu_assign_pointer(flow->sf_acts, acts); 681 packet->priority = flow->key.phy.priority; 682 packet->mark = flow->key.phy.skb_mark; 683 684 rcu_read_lock(); 685 dp = get_dp_rcu(net, ovs_header->dp_ifindex); 686 err = -ENODEV; 687 if (!dp) 688 goto err_unlock; 689 690 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port); 691 if (!input_vport) 692 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL); 693 694 if (!input_vport) 695 goto err_unlock; 696 697 packet->dev = input_vport->dev; 698 OVS_CB(packet)->input_vport = input_vport; 699 sf_acts = rcu_dereference(flow->sf_acts); 700 701 local_bh_disable(); 702 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock); 703 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 704 this_cpu_write(ovs_pcpu_storage->owner, current); 705 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key); 706 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 707 this_cpu_write(ovs_pcpu_storage->owner, NULL); 708 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock); 709 local_bh_enable(); 710 rcu_read_unlock(); 711 712 ovs_flow_free(flow, false); 713 return err; 714 715 err_unlock: 716 rcu_read_unlock(); 717 err_flow_free: 718 ovs_flow_free(flow, false); 719 err_kfree_skb: 720 kfree_skb(packet); 721 err: 722 return err; 723 } 724 725 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = { 726 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN }, 727 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED }, 728 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED }, 729 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG }, 730 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 }, 731 [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 }, 732 [OVS_PACKET_ATTR_UPCALL_PID] = { .type = NLA_U32 }, 733 }; 734 735 static const struct genl_small_ops dp_packet_genl_ops[] = { 736 { .cmd = OVS_PACKET_CMD_EXECUTE, 737 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 738 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 739 .doit = ovs_packet_cmd_execute 740 } 741 }; 742 743 static struct genl_family dp_packet_genl_family __ro_after_init = { 744 .hdrsize = sizeof(struct ovs_header), 745 .name = OVS_PACKET_FAMILY, 746 .version = OVS_PACKET_VERSION, 747 .maxattr = OVS_PACKET_ATTR_MAX, 748 .policy = packet_policy, 749 .netnsok = true, 750 .parallel_ops = true, 751 .small_ops = dp_packet_genl_ops, 752 .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops), 753 .resv_start_op = OVS_PACKET_CMD_EXECUTE + 1, 754 .module = THIS_MODULE, 755 }; 756 757 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats, 758 struct ovs_dp_megaflow_stats *mega_stats) 759 { 760 int i; 761 762 memset(mega_stats, 0, sizeof(*mega_stats)); 763 764 stats->n_flows = ovs_flow_tbl_count(&dp->table); 765 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table); 766 767 stats->n_hit = stats->n_missed = stats->n_lost = 0; 768 769 for_each_possible_cpu(i) { 770 const struct dp_stats_percpu *percpu_stats; 771 struct dp_stats_percpu local_stats; 772 unsigned int start; 773 774 percpu_stats = per_cpu_ptr(dp->stats_percpu, i); 775 776 do { 777 start = u64_stats_fetch_begin(&percpu_stats->syncp); 778 local_stats = *percpu_stats; 779 } while (u64_stats_fetch_retry(&percpu_stats->syncp, start)); 780 781 stats->n_hit += local_stats.n_hit; 782 stats->n_missed += local_stats.n_missed; 783 stats->n_lost += local_stats.n_lost; 784 mega_stats->n_mask_hit += local_stats.n_mask_hit; 785 mega_stats->n_cache_hit += local_stats.n_cache_hit; 786 } 787 } 788 789 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags) 790 { 791 return ovs_identifier_is_ufid(sfid) && 792 !(ufid_flags & OVS_UFID_F_OMIT_KEY); 793 } 794 795 static bool should_fill_mask(uint32_t ufid_flags) 796 { 797 return !(ufid_flags & OVS_UFID_F_OMIT_MASK); 798 } 799 800 static bool should_fill_actions(uint32_t ufid_flags) 801 { 802 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS); 803 } 804 805 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts, 806 const struct sw_flow_id *sfid, 807 uint32_t ufid_flags) 808 { 809 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header)); 810 811 /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback 812 * see ovs_nla_put_identifier() 813 */ 814 if (sfid && ovs_identifier_is_ufid(sfid)) 815 len += nla_total_size(sfid->ufid_len); 816 else 817 len += nla_total_size(ovs_key_attr_size()); 818 819 /* OVS_FLOW_ATTR_KEY */ 820 if (!sfid || should_fill_key(sfid, ufid_flags)) 821 len += nla_total_size(ovs_key_attr_size()); 822 823 /* OVS_FLOW_ATTR_MASK */ 824 if (should_fill_mask(ufid_flags)) 825 len += nla_total_size(ovs_key_attr_size()); 826 827 /* OVS_FLOW_ATTR_ACTIONS */ 828 if (should_fill_actions(ufid_flags)) 829 len += nla_total_size(acts->orig_len); 830 831 return len 832 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */ 833 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */ 834 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */ 835 } 836 837 /* Called with ovs_mutex or RCU read lock. */ 838 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow, 839 struct sk_buff *skb) 840 { 841 struct ovs_flow_stats stats; 842 __be16 tcp_flags; 843 unsigned long used; 844 845 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags); 846 847 if (used && 848 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used), 849 OVS_FLOW_ATTR_PAD)) 850 return -EMSGSIZE; 851 852 if (stats.n_packets && 853 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS, 854 sizeof(struct ovs_flow_stats), &stats, 855 OVS_FLOW_ATTR_PAD)) 856 return -EMSGSIZE; 857 858 if ((u8)ntohs(tcp_flags) && 859 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags))) 860 return -EMSGSIZE; 861 862 return 0; 863 } 864 865 /* Called with ovs_mutex or RCU read lock. */ 866 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow, 867 struct sk_buff *skb, int skb_orig_len) 868 { 869 struct nlattr *start; 870 int err; 871 872 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if 873 * this is the first flow to be dumped into 'skb'. This is unusual for 874 * Netlink but individual action lists can be longer than 875 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this. 876 * The userspace caller can always fetch the actions separately if it 877 * really wants them. (Most userspace callers in fact don't care.) 878 * 879 * This can only fail for dump operations because the skb is always 880 * properly sized for single flows. 881 */ 882 start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS); 883 if (start) { 884 const struct sw_flow_actions *sf_acts; 885 886 sf_acts = rcu_dereference_ovsl(flow->sf_acts); 887 err = ovs_nla_put_actions(sf_acts->actions, 888 sf_acts->actions_len, skb); 889 890 if (!err) 891 nla_nest_end(skb, start); 892 else { 893 if (skb_orig_len) 894 return err; 895 896 nla_nest_cancel(skb, start); 897 } 898 } else if (skb_orig_len) { 899 return -EMSGSIZE; 900 } 901 902 return 0; 903 } 904 905 /* Called with ovs_mutex or RCU read lock. */ 906 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex, 907 struct sk_buff *skb, u32 portid, 908 u32 seq, u32 flags, u8 cmd, u32 ufid_flags) 909 { 910 const int skb_orig_len = skb->len; 911 struct ovs_header *ovs_header; 912 int err; 913 914 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, 915 flags, cmd); 916 if (!ovs_header) 917 return -EMSGSIZE; 918 919 ovs_header->dp_ifindex = dp_ifindex; 920 921 err = ovs_nla_put_identifier(flow, skb); 922 if (err) 923 goto error; 924 925 if (should_fill_key(&flow->id, ufid_flags)) { 926 err = ovs_nla_put_masked_key(flow, skb); 927 if (err) 928 goto error; 929 } 930 931 if (should_fill_mask(ufid_flags)) { 932 err = ovs_nla_put_mask(flow, skb); 933 if (err) 934 goto error; 935 } 936 937 err = ovs_flow_cmd_fill_stats(flow, skb); 938 if (err) 939 goto error; 940 941 if (should_fill_actions(ufid_flags)) { 942 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len); 943 if (err) 944 goto error; 945 } 946 947 genlmsg_end(skb, ovs_header); 948 return 0; 949 950 error: 951 genlmsg_cancel(skb, ovs_header); 952 return err; 953 } 954 955 /* May not be called with RCU read lock. */ 956 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts, 957 const struct sw_flow_id *sfid, 958 struct genl_info *info, 959 bool always, 960 uint32_t ufid_flags) 961 { 962 struct sk_buff *skb; 963 size_t len; 964 965 if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0)) 966 return NULL; 967 968 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags); 969 skb = genlmsg_new(len, GFP_KERNEL); 970 if (!skb) 971 return ERR_PTR(-ENOMEM); 972 973 return skb; 974 } 975 976 /* Called with ovs_mutex. */ 977 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow, 978 int dp_ifindex, 979 struct genl_info *info, u8 cmd, 980 bool always, u32 ufid_flags) 981 { 982 struct sk_buff *skb; 983 int retval; 984 985 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), 986 &flow->id, info, always, ufid_flags); 987 if (IS_ERR_OR_NULL(skb)) 988 return skb; 989 990 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb, 991 info->snd_portid, info->snd_seq, 0, 992 cmd, ufid_flags); 993 if (WARN_ON_ONCE(retval < 0)) { 994 kfree_skb(skb); 995 skb = ERR_PTR(retval); 996 } 997 return skb; 998 } 999 1000 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) 1001 { 1002 struct net *net = sock_net(skb->sk); 1003 struct nlattr **a = info->attrs; 1004 struct ovs_header *ovs_header = genl_info_userhdr(info); 1005 struct sw_flow *flow = NULL, *new_flow; 1006 struct sw_flow_mask mask; 1007 struct sk_buff *reply; 1008 struct datapath *dp; 1009 struct sw_flow_key *key; 1010 struct sw_flow_actions *acts; 1011 struct sw_flow_match match; 1012 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1013 int error; 1014 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1015 1016 /* Must have key and actions. */ 1017 error = -EINVAL; 1018 if (!a[OVS_FLOW_ATTR_KEY]) { 1019 OVS_NLERR(log, "Flow key attr not present in new flow."); 1020 goto error; 1021 } 1022 if (!a[OVS_FLOW_ATTR_ACTIONS]) { 1023 OVS_NLERR(log, "Flow actions attr not present in new flow."); 1024 goto error; 1025 } 1026 1027 /* Most of the time we need to allocate a new flow, do it before 1028 * locking. 1029 */ 1030 new_flow = ovs_flow_alloc(); 1031 if (IS_ERR(new_flow)) { 1032 error = PTR_ERR(new_flow); 1033 goto error; 1034 } 1035 1036 /* Extract key. */ 1037 key = kzalloc_obj(*key); 1038 if (!key) { 1039 error = -ENOMEM; 1040 goto err_kfree_flow; 1041 } 1042 1043 ovs_match_init(&match, key, false, &mask); 1044 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], 1045 a[OVS_FLOW_ATTR_MASK], log); 1046 if (error) 1047 goto err_kfree_key; 1048 1049 ovs_flow_mask_key(&new_flow->key, key, true, &mask); 1050 1051 /* Extract flow identifier. */ 1052 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID], 1053 key, log); 1054 if (error) 1055 goto err_kfree_key; 1056 1057 /* Validate actions. */ 1058 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS], 1059 &new_flow->key, &acts, log); 1060 if (error) { 1061 OVS_NLERR(log, "Flow actions may not be safe on all matching packets."); 1062 goto err_kfree_key; 1063 } 1064 1065 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false, 1066 ufid_flags); 1067 if (IS_ERR(reply)) { 1068 error = PTR_ERR(reply); 1069 goto err_kfree_acts; 1070 } 1071 1072 ovs_lock(); 1073 dp = get_dp(net, ovs_header->dp_ifindex); 1074 if (unlikely(!dp)) { 1075 error = -ENODEV; 1076 goto err_unlock_ovs; 1077 } 1078 1079 /* Check if this is a duplicate flow */ 1080 if (ovs_identifier_is_ufid(&new_flow->id)) 1081 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id); 1082 if (!flow) 1083 flow = ovs_flow_tbl_lookup(&dp->table, key); 1084 if (likely(!flow)) { 1085 rcu_assign_pointer(new_flow->sf_acts, acts); 1086 1087 /* Put flow in bucket. */ 1088 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask); 1089 if (unlikely(error)) { 1090 acts = NULL; 1091 goto err_unlock_ovs; 1092 } 1093 1094 if (unlikely(reply)) { 1095 error = ovs_flow_cmd_fill_info(new_flow, 1096 ovs_header->dp_ifindex, 1097 reply, info->snd_portid, 1098 info->snd_seq, 0, 1099 OVS_FLOW_CMD_NEW, 1100 ufid_flags); 1101 BUG_ON(error < 0); 1102 } 1103 ovs_unlock(); 1104 } else { 1105 struct sw_flow_actions *old_acts; 1106 1107 /* Bail out if we're not allowed to modify an existing flow. 1108 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL 1109 * because Generic Netlink treats the latter as a dump 1110 * request. We also accept NLM_F_EXCL in case that bug ever 1111 * gets fixed. 1112 */ 1113 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE 1114 | NLM_F_EXCL))) { 1115 error = -EEXIST; 1116 goto err_unlock_ovs; 1117 } 1118 1119 /* Look for any overlapping flow. */ 1120 if (unlikely(!ovs_flow_cmp(flow, &match))) { 1121 if (ovs_identifier_is_key(&flow->id)) 1122 flow = ovs_flow_tbl_lookup_exact(&dp->table, 1123 &match); 1124 else /* UFID matches but key is different */ 1125 flow = NULL; 1126 if (!flow) { 1127 error = -ENOENT; 1128 goto err_unlock_ovs; 1129 } 1130 } 1131 1132 if (unlikely(reply)) { 1133 size_t cur, req; 1134 1135 cur = ovs_flow_cmd_msg_size(acts, &new_flow->id, 1136 ufid_flags); 1137 req = ovs_flow_cmd_msg_size(acts, &flow->id, 1138 ufid_flags); 1139 if (cur < req) { 1140 struct sk_buff *resized; 1141 1142 resized = ovs_flow_cmd_alloc_info(acts, 1143 &flow->id, 1144 info, false, 1145 ufid_flags); 1146 if (IS_ERR(resized)) { 1147 error = PTR_ERR(resized); 1148 goto err_unlock_ovs; 1149 } 1150 kfree_skb(reply); 1151 reply = resized; 1152 } 1153 } 1154 1155 /* Update actions. */ 1156 old_acts = ovsl_dereference(flow->sf_acts); 1157 rcu_assign_pointer(flow->sf_acts, acts); 1158 1159 if (unlikely(reply)) { 1160 error = ovs_flow_cmd_fill_info(flow, 1161 ovs_header->dp_ifindex, 1162 reply, info->snd_portid, 1163 info->snd_seq, 0, 1164 OVS_FLOW_CMD_NEW, 1165 ufid_flags); 1166 BUG_ON(error < 0); 1167 } 1168 ovs_unlock(); 1169 1170 ovs_nla_free_flow_actions_rcu(old_acts); 1171 ovs_flow_free(new_flow, false); 1172 } 1173 1174 if (reply) 1175 ovs_notify(&dp_flow_genl_family, reply, info); 1176 1177 kfree(key); 1178 return 0; 1179 1180 err_unlock_ovs: 1181 ovs_unlock(); 1182 kfree_skb(reply); 1183 err_kfree_acts: 1184 ovs_nla_free_flow_actions(acts); 1185 err_kfree_key: 1186 kfree(key); 1187 err_kfree_flow: 1188 ovs_flow_free(new_flow, false); 1189 error: 1190 return error; 1191 } 1192 1193 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */ 1194 static noinline_for_stack 1195 struct sw_flow_actions *get_flow_actions(struct net *net, 1196 const struct nlattr *a, 1197 const struct sw_flow_key *key, 1198 const struct sw_flow_mask *mask, 1199 bool log) 1200 { 1201 struct sw_flow_actions *acts; 1202 struct sw_flow_key masked_key; 1203 int error; 1204 1205 ovs_flow_mask_key(&masked_key, key, true, mask); 1206 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log); 1207 if (error) { 1208 OVS_NLERR(log, 1209 "Actions may not be safe on all matching packets"); 1210 return ERR_PTR(error); 1211 } 1212 1213 return acts; 1214 } 1215 1216 /* Factor out match-init and action-copy to avoid 1217 * "Wframe-larger-than=1024" warning. Because mask is only 1218 * used to get actions, we new a function to save some 1219 * stack space. 1220 * 1221 * If there are not key and action attrs, we return 0 1222 * directly. In the case, the caller will also not use the 1223 * match as before. If there is action attr, we try to get 1224 * actions and save them to *acts. Before returning from 1225 * the function, we reset the match->mask pointer. Because 1226 * we should not to return match object with dangling reference 1227 * to mask. 1228 * */ 1229 static noinline_for_stack int 1230 ovs_nla_init_match_and_action(struct net *net, 1231 struct sw_flow_match *match, 1232 struct sw_flow_key *key, 1233 struct nlattr **a, 1234 struct sw_flow_actions **acts, 1235 bool log) 1236 { 1237 struct sw_flow_mask mask; 1238 int error = 0; 1239 1240 if (a[OVS_FLOW_ATTR_KEY]) { 1241 ovs_match_init(match, key, true, &mask); 1242 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY], 1243 a[OVS_FLOW_ATTR_MASK], log); 1244 if (error) 1245 goto error; 1246 } 1247 1248 if (a[OVS_FLOW_ATTR_ACTIONS]) { 1249 if (!a[OVS_FLOW_ATTR_KEY]) { 1250 OVS_NLERR(log, 1251 "Flow key attribute not present in set flow."); 1252 error = -EINVAL; 1253 goto error; 1254 } 1255 1256 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key, 1257 &mask, log); 1258 if (IS_ERR(*acts)) { 1259 error = PTR_ERR(*acts); 1260 goto error; 1261 } 1262 } 1263 1264 /* On success, error is 0. */ 1265 error: 1266 match->mask = NULL; 1267 return error; 1268 } 1269 1270 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) 1271 { 1272 struct net *net = sock_net(skb->sk); 1273 struct nlattr **a = info->attrs; 1274 struct ovs_header *ovs_header = genl_info_userhdr(info); 1275 struct sw_flow_key key; 1276 struct sw_flow *flow; 1277 struct sk_buff *reply = NULL; 1278 struct datapath *dp; 1279 struct sw_flow_actions *old_acts = NULL, *acts = NULL; 1280 struct sw_flow_match match; 1281 struct sw_flow_id sfid; 1282 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1283 int error = 0; 1284 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1285 bool ufid_present; 1286 1287 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log); 1288 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) { 1289 OVS_NLERR(log, 1290 "Flow set message rejected, Key attribute missing."); 1291 return -EINVAL; 1292 } 1293 1294 error = ovs_nla_init_match_and_action(net, &match, &key, a, 1295 &acts, log); 1296 if (error) 1297 goto error; 1298 1299 if (acts) { 1300 /* Can allocate before locking if have acts. */ 1301 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false, 1302 ufid_flags); 1303 if (IS_ERR(reply)) { 1304 error = PTR_ERR(reply); 1305 goto err_kfree_acts; 1306 } 1307 } 1308 1309 ovs_lock(); 1310 dp = get_dp(net, ovs_header->dp_ifindex); 1311 if (unlikely(!dp)) { 1312 error = -ENODEV; 1313 goto err_unlock_ovs; 1314 } 1315 /* Check that the flow exists. */ 1316 if (ufid_present) 1317 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid); 1318 else 1319 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1320 if (unlikely(!flow)) { 1321 error = -ENOENT; 1322 goto err_unlock_ovs; 1323 } 1324 1325 /* Update actions, if present. */ 1326 if (likely(acts)) { 1327 old_acts = ovsl_dereference(flow->sf_acts); 1328 rcu_assign_pointer(flow->sf_acts, acts); 1329 1330 if (unlikely(reply)) { 1331 error = ovs_flow_cmd_fill_info(flow, 1332 ovs_header->dp_ifindex, 1333 reply, info->snd_portid, 1334 info->snd_seq, 0, 1335 OVS_FLOW_CMD_SET, 1336 ufid_flags); 1337 BUG_ON(error < 0); 1338 } 1339 } else { 1340 /* Could not alloc without acts before locking. */ 1341 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, 1342 info, OVS_FLOW_CMD_SET, false, 1343 ufid_flags); 1344 1345 if (IS_ERR(reply)) { 1346 error = PTR_ERR(reply); 1347 reply = NULL; 1348 goto err_unlock_ovs; 1349 } 1350 } 1351 1352 /* Clear stats. */ 1353 if (a[OVS_FLOW_ATTR_CLEAR]) 1354 ovs_flow_stats_clear(flow); 1355 ovs_unlock(); 1356 1357 if (reply) 1358 ovs_notify(&dp_flow_genl_family, reply, info); 1359 if (old_acts) 1360 ovs_nla_free_flow_actions_rcu(old_acts); 1361 1362 return 0; 1363 1364 err_unlock_ovs: 1365 ovs_unlock(); 1366 kfree_skb(reply); 1367 err_kfree_acts: 1368 ovs_nla_free_flow_actions(acts); 1369 error: 1370 return error; 1371 } 1372 1373 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) 1374 { 1375 struct nlattr **a = info->attrs; 1376 struct ovs_header *ovs_header = genl_info_userhdr(info); 1377 struct net *net = sock_net(skb->sk); 1378 struct sw_flow_key key; 1379 struct sk_buff *reply; 1380 struct sw_flow *flow; 1381 struct datapath *dp; 1382 struct sw_flow_match match; 1383 struct sw_flow_id ufid; 1384 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1385 int err = 0; 1386 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1387 bool ufid_present; 1388 1389 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log); 1390 if (a[OVS_FLOW_ATTR_KEY]) { 1391 ovs_match_init(&match, &key, true, NULL); 1392 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL, 1393 log); 1394 } else if (!ufid_present) { 1395 OVS_NLERR(log, 1396 "Flow get message rejected, Key attribute missing."); 1397 err = -EINVAL; 1398 } 1399 if (err) 1400 return err; 1401 1402 ovs_lock(); 1403 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 1404 if (!dp) { 1405 err = -ENODEV; 1406 goto unlock; 1407 } 1408 1409 if (ufid_present) 1410 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid); 1411 else 1412 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1413 if (!flow) { 1414 err = -ENOENT; 1415 goto unlock; 1416 } 1417 1418 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info, 1419 OVS_FLOW_CMD_GET, true, ufid_flags); 1420 if (IS_ERR(reply)) { 1421 err = PTR_ERR(reply); 1422 goto unlock; 1423 } 1424 1425 ovs_unlock(); 1426 return genlmsg_reply(reply, info); 1427 unlock: 1428 ovs_unlock(); 1429 return err; 1430 } 1431 1432 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) 1433 { 1434 struct nlattr **a = info->attrs; 1435 struct ovs_header *ovs_header = genl_info_userhdr(info); 1436 struct net *net = sock_net(skb->sk); 1437 struct sw_flow_key key; 1438 struct sk_buff *reply; 1439 struct sw_flow *flow = NULL; 1440 struct datapath *dp; 1441 struct sw_flow_match match; 1442 struct sw_flow_id ufid; 1443 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1444 int err; 1445 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1446 bool ufid_present; 1447 1448 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log); 1449 if (a[OVS_FLOW_ATTR_KEY]) { 1450 ovs_match_init(&match, &key, true, NULL); 1451 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], 1452 NULL, log); 1453 if (unlikely(err)) 1454 return err; 1455 } 1456 1457 ovs_lock(); 1458 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 1459 if (unlikely(!dp)) { 1460 err = -ENODEV; 1461 goto unlock; 1462 } 1463 1464 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) { 1465 err = ovs_flow_tbl_flush(&dp->table); 1466 goto unlock; 1467 } 1468 1469 if (ufid_present) 1470 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid); 1471 else 1472 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1473 if (unlikely(!flow)) { 1474 err = -ENOENT; 1475 goto unlock; 1476 } 1477 1478 reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), 1479 &flow->id, info, false, ufid_flags); 1480 if (IS_ERR(reply)) { 1481 netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, 1482 PTR_ERR(reply)); 1483 reply = NULL; 1484 } 1485 1486 if (likely(reply)) { 1487 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, 1488 reply, info->snd_portid, 1489 info->snd_seq, 0, 1490 OVS_FLOW_CMD_DEL, ufid_flags); 1491 if (WARN_ON_ONCE(err < 0)) { 1492 kfree_skb(reply); 1493 reply = NULL; 1494 } 1495 } 1496 /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses 1497 * the flow->mask that can be scheduled to be freed by the 1498 * ovs_flow_tbl_remove() and we're not holding the RCU read lock. 1499 */ 1500 ovs_flow_tbl_remove(&dp->table, flow); 1501 ovs_unlock(); 1502 1503 if (likely(reply)) 1504 ovs_notify(&dp_flow_genl_family, reply, info); 1505 1506 ovs_flow_free(flow, true); 1507 return 0; 1508 unlock: 1509 ovs_unlock(); 1510 return err; 1511 } 1512 1513 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 1514 { 1515 struct nlattr *a[__OVS_FLOW_ATTR_MAX]; 1516 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh)); 1517 struct table_instance *ti; 1518 struct datapath *dp; 1519 u32 ufid_flags; 1520 int err; 1521 1522 err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a, 1523 OVS_FLOW_ATTR_MAX, flow_policy, NULL); 1524 if (err) 1525 return err; 1526 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1527 1528 rcu_read_lock(); 1529 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex); 1530 if (!dp) { 1531 rcu_read_unlock(); 1532 return -ENODEV; 1533 } 1534 1535 /* 1536 * Not needed for safety. Stops every spin_unlock_bh() in 1537 * ovs_flow_stats_get() from running the softirq backlog. 1538 */ 1539 local_bh_disable(); 1540 1541 ti = rcu_dereference(dp->table.ti); 1542 for (;;) { 1543 struct sw_flow *flow; 1544 u32 bucket, obj; 1545 1546 bucket = cb->args[0]; 1547 obj = cb->args[1]; 1548 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj); 1549 if (!flow) 1550 break; 1551 1552 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb, 1553 NETLINK_CB(cb->skb).portid, 1554 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1555 OVS_FLOW_CMD_GET, ufid_flags) < 0) 1556 break; 1557 1558 cb->args[0] = bucket; 1559 cb->args[1] = obj; 1560 } 1561 1562 local_bh_enable(); 1563 rcu_read_unlock(); 1564 return skb->len; 1565 } 1566 1567 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = { 1568 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED }, 1569 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED }, 1570 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED }, 1571 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG }, 1572 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG }, 1573 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 }, 1574 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 }, 1575 }; 1576 1577 static const struct genl_small_ops dp_flow_genl_ops[] = { 1578 { .cmd = OVS_FLOW_CMD_NEW, 1579 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1580 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1581 .doit = ovs_flow_cmd_new 1582 }, 1583 { .cmd = OVS_FLOW_CMD_DEL, 1584 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1585 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1586 .doit = ovs_flow_cmd_del 1587 }, 1588 { .cmd = OVS_FLOW_CMD_GET, 1589 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1590 .flags = 0, /* OK for unprivileged users. */ 1591 .doit = ovs_flow_cmd_get, 1592 .dumpit = ovs_flow_cmd_dump 1593 }, 1594 { .cmd = OVS_FLOW_CMD_SET, 1595 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1596 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1597 .doit = ovs_flow_cmd_set, 1598 }, 1599 }; 1600 1601 static struct genl_family dp_flow_genl_family __ro_after_init = { 1602 .hdrsize = sizeof(struct ovs_header), 1603 .name = OVS_FLOW_FAMILY, 1604 .version = OVS_FLOW_VERSION, 1605 .maxattr = OVS_FLOW_ATTR_MAX, 1606 .policy = flow_policy, 1607 .netnsok = true, 1608 .parallel_ops = true, 1609 .small_ops = dp_flow_genl_ops, 1610 .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops), 1611 .resv_start_op = OVS_FLOW_CMD_SET + 1, 1612 .mcgrps = &ovs_dp_flow_multicast_group, 1613 .n_mcgrps = 1, 1614 .module = THIS_MODULE, 1615 }; 1616 1617 static size_t ovs_dp_cmd_msg_size(void) 1618 { 1619 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header)); 1620 1621 msgsize += nla_total_size(IFNAMSIZ); 1622 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats)); 1623 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats)); 1624 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */ 1625 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */ 1626 msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */ 1627 1628 return msgsize; 1629 } 1630 1631 /* Called with ovs_mutex. */ 1632 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, 1633 u32 portid, u32 seq, u32 flags, u8 cmd) 1634 { 1635 struct ovs_header *ovs_header; 1636 struct ovs_dp_stats dp_stats; 1637 struct ovs_dp_megaflow_stats dp_megaflow_stats; 1638 struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids); 1639 int err, pids_len; 1640 1641 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family, 1642 flags, cmd); 1643 if (!ovs_header) 1644 goto error; 1645 1646 ovs_header->dp_ifindex = get_dpifindex(dp); 1647 1648 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp)); 1649 if (err) 1650 goto nla_put_failure; 1651 1652 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats); 1653 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats), 1654 &dp_stats, OVS_DP_ATTR_PAD)) 1655 goto nla_put_failure; 1656 1657 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS, 1658 sizeof(struct ovs_dp_megaflow_stats), 1659 &dp_megaflow_stats, OVS_DP_ATTR_PAD)) 1660 goto nla_put_failure; 1661 1662 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features)) 1663 goto nla_put_failure; 1664 1665 if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE, 1666 ovs_flow_tbl_masks_cache_size(&dp->table))) 1667 goto nla_put_failure; 1668 1669 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) { 1670 pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32); 1671 if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids)) 1672 goto nla_put_failure; 1673 } 1674 1675 genlmsg_end(skb, ovs_header); 1676 return 0; 1677 1678 nla_put_failure: 1679 genlmsg_cancel(skb, ovs_header); 1680 error: 1681 return -EMSGSIZE; 1682 } 1683 1684 static struct sk_buff *ovs_dp_cmd_alloc_info(void) 1685 { 1686 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL); 1687 } 1688 1689 /* Called with rcu_read_lock or ovs_mutex. */ 1690 static struct datapath *lookup_datapath(struct net *net, 1691 const struct ovs_header *ovs_header, 1692 struct nlattr *a[OVS_DP_ATTR_MAX + 1]) 1693 { 1694 struct datapath *dp; 1695 1696 if (!a[OVS_DP_ATTR_NAME]) 1697 dp = get_dp(net, ovs_header->dp_ifindex); 1698 else { 1699 struct vport *vport; 1700 1701 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME])); 1702 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL; 1703 } 1704 return dp ? dp : ERR_PTR(-ENODEV); 1705 } 1706 1707 static void ovs_dp_reset_user_features(struct sk_buff *skb, 1708 struct genl_info *info) 1709 { 1710 struct datapath *dp; 1711 1712 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 1713 info->attrs); 1714 if (IS_ERR(dp)) 1715 return; 1716 1717 pr_warn("%s: Dropping previously announced user features\n", 1718 ovs_dp_name(dp)); 1719 dp->user_features = 0; 1720 } 1721 1722 static int ovs_dp_set_upcall_portids(struct datapath *dp, 1723 const struct nlattr *ids) 1724 { 1725 struct dp_nlsk_pids *old, *dp_nlsk_pids; 1726 1727 if (!nla_len(ids) || nla_len(ids) % sizeof(u32)) 1728 return -EINVAL; 1729 1730 old = ovsl_dereference(dp->upcall_portids); 1731 1732 dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids), 1733 GFP_KERNEL); 1734 if (!dp_nlsk_pids) 1735 return -ENOMEM; 1736 1737 dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32); 1738 nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids)); 1739 1740 rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids); 1741 1742 kfree_rcu(old, rcu); 1743 1744 return 0; 1745 } 1746 1747 u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id) 1748 { 1749 struct dp_nlsk_pids *dp_nlsk_pids; 1750 1751 dp_nlsk_pids = rcu_dereference(dp->upcall_portids); 1752 1753 if (dp_nlsk_pids) { 1754 if (cpu_id < dp_nlsk_pids->n_pids) { 1755 return dp_nlsk_pids->pids[cpu_id]; 1756 } else if (dp_nlsk_pids->n_pids > 0 && 1757 cpu_id >= dp_nlsk_pids->n_pids) { 1758 /* If the number of netlink PIDs is mismatched with 1759 * the number of CPUs as seen by the kernel, log this 1760 * and send the upcall to an arbitrary socket (0) in 1761 * order to not drop packets 1762 */ 1763 pr_info_ratelimited("cpu_id mismatch with handler threads"); 1764 return dp_nlsk_pids->pids[cpu_id % 1765 dp_nlsk_pids->n_pids]; 1766 } else { 1767 return 0; 1768 } 1769 } else { 1770 return 0; 1771 } 1772 } 1773 1774 static int ovs_dp_change(struct datapath *dp, struct nlattr *a[]) 1775 { 1776 u32 user_features = 0, old_features = dp->user_features; 1777 int err; 1778 1779 if (a[OVS_DP_ATTR_USER_FEATURES]) { 1780 user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]); 1781 1782 if (user_features & ~(OVS_DP_F_VPORT_PIDS | 1783 OVS_DP_F_UNALIGNED | 1784 OVS_DP_F_TC_RECIRC_SHARING | 1785 OVS_DP_F_DISPATCH_UPCALL_PER_CPU)) 1786 return -EOPNOTSUPP; 1787 1788 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 1789 if (user_features & OVS_DP_F_TC_RECIRC_SHARING) 1790 return -EOPNOTSUPP; 1791 #endif 1792 } 1793 1794 if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) { 1795 int err; 1796 u32 cache_size; 1797 1798 cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]); 1799 err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size); 1800 if (err) 1801 return err; 1802 } 1803 1804 dp->user_features = user_features; 1805 1806 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && 1807 a[OVS_DP_ATTR_PER_CPU_PIDS]) { 1808 /* Upcall Netlink Port IDs have been updated */ 1809 err = ovs_dp_set_upcall_portids(dp, 1810 a[OVS_DP_ATTR_PER_CPU_PIDS]); 1811 if (err) 1812 return err; 1813 } 1814 1815 if ((dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) && 1816 !(old_features & OVS_DP_F_TC_RECIRC_SHARING)) 1817 tc_skb_ext_tc_enable(); 1818 else if (!(dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) && 1819 (old_features & OVS_DP_F_TC_RECIRC_SHARING)) 1820 tc_skb_ext_tc_disable(); 1821 1822 return 0; 1823 } 1824 1825 static int ovs_dp_stats_init(struct datapath *dp) 1826 { 1827 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu); 1828 if (!dp->stats_percpu) 1829 return -ENOMEM; 1830 1831 return 0; 1832 } 1833 1834 static int ovs_dp_vport_init(struct datapath *dp) 1835 { 1836 int i; 1837 1838 dp->ports = kmalloc_objs(struct hlist_head, DP_VPORT_HASH_BUCKETS); 1839 if (!dp->ports) 1840 return -ENOMEM; 1841 1842 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) 1843 INIT_HLIST_HEAD(&dp->ports[i]); 1844 1845 return 0; 1846 } 1847 1848 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) 1849 { 1850 struct nlattr **a = info->attrs; 1851 struct vport_parms parms; 1852 struct sk_buff *reply; 1853 struct datapath *dp; 1854 struct vport *vport; 1855 struct ovs_net *ovs_net; 1856 int err; 1857 1858 err = -EINVAL; 1859 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID]) 1860 goto err; 1861 1862 reply = ovs_dp_cmd_alloc_info(); 1863 if (!reply) 1864 return -ENOMEM; 1865 1866 err = -ENOMEM; 1867 dp = kzalloc_obj(*dp); 1868 if (dp == NULL) 1869 goto err_destroy_reply; 1870 1871 ovs_dp_set_net(dp, sock_net(skb->sk)); 1872 1873 /* Allocate table. */ 1874 err = ovs_flow_tbl_init(&dp->table); 1875 if (err) 1876 goto err_destroy_dp; 1877 1878 err = ovs_dp_stats_init(dp); 1879 if (err) 1880 goto err_destroy_table; 1881 1882 err = ovs_dp_vport_init(dp); 1883 if (err) 1884 goto err_destroy_stats; 1885 1886 err = ovs_meters_init(dp); 1887 if (err) 1888 goto err_destroy_ports; 1889 1890 /* Set up our datapath device. */ 1891 parms.name = nla_data(a[OVS_DP_ATTR_NAME]); 1892 parms.type = OVS_VPORT_TYPE_INTERNAL; 1893 parms.dp = dp; 1894 parms.port_no = OVSP_LOCAL; 1895 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID]; 1896 parms.desired_ifindex = nla_get_s32_default(a[OVS_DP_ATTR_IFINDEX], 0); 1897 1898 /* So far only local changes have been made, now need the lock. */ 1899 ovs_lock(); 1900 1901 err = ovs_dp_change(dp, a); 1902 if (err) 1903 goto err_unlock_and_destroy_meters; 1904 1905 vport = new_vport(&parms); 1906 if (IS_ERR(vport)) { 1907 err = PTR_ERR(vport); 1908 if (err == -EBUSY) 1909 err = -EEXIST; 1910 1911 if (err == -EEXIST) { 1912 /* An outdated user space instance that does not understand 1913 * the concept of user_features has attempted to create a new 1914 * datapath and is likely to reuse it. Drop all user features. 1915 */ 1916 if (info->genlhdr->version < OVS_DP_VER_FEATURES) 1917 ovs_dp_reset_user_features(skb, info); 1918 } 1919 1920 goto err_destroy_portids; 1921 } 1922 1923 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 1924 info->snd_seq, 0, OVS_DP_CMD_NEW); 1925 BUG_ON(err < 0); 1926 1927 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id); 1928 list_add_tail_rcu(&dp->list_node, &ovs_net->dps); 1929 1930 ovs_unlock(); 1931 1932 ovs_notify(&dp_datapath_genl_family, reply, info); 1933 return 0; 1934 1935 err_destroy_portids: 1936 kfree(rcu_dereference_raw(dp->upcall_portids)); 1937 err_unlock_and_destroy_meters: 1938 ovs_unlock(); 1939 ovs_meters_exit(dp); 1940 err_destroy_ports: 1941 kfree(dp->ports); 1942 err_destroy_stats: 1943 free_percpu(dp->stats_percpu); 1944 err_destroy_table: 1945 ovs_flow_tbl_destroy(&dp->table); 1946 err_destroy_dp: 1947 kfree(dp); 1948 err_destroy_reply: 1949 kfree_skb(reply); 1950 err: 1951 return err; 1952 } 1953 1954 /* Called with ovs_mutex. */ 1955 static void __dp_destroy(struct datapath *dp) 1956 { 1957 struct flow_table *table = &dp->table; 1958 int i; 1959 1960 if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) 1961 tc_skb_ext_tc_disable(); 1962 1963 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 1964 struct vport *vport; 1965 struct hlist_node *n; 1966 1967 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node) 1968 if (vport->port_no != OVSP_LOCAL) 1969 ovs_dp_detach_port(vport); 1970 } 1971 1972 list_del_rcu(&dp->list_node); 1973 1974 /* OVSP_LOCAL is datapath internal port. We need to make sure that 1975 * all ports in datapath are destroyed first before freeing datapath. 1976 */ 1977 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL)); 1978 1979 /* Flush sw_flow in the tables. RCU cb only releases resource 1980 * such as dp, ports and tables. That may avoid some issues 1981 * such as RCU usage warning. 1982 */ 1983 table_instance_flow_flush(table, ovsl_dereference(table->ti), 1984 ovsl_dereference(table->ufid_ti)); 1985 1986 /* RCU destroy the ports, meters and flow tables. */ 1987 call_rcu(&dp->rcu, destroy_dp_rcu); 1988 } 1989 1990 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info) 1991 { 1992 struct sk_buff *reply; 1993 struct datapath *dp; 1994 int err; 1995 1996 reply = ovs_dp_cmd_alloc_info(); 1997 if (!reply) 1998 return -ENOMEM; 1999 2000 ovs_lock(); 2001 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 2002 info->attrs); 2003 err = PTR_ERR(dp); 2004 if (IS_ERR(dp)) 2005 goto err_unlock_free; 2006 2007 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2008 info->snd_seq, 0, OVS_DP_CMD_DEL); 2009 BUG_ON(err < 0); 2010 2011 __dp_destroy(dp); 2012 ovs_unlock(); 2013 2014 ovs_notify(&dp_datapath_genl_family, reply, info); 2015 2016 return 0; 2017 2018 err_unlock_free: 2019 ovs_unlock(); 2020 kfree_skb(reply); 2021 return err; 2022 } 2023 2024 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info) 2025 { 2026 struct sk_buff *reply; 2027 struct datapath *dp; 2028 int err; 2029 2030 reply = ovs_dp_cmd_alloc_info(); 2031 if (!reply) 2032 return -ENOMEM; 2033 2034 ovs_lock(); 2035 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 2036 info->attrs); 2037 err = PTR_ERR(dp); 2038 if (IS_ERR(dp)) 2039 goto err_unlock_free; 2040 2041 err = ovs_dp_change(dp, info->attrs); 2042 if (err) 2043 goto err_unlock_free; 2044 2045 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2046 info->snd_seq, 0, OVS_DP_CMD_SET); 2047 BUG_ON(err < 0); 2048 2049 ovs_unlock(); 2050 ovs_notify(&dp_datapath_genl_family, reply, info); 2051 2052 return 0; 2053 2054 err_unlock_free: 2055 ovs_unlock(); 2056 kfree_skb(reply); 2057 return err; 2058 } 2059 2060 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info) 2061 { 2062 struct sk_buff *reply; 2063 struct datapath *dp; 2064 int err; 2065 2066 reply = ovs_dp_cmd_alloc_info(); 2067 if (!reply) 2068 return -ENOMEM; 2069 2070 ovs_lock(); 2071 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 2072 info->attrs); 2073 if (IS_ERR(dp)) { 2074 err = PTR_ERR(dp); 2075 goto err_unlock_free; 2076 } 2077 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2078 info->snd_seq, 0, OVS_DP_CMD_GET); 2079 BUG_ON(err < 0); 2080 ovs_unlock(); 2081 2082 return genlmsg_reply(reply, info); 2083 2084 err_unlock_free: 2085 ovs_unlock(); 2086 kfree_skb(reply); 2087 return err; 2088 } 2089 2090 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 2091 { 2092 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); 2093 struct datapath *dp; 2094 int skip = cb->args[0]; 2095 int i = 0; 2096 2097 ovs_lock(); 2098 list_for_each_entry(dp, &ovs_net->dps, list_node) { 2099 if (i >= skip && 2100 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid, 2101 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2102 OVS_DP_CMD_GET) < 0) 2103 break; 2104 i++; 2105 } 2106 ovs_unlock(); 2107 2108 cb->args[0] = i; 2109 2110 return skb->len; 2111 } 2112 2113 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = { 2114 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, 2115 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 }, 2116 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 }, 2117 [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0, 2118 PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)), 2119 [OVS_DP_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0), 2120 }; 2121 2122 static const struct genl_small_ops dp_datapath_genl_ops[] = { 2123 { .cmd = OVS_DP_CMD_NEW, 2124 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2125 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2126 .doit = ovs_dp_cmd_new 2127 }, 2128 { .cmd = OVS_DP_CMD_DEL, 2129 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2130 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2131 .doit = ovs_dp_cmd_del 2132 }, 2133 { .cmd = OVS_DP_CMD_GET, 2134 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2135 .flags = 0, /* OK for unprivileged users. */ 2136 .doit = ovs_dp_cmd_get, 2137 .dumpit = ovs_dp_cmd_dump 2138 }, 2139 { .cmd = OVS_DP_CMD_SET, 2140 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2141 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2142 .doit = ovs_dp_cmd_set, 2143 }, 2144 }; 2145 2146 static struct genl_family dp_datapath_genl_family __ro_after_init = { 2147 .hdrsize = sizeof(struct ovs_header), 2148 .name = OVS_DATAPATH_FAMILY, 2149 .version = OVS_DATAPATH_VERSION, 2150 .maxattr = OVS_DP_ATTR_MAX, 2151 .policy = datapath_policy, 2152 .netnsok = true, 2153 .parallel_ops = true, 2154 .small_ops = dp_datapath_genl_ops, 2155 .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops), 2156 .resv_start_op = OVS_DP_CMD_SET + 1, 2157 .mcgrps = &ovs_dp_datapath_multicast_group, 2158 .n_mcgrps = 1, 2159 .module = THIS_MODULE, 2160 }; 2161 2162 /* Called with ovs_mutex or RCU read lock. */ 2163 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb, 2164 struct net *net, u32 portid, u32 seq, 2165 u32 flags, u8 cmd, gfp_t gfp) 2166 { 2167 struct ovs_header *ovs_header; 2168 struct ovs_vport_stats vport_stats; 2169 struct net *net_vport; 2170 int err; 2171 2172 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family, 2173 flags, cmd); 2174 if (!ovs_header) 2175 return -EMSGSIZE; 2176 2177 ovs_header->dp_ifindex = get_dpifindex(vport->dp); 2178 2179 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) || 2180 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) || 2181 nla_put_string(skb, OVS_VPORT_ATTR_NAME, 2182 ovs_vport_name(vport)) || 2183 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex)) 2184 goto nla_put_failure; 2185 2186 rcu_read_lock(); 2187 net_vport = dev_net_rcu(vport->dev); 2188 if (!net_eq(net, net_vport)) { 2189 int id = peernet2id_alloc(net, net_vport, GFP_ATOMIC); 2190 2191 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id)) 2192 goto nla_put_failure_unlock; 2193 } 2194 rcu_read_unlock(); 2195 2196 ovs_vport_get_stats(vport, &vport_stats); 2197 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS, 2198 sizeof(struct ovs_vport_stats), &vport_stats, 2199 OVS_VPORT_ATTR_PAD)) 2200 goto nla_put_failure; 2201 2202 if (ovs_vport_get_upcall_stats(vport, skb)) 2203 goto nla_put_failure; 2204 2205 if (ovs_vport_get_upcall_portids(vport, skb)) 2206 goto nla_put_failure; 2207 2208 genlmsg_end(skb, ovs_header); 2209 return 0; 2210 2211 nla_put_failure_unlock: 2212 rcu_read_unlock(); 2213 nla_put_failure: 2214 err = -EMSGSIZE; 2215 genlmsg_cancel(skb, ovs_header); 2216 return err; 2217 } 2218 2219 static size_t ovs_vport_cmd_msg_size(void) 2220 { 2221 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header)); 2222 2223 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_PORT_NO */ 2224 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_TYPE */ 2225 msgsize += nla_total_size(IFNAMSIZ); /* OVS_VPORT_ATTR_NAME */ 2226 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_IFINDEX */ 2227 msgsize += nla_total_size(sizeof(s32)); /* OVS_VPORT_ATTR_NETNSID */ 2228 2229 /* OVS_VPORT_ATTR_STATS */ 2230 msgsize += nla_total_size_64bit(sizeof(struct ovs_vport_stats)); 2231 2232 /* OVS_VPORT_ATTR_UPCALL_STATS(OVS_VPORT_UPCALL_ATTR_SUCCESS + 2233 * OVS_VPORT_UPCALL_ATTR_FAIL) 2234 */ 2235 msgsize += nla_total_size(nla_total_size_64bit(sizeof(u64)) + 2236 nla_total_size_64bit(sizeof(u64))); 2237 2238 /* OVS_VPORT_ATTR_UPCALL_PID */ 2239 msgsize += nla_total_size(nr_cpu_ids * sizeof(u32)); 2240 2241 return msgsize; 2242 } 2243 2244 static struct sk_buff *ovs_vport_cmd_alloc_info(void) 2245 { 2246 return genlmsg_new(ovs_vport_cmd_msg_size(), GFP_KERNEL); 2247 } 2248 2249 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */ 2250 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net, 2251 u32 portid, u32 seq, u8 cmd) 2252 { 2253 struct sk_buff *skb; 2254 int retval; 2255 2256 skb = ovs_vport_cmd_alloc_info(); 2257 if (!skb) 2258 return ERR_PTR(-ENOMEM); 2259 2260 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd, 2261 GFP_KERNEL); 2262 BUG_ON(retval < 0); 2263 2264 return skb; 2265 } 2266 2267 /* Called with ovs_mutex or RCU read lock. */ 2268 static struct vport *lookup_vport(struct net *net, 2269 const struct ovs_header *ovs_header, 2270 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1]) 2271 { 2272 struct datapath *dp; 2273 struct vport *vport; 2274 2275 if (a[OVS_VPORT_ATTR_IFINDEX]) 2276 return ERR_PTR(-EOPNOTSUPP); 2277 if (a[OVS_VPORT_ATTR_NAME]) { 2278 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME])); 2279 if (!vport) 2280 return ERR_PTR(-ENODEV); 2281 if (ovs_header->dp_ifindex && 2282 ovs_header->dp_ifindex != get_dpifindex(vport->dp)) 2283 return ERR_PTR(-ENODEV); 2284 return vport; 2285 } else if (a[OVS_VPORT_ATTR_PORT_NO]) { 2286 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]); 2287 2288 if (port_no >= DP_MAX_PORTS) 2289 return ERR_PTR(-EFBIG); 2290 2291 dp = get_dp(net, ovs_header->dp_ifindex); 2292 if (!dp) 2293 return ERR_PTR(-ENODEV); 2294 2295 vport = ovs_vport_ovsl_rcu(dp, port_no); 2296 if (!vport) 2297 return ERR_PTR(-ENODEV); 2298 return vport; 2299 } else 2300 return ERR_PTR(-EINVAL); 2301 2302 } 2303 2304 static unsigned int ovs_get_max_headroom(struct datapath *dp) 2305 { 2306 unsigned int dev_headroom, max_headroom = 0; 2307 struct net_device *dev; 2308 struct vport *vport; 2309 int i; 2310 2311 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2312 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node, 2313 lockdep_ovsl_is_held()) { 2314 dev = vport->dev; 2315 dev_headroom = netdev_get_fwd_headroom(dev); 2316 if (dev_headroom > max_headroom) 2317 max_headroom = dev_headroom; 2318 } 2319 } 2320 2321 return max_headroom; 2322 } 2323 2324 /* Called with ovs_mutex */ 2325 static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom) 2326 { 2327 struct vport *vport; 2328 int i; 2329 2330 dp->max_headroom = new_headroom; 2331 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2332 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node, 2333 lockdep_ovsl_is_held()) 2334 netdev_set_rx_headroom(vport->dev, new_headroom); 2335 } 2336 } 2337 2338 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info) 2339 { 2340 struct nlattr **a = info->attrs; 2341 struct ovs_header *ovs_header = genl_info_userhdr(info); 2342 struct vport_parms parms; 2343 struct sk_buff *reply; 2344 struct vport *vport; 2345 struct datapath *dp; 2346 unsigned int new_headroom; 2347 u32 port_no; 2348 int err; 2349 2350 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] || 2351 !a[OVS_VPORT_ATTR_UPCALL_PID]) 2352 return -EINVAL; 2353 2354 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]); 2355 2356 if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL) 2357 return -EOPNOTSUPP; 2358 2359 port_no = nla_get_u32_default(a[OVS_VPORT_ATTR_PORT_NO], 0); 2360 if (port_no >= DP_MAX_PORTS) 2361 return -EFBIG; 2362 2363 reply = ovs_vport_cmd_alloc_info(); 2364 if (!reply) 2365 return -ENOMEM; 2366 2367 ovs_lock(); 2368 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 2369 err = -ENODEV; 2370 if (!dp) 2371 goto exit_unlock_free; 2372 2373 if (port_no) { 2374 vport = ovs_vport_ovsl(dp, port_no); 2375 err = -EBUSY; 2376 if (vport) 2377 goto exit_unlock_free; 2378 } else { 2379 for (port_no = 1; ; port_no++) { 2380 if (port_no >= DP_MAX_PORTS) { 2381 err = -EFBIG; 2382 goto exit_unlock_free; 2383 } 2384 vport = ovs_vport_ovsl(dp, port_no); 2385 if (!vport) 2386 break; 2387 } 2388 } 2389 2390 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]); 2391 parms.dp = dp; 2392 parms.port_no = port_no; 2393 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID]; 2394 parms.desired_ifindex = nla_get_s32_default(a[OVS_VPORT_ATTR_IFINDEX], 2395 0); 2396 2397 vport = new_vport(&parms); 2398 err = PTR_ERR(vport); 2399 if (IS_ERR(vport)) 2400 goto exit_unlock_free; 2401 2402 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2403 info->snd_portid, info->snd_seq, 0, 2404 OVS_VPORT_CMD_NEW, GFP_KERNEL); 2405 2406 new_headroom = netdev_get_fwd_headroom(vport->dev); 2407 2408 if (new_headroom > dp->max_headroom) 2409 ovs_update_headroom(dp, new_headroom); 2410 else 2411 netdev_set_rx_headroom(vport->dev, dp->max_headroom); 2412 2413 BUG_ON(err < 0); 2414 ovs_unlock(); 2415 2416 ovs_notify(&dp_vport_genl_family, reply, info); 2417 return 0; 2418 2419 exit_unlock_free: 2420 ovs_unlock(); 2421 kfree_skb(reply); 2422 return err; 2423 } 2424 2425 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info) 2426 { 2427 struct nlattr **a = info->attrs; 2428 struct sk_buff *reply; 2429 struct vport *vport; 2430 int err; 2431 2432 reply = ovs_vport_cmd_alloc_info(); 2433 if (!reply) 2434 return -ENOMEM; 2435 2436 ovs_lock(); 2437 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a); 2438 err = PTR_ERR(vport); 2439 if (IS_ERR(vport)) 2440 goto exit_unlock_free; 2441 2442 if (a[OVS_VPORT_ATTR_TYPE] && 2443 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) { 2444 err = -EINVAL; 2445 goto exit_unlock_free; 2446 } 2447 2448 if (a[OVS_VPORT_ATTR_OPTIONS]) { 2449 /* There are no vport types that support legacy options. */ 2450 err = -EOPNOTSUPP; 2451 goto exit_unlock_free; 2452 } 2453 2454 if (a[OVS_VPORT_ATTR_UPCALL_PID]) { 2455 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID]; 2456 2457 err = ovs_vport_set_upcall_portids(vport, ids); 2458 if (err) 2459 goto exit_unlock_free; 2460 } 2461 2462 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2463 info->snd_portid, info->snd_seq, 0, 2464 OVS_VPORT_CMD_SET, GFP_KERNEL); 2465 BUG_ON(err < 0); 2466 2467 ovs_unlock(); 2468 ovs_notify(&dp_vport_genl_family, reply, info); 2469 return 0; 2470 2471 exit_unlock_free: 2472 ovs_unlock(); 2473 kfree_skb(reply); 2474 return err; 2475 } 2476 2477 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info) 2478 { 2479 bool update_headroom = false; 2480 struct nlattr **a = info->attrs; 2481 struct sk_buff *reply; 2482 struct datapath *dp; 2483 struct vport *vport; 2484 unsigned int new_headroom; 2485 int err; 2486 2487 reply = ovs_vport_cmd_alloc_info(); 2488 if (!reply) 2489 return -ENOMEM; 2490 2491 ovs_lock(); 2492 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a); 2493 err = PTR_ERR(vport); 2494 if (IS_ERR(vport)) 2495 goto exit_unlock_free; 2496 2497 if (vport->port_no == OVSP_LOCAL) { 2498 err = -EINVAL; 2499 goto exit_unlock_free; 2500 } 2501 2502 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2503 info->snd_portid, info->snd_seq, 0, 2504 OVS_VPORT_CMD_DEL, GFP_KERNEL); 2505 BUG_ON(err < 0); 2506 2507 /* the vport deletion may trigger dp headroom update */ 2508 dp = vport->dp; 2509 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom) 2510 update_headroom = true; 2511 2512 netdev_reset_rx_headroom(vport->dev); 2513 ovs_dp_detach_port(vport); 2514 2515 if (update_headroom) { 2516 new_headroom = ovs_get_max_headroom(dp); 2517 2518 if (new_headroom < dp->max_headroom) 2519 ovs_update_headroom(dp, new_headroom); 2520 } 2521 ovs_unlock(); 2522 2523 ovs_notify(&dp_vport_genl_family, reply, info); 2524 return 0; 2525 2526 exit_unlock_free: 2527 ovs_unlock(); 2528 kfree_skb(reply); 2529 return err; 2530 } 2531 2532 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info) 2533 { 2534 struct nlattr **a = info->attrs; 2535 struct ovs_header *ovs_header = genl_info_userhdr(info); 2536 struct sk_buff *reply; 2537 struct vport *vport; 2538 int err; 2539 2540 reply = ovs_vport_cmd_alloc_info(); 2541 if (!reply) 2542 return -ENOMEM; 2543 2544 rcu_read_lock(); 2545 vport = lookup_vport(sock_net(skb->sk), ovs_header, a); 2546 err = PTR_ERR(vport); 2547 if (IS_ERR(vport)) 2548 goto exit_unlock_free; 2549 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2550 info->snd_portid, info->snd_seq, 0, 2551 OVS_VPORT_CMD_GET, GFP_ATOMIC); 2552 BUG_ON(err < 0); 2553 rcu_read_unlock(); 2554 2555 return genlmsg_reply(reply, info); 2556 2557 exit_unlock_free: 2558 rcu_read_unlock(); 2559 kfree_skb(reply); 2560 return err; 2561 } 2562 2563 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 2564 { 2565 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh)); 2566 struct datapath *dp; 2567 int bucket = cb->args[0], skip = cb->args[1]; 2568 int i, j = 0; 2569 2570 rcu_read_lock(); 2571 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex); 2572 if (!dp) { 2573 rcu_read_unlock(); 2574 return -ENODEV; 2575 } 2576 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) { 2577 struct vport *vport; 2578 2579 j = 0; 2580 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) { 2581 if (j >= skip && 2582 ovs_vport_cmd_fill_info(vport, skb, 2583 sock_net(skb->sk), 2584 NETLINK_CB(cb->skb).portid, 2585 cb->nlh->nlmsg_seq, 2586 NLM_F_MULTI, 2587 OVS_VPORT_CMD_GET, 2588 GFP_ATOMIC) < 0) 2589 goto out; 2590 2591 j++; 2592 } 2593 skip = 0; 2594 } 2595 out: 2596 rcu_read_unlock(); 2597 2598 cb->args[0] = i; 2599 cb->args[1] = j; 2600 2601 return skb->len; 2602 } 2603 2604 static void ovs_dp_masks_rebalance(struct work_struct *work) 2605 { 2606 struct ovs_net *ovs_net = container_of(work, struct ovs_net, 2607 masks_rebalance.work); 2608 struct datapath *dp; 2609 2610 ovs_lock(); 2611 2612 list_for_each_entry(dp, &ovs_net->dps, list_node) 2613 ovs_flow_masks_rebalance(&dp->table); 2614 2615 ovs_unlock(); 2616 2617 schedule_delayed_work(&ovs_net->masks_rebalance, 2618 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL)); 2619 } 2620 2621 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = { 2622 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, 2623 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) }, 2624 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 }, 2625 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 }, 2626 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC }, 2627 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED }, /* Unused. */ 2628 [OVS_VPORT_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0), 2629 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 }, 2630 [OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED }, 2631 }; 2632 2633 static const struct genl_small_ops dp_vport_genl_ops[] = { 2634 { .cmd = OVS_VPORT_CMD_NEW, 2635 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2636 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2637 .doit = ovs_vport_cmd_new 2638 }, 2639 { .cmd = OVS_VPORT_CMD_DEL, 2640 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2641 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2642 .doit = ovs_vport_cmd_del 2643 }, 2644 { .cmd = OVS_VPORT_CMD_GET, 2645 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2646 .flags = 0, /* OK for unprivileged users. */ 2647 .doit = ovs_vport_cmd_get, 2648 .dumpit = ovs_vport_cmd_dump 2649 }, 2650 { .cmd = OVS_VPORT_CMD_SET, 2651 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2652 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2653 .doit = ovs_vport_cmd_set, 2654 }, 2655 }; 2656 2657 struct genl_family dp_vport_genl_family __ro_after_init = { 2658 .hdrsize = sizeof(struct ovs_header), 2659 .name = OVS_VPORT_FAMILY, 2660 .version = OVS_VPORT_VERSION, 2661 .maxattr = OVS_VPORT_ATTR_MAX, 2662 .policy = vport_policy, 2663 .netnsok = true, 2664 .parallel_ops = true, 2665 .small_ops = dp_vport_genl_ops, 2666 .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops), 2667 .resv_start_op = OVS_VPORT_CMD_SET + 1, 2668 .mcgrps = &ovs_dp_vport_multicast_group, 2669 .n_mcgrps = 1, 2670 .module = THIS_MODULE, 2671 }; 2672 2673 static struct genl_family * const dp_genl_families[] = { 2674 &dp_datapath_genl_family, 2675 &dp_vport_genl_family, 2676 &dp_flow_genl_family, 2677 &dp_packet_genl_family, 2678 &dp_meter_genl_family, 2679 #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) 2680 &dp_ct_limit_genl_family, 2681 #endif 2682 }; 2683 2684 static void dp_unregister_genl(int n_families) 2685 { 2686 int i; 2687 2688 for (i = 0; i < n_families; i++) 2689 genl_unregister_family(dp_genl_families[i]); 2690 } 2691 2692 static int __init dp_register_genl(void) 2693 { 2694 int err; 2695 int i; 2696 2697 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) { 2698 2699 err = genl_register_family(dp_genl_families[i]); 2700 if (err) 2701 goto error; 2702 } 2703 2704 return 0; 2705 2706 error: 2707 dp_unregister_genl(i); 2708 return err; 2709 } 2710 2711 static int __net_init ovs_init_net(struct net *net) 2712 { 2713 struct ovs_net *ovs_net = net_generic(net, ovs_net_id); 2714 int err; 2715 2716 INIT_LIST_HEAD(&ovs_net->dps); 2717 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq); 2718 INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance); 2719 2720 err = ovs_ct_init(net); 2721 if (err) 2722 return err; 2723 2724 schedule_delayed_work(&ovs_net->masks_rebalance, 2725 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL)); 2726 return 0; 2727 } 2728 2729 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet, 2730 struct list_head *head) 2731 { 2732 struct ovs_net *ovs_net = net_generic(net, ovs_net_id); 2733 struct datapath *dp; 2734 2735 list_for_each_entry(dp, &ovs_net->dps, list_node) { 2736 int i; 2737 2738 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2739 struct vport *vport; 2740 2741 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) { 2742 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL) 2743 continue; 2744 2745 if (dev_net(vport->dev) == dnet) 2746 list_add(&vport->detach_list, head); 2747 } 2748 } 2749 } 2750 } 2751 2752 static void __net_exit ovs_pre_exit_net(struct net *dnet) 2753 { 2754 ovs_lock(); 2755 ovs_ct_exit_start(dnet); 2756 ovs_unlock(); 2757 } 2758 2759 static void __net_exit ovs_exit_net(struct net *dnet) 2760 { 2761 struct datapath *dp, *dp_next; 2762 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id); 2763 struct vport *vport, *vport_next; 2764 struct net *net; 2765 LIST_HEAD(head); 2766 2767 ovs_lock(); 2768 2769 ovs_ct_exit_finish(dnet); 2770 2771 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node) 2772 __dp_destroy(dp); 2773 2774 down_read(&net_rwsem); 2775 for_each_net(net) 2776 list_vports_from_net(net, dnet, &head); 2777 up_read(&net_rwsem); 2778 2779 /* Detach all vports from given namespace. */ 2780 list_for_each_entry_safe(vport, vport_next, &head, detach_list) { 2781 list_del(&vport->detach_list); 2782 ovs_dp_detach_port(vport); 2783 } 2784 2785 ovs_unlock(); 2786 2787 cancel_delayed_work_sync(&ovs_net->masks_rebalance); 2788 cancel_work_sync(&ovs_net->dp_notify_work); 2789 } 2790 2791 static struct pernet_operations ovs_net_ops = { 2792 .init = ovs_init_net, 2793 .pre_exit = ovs_pre_exit_net, 2794 .exit = ovs_exit_net, 2795 .id = &ovs_net_id, 2796 .size = sizeof(struct ovs_net), 2797 }; 2798 2799 static const char * const ovs_drop_reasons[] = { 2800 #define S(x) [(x) & ~SKB_DROP_REASON_SUBSYS_MASK] = (#x), 2801 OVS_DROP_REASONS(S) 2802 #undef S 2803 }; 2804 2805 static struct drop_reason_list drop_reason_list_ovs = { 2806 .reasons = ovs_drop_reasons, 2807 .n_reasons = ARRAY_SIZE(ovs_drop_reasons), 2808 }; 2809 2810 static int __init ovs_alloc_percpu_storage(void) 2811 { 2812 unsigned int cpu; 2813 2814 ovs_pcpu_storage = alloc_percpu(*ovs_pcpu_storage); 2815 if (!ovs_pcpu_storage) 2816 return -ENOMEM; 2817 2818 for_each_possible_cpu(cpu) { 2819 struct ovs_pcpu_storage *ovs_pcpu; 2820 2821 ovs_pcpu = per_cpu_ptr(ovs_pcpu_storage, cpu); 2822 local_lock_init(&ovs_pcpu->bh_lock); 2823 } 2824 return 0; 2825 } 2826 2827 static void ovs_free_percpu_storage(void) 2828 { 2829 free_percpu(ovs_pcpu_storage); 2830 } 2831 2832 static int __init dp_init(void) 2833 { 2834 int err; 2835 2836 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > 2837 sizeof_field(struct sk_buff, cb)); 2838 2839 pr_info("Open vSwitch switching datapath\n"); 2840 2841 err = ovs_alloc_percpu_storage(); 2842 if (err) 2843 goto error; 2844 2845 err = ovs_internal_dev_rtnl_link_register(); 2846 if (err) 2847 goto error; 2848 2849 err = ovs_flow_init(); 2850 if (err) 2851 goto error_unreg_rtnl_link; 2852 2853 err = ovs_vport_init(); 2854 if (err) 2855 goto error_flow_exit; 2856 2857 err = register_pernet_device(&ovs_net_ops); 2858 if (err) 2859 goto error_vport_exit; 2860 2861 err = register_netdevice_notifier(&ovs_dp_device_notifier); 2862 if (err) 2863 goto error_netns_exit; 2864 2865 err = ovs_netdev_init(); 2866 if (err) 2867 goto error_unreg_notifier; 2868 2869 err = dp_register_genl(); 2870 if (err < 0) 2871 goto error_unreg_netdev; 2872 2873 drop_reasons_register_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH, 2874 &drop_reason_list_ovs); 2875 2876 return 0; 2877 2878 error_unreg_netdev: 2879 ovs_netdev_exit(); 2880 error_unreg_notifier: 2881 unregister_netdevice_notifier(&ovs_dp_device_notifier); 2882 error_netns_exit: 2883 unregister_pernet_device(&ovs_net_ops); 2884 error_vport_exit: 2885 ovs_vport_exit(); 2886 error_flow_exit: 2887 ovs_flow_exit(); 2888 error_unreg_rtnl_link: 2889 ovs_internal_dev_rtnl_link_unregister(); 2890 error: 2891 ovs_free_percpu_storage(); 2892 return err; 2893 } 2894 2895 static void dp_cleanup(void) 2896 { 2897 dp_unregister_genl(ARRAY_SIZE(dp_genl_families)); 2898 ovs_netdev_exit(); 2899 unregister_netdevice_notifier(&ovs_dp_device_notifier); 2900 unregister_pernet_device(&ovs_net_ops); 2901 drop_reasons_unregister_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH); 2902 rcu_barrier(); 2903 ovs_vport_exit(); 2904 ovs_flow_exit(); 2905 ovs_internal_dev_rtnl_link_unregister(); 2906 ovs_free_percpu_storage(); 2907 } 2908 2909 module_init(dp_init); 2910 module_exit(dp_cleanup); 2911 2912 MODULE_DESCRIPTION("Open vSwitch switching datapath"); 2913 MODULE_LICENSE("GPL"); 2914 MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY); 2915 MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY); 2916 MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY); 2917 MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY); 2918 MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY); 2919 MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY); 2920