1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2007-2014 Nicira, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/init.h> 9 #include <linux/module.h> 10 #include <linux/if_arp.h> 11 #include <linux/if_vlan.h> 12 #include <linux/in.h> 13 #include <linux/ip.h> 14 #include <linux/jhash.h> 15 #include <linux/delay.h> 16 #include <linux/time.h> 17 #include <linux/etherdevice.h> 18 #include <linux/kernel.h> 19 #include <linux/kthread.h> 20 #include <linux/mutex.h> 21 #include <linux/percpu.h> 22 #include <linux/rcupdate.h> 23 #include <linux/tcp.h> 24 #include <linux/udp.h> 25 #include <linux/ethtool.h> 26 #include <linux/wait.h> 27 #include <asm/div64.h> 28 #include <linux/highmem.h> 29 #include <linux/netfilter_bridge.h> 30 #include <linux/netfilter_ipv4.h> 31 #include <linux/inetdevice.h> 32 #include <linux/list.h> 33 #include <linux/openvswitch.h> 34 #include <linux/rculist.h> 35 #include <linux/dmi.h> 36 #include <net/genetlink.h> 37 #include <net/gso.h> 38 #include <net/net_namespace.h> 39 #include <net/netns/generic.h> 40 #include <net/pkt_cls.h> 41 42 #include "datapath.h" 43 #include "drop.h" 44 #include "flow.h" 45 #include "flow_table.h" 46 #include "flow_netlink.h" 47 #include "meter.h" 48 #include "openvswitch_trace.h" 49 #include "vport-internal_dev.h" 50 #include "vport-netdev.h" 51 52 unsigned int ovs_net_id __read_mostly; 53 54 static struct genl_family dp_packet_genl_family; 55 static struct genl_family dp_flow_genl_family; 56 static struct genl_family dp_datapath_genl_family; 57 58 static const struct nla_policy flow_policy[]; 59 60 static const struct genl_multicast_group ovs_dp_flow_multicast_group = { 61 .name = OVS_FLOW_MCGROUP, 62 }; 63 64 static const struct genl_multicast_group ovs_dp_datapath_multicast_group = { 65 .name = OVS_DATAPATH_MCGROUP, 66 }; 67 68 static const struct genl_multicast_group ovs_dp_vport_multicast_group = { 69 .name = OVS_VPORT_MCGROUP, 70 }; 71 72 /* Check if need to build a reply message. 73 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */ 74 static bool ovs_must_notify(struct genl_family *family, struct genl_info *info, 75 unsigned int group) 76 { 77 return info->nlhdr->nlmsg_flags & NLM_F_ECHO || 78 genl_has_listeners(family, genl_info_net(info), group); 79 } 80 81 static void ovs_notify(struct genl_family *family, 82 struct sk_buff *skb, struct genl_info *info) 83 { 84 genl_notify(family, skb, info, 0, GFP_KERNEL); 85 } 86 87 /** 88 * DOC: Locking: 89 * 90 * All writes e.g. Writes to device state (add/remove datapath, port, set 91 * operations on vports, etc.), Writes to other state (flow table 92 * modifications, set miscellaneous datapath parameters, etc.) are protected 93 * by ovs_lock. 94 * 95 * Reads are protected by RCU. 96 * 97 * There are a few special cases (mostly stats) that have their own 98 * synchronization but they nest under all of above and don't interact with 99 * each other. 100 * 101 * The RTNL lock nests inside ovs_mutex. 102 */ 103 104 static DEFINE_MUTEX(ovs_mutex); 105 106 void ovs_lock(void) 107 { 108 mutex_lock(&ovs_mutex); 109 } 110 111 void ovs_unlock(void) 112 { 113 mutex_unlock(&ovs_mutex); 114 } 115 116 #ifdef CONFIG_LOCKDEP 117 int lockdep_ovsl_is_held(void) 118 { 119 if (debug_locks) 120 return lockdep_is_held(&ovs_mutex); 121 else 122 return 1; 123 } 124 #endif 125 126 static struct vport *new_vport(const struct vport_parms *); 127 static int queue_gso_packets(struct datapath *dp, struct sk_buff *, 128 const struct sw_flow_key *, 129 const struct dp_upcall_info *, 130 uint32_t cutlen); 131 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *, 132 const struct sw_flow_key *, 133 const struct dp_upcall_info *, 134 uint32_t cutlen); 135 136 static void ovs_dp_masks_rebalance(struct work_struct *work); 137 138 static int ovs_dp_set_upcall_portids(struct datapath *, const struct nlattr *); 139 140 /* Must be called with rcu_read_lock or ovs_mutex. */ 141 const char *ovs_dp_name(const struct datapath *dp) 142 { 143 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL); 144 return ovs_vport_name(vport); 145 } 146 147 static int get_dpifindex(const struct datapath *dp) 148 { 149 struct vport *local; 150 int ifindex; 151 152 rcu_read_lock(); 153 154 local = ovs_vport_rcu(dp, OVSP_LOCAL); 155 if (local) 156 ifindex = local->dev->ifindex; 157 else 158 ifindex = 0; 159 160 rcu_read_unlock(); 161 162 return ifindex; 163 } 164 165 static void destroy_dp_rcu(struct rcu_head *rcu) 166 { 167 struct datapath *dp = container_of(rcu, struct datapath, rcu); 168 169 ovs_flow_tbl_destroy(&dp->table); 170 free_percpu(dp->stats_percpu); 171 kfree(dp->ports); 172 ovs_meters_exit(dp); 173 kfree(rcu_dereference_raw(dp->upcall_portids)); 174 kfree(dp); 175 } 176 177 static struct hlist_head *vport_hash_bucket(const struct datapath *dp, 178 u16 port_no) 179 { 180 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)]; 181 } 182 183 /* Called with ovs_mutex or RCU read lock. */ 184 struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no) 185 { 186 struct vport *vport; 187 struct hlist_head *head; 188 189 head = vport_hash_bucket(dp, port_no); 190 hlist_for_each_entry_rcu(vport, head, dp_hash_node, 191 lockdep_ovsl_is_held()) { 192 if (vport->port_no == port_no) 193 return vport; 194 } 195 return NULL; 196 } 197 198 /* Called with ovs_mutex. */ 199 static struct vport *new_vport(const struct vport_parms *parms) 200 { 201 struct vport *vport; 202 203 vport = ovs_vport_add(parms); 204 if (!IS_ERR(vport)) { 205 struct datapath *dp = parms->dp; 206 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no); 207 208 hlist_add_head_rcu(&vport->dp_hash_node, head); 209 } 210 return vport; 211 } 212 213 static void ovs_vport_update_upcall_stats(struct sk_buff *skb, 214 const struct dp_upcall_info *upcall_info, 215 bool upcall_result) 216 { 217 struct vport *p = OVS_CB(skb)->input_vport; 218 struct vport_upcall_stats_percpu *stats; 219 220 if (upcall_info->cmd != OVS_PACKET_CMD_MISS && 221 upcall_info->cmd != OVS_PACKET_CMD_ACTION) 222 return; 223 224 stats = this_cpu_ptr(p->upcall_stats); 225 u64_stats_update_begin(&stats->syncp); 226 if (upcall_result) 227 u64_stats_inc(&stats->n_success); 228 else 229 u64_stats_inc(&stats->n_fail); 230 u64_stats_update_end(&stats->syncp); 231 } 232 233 void ovs_dp_detach_port(struct vport *p) 234 { 235 ASSERT_OVSL(); 236 237 /* First drop references to device. */ 238 hlist_del_rcu(&p->dp_hash_node); 239 240 /* Then destroy it. */ 241 ovs_vport_del(p); 242 } 243 244 /* Must be called with rcu_read_lock. */ 245 void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) 246 { 247 struct ovs_pcpu_storage *ovs_pcpu = this_cpu_ptr(ovs_pcpu_storage); 248 const struct vport *p = OVS_CB(skb)->input_vport; 249 struct datapath *dp = p->dp; 250 struct sw_flow *flow; 251 struct sw_flow_actions *sf_acts; 252 struct dp_stats_percpu *stats; 253 bool ovs_pcpu_locked = false; 254 u64 *stats_counter; 255 u32 n_mask_hit; 256 u32 n_cache_hit; 257 int error; 258 259 stats = this_cpu_ptr(dp->stats_percpu); 260 261 /* Look up flow. */ 262 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb), 263 &n_mask_hit, &n_cache_hit); 264 if (unlikely(!flow)) { 265 struct dp_upcall_info upcall; 266 267 memset(&upcall, 0, sizeof(upcall)); 268 upcall.cmd = OVS_PACKET_CMD_MISS; 269 270 if (OVS_CB(skb)->upcall_pid) 271 upcall.portid = OVS_CB(skb)->upcall_pid; 272 else if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU) 273 upcall.portid = 274 ovs_dp_get_upcall_portid(dp, smp_processor_id()); 275 else 276 upcall.portid = ovs_vport_find_upcall_portid(p, skb); 277 278 upcall.mru = OVS_CB(skb)->mru; 279 error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX); 280 switch (error) { 281 case 0: 282 case -EAGAIN: 283 case -ERESTARTSYS: 284 case -EINTR: 285 consume_skb(skb); 286 break; 287 default: 288 skb_tx_error(skb); 289 kfree_skb(skb); 290 break; 291 } 292 stats_counter = &stats->n_missed; 293 goto out; 294 } 295 296 ovs_flow_stats_update(flow, key->tp.flags, skb); 297 sf_acts = rcu_dereference(flow->sf_acts); 298 /* This path can be invoked recursively: Use the current task to 299 * identify recursive invocation - the lock must be acquired only once. 300 * Even with disabled bottom halves this can be preempted on PREEMPT_RT. 301 * Limit the locking to RT to avoid assigning `owner' if it can be 302 * avoided. 303 */ 304 if (IS_ENABLED(CONFIG_PREEMPT_RT) && ovs_pcpu->owner != current) { 305 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock); 306 ovs_pcpu->owner = current; 307 ovs_pcpu_locked = true; 308 } 309 310 error = ovs_execute_actions(dp, skb, sf_acts, key); 311 if (unlikely(error)) 312 net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n", 313 ovs_dp_name(dp), error); 314 if (ovs_pcpu_locked) { 315 ovs_pcpu->owner = NULL; 316 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock); 317 } 318 319 stats_counter = &stats->n_hit; 320 321 out: 322 /* Update datapath statistics. */ 323 u64_stats_update_begin(&stats->syncp); 324 (*stats_counter)++; 325 stats->n_mask_hit += n_mask_hit; 326 stats->n_cache_hit += n_cache_hit; 327 u64_stats_update_end(&stats->syncp); 328 } 329 330 int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb, 331 const struct sw_flow_key *key, 332 const struct dp_upcall_info *upcall_info, 333 uint32_t cutlen) 334 { 335 struct dp_stats_percpu *stats; 336 int err; 337 338 if (trace_ovs_dp_upcall_enabled()) 339 trace_ovs_dp_upcall(dp, skb, key, upcall_info); 340 341 if (upcall_info->portid == 0) { 342 err = -ENOTCONN; 343 goto err; 344 } 345 346 if (!skb_is_gso(skb)) 347 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen); 348 else 349 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen); 350 351 ovs_vport_update_upcall_stats(skb, upcall_info, !err); 352 if (err) 353 goto err; 354 355 return 0; 356 357 err: 358 stats = this_cpu_ptr(dp->stats_percpu); 359 360 u64_stats_update_begin(&stats->syncp); 361 stats->n_lost++; 362 u64_stats_update_end(&stats->syncp); 363 364 return err; 365 } 366 367 static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb, 368 const struct sw_flow_key *key, 369 const struct dp_upcall_info *upcall_info, 370 uint32_t cutlen) 371 { 372 unsigned int gso_type = skb_shinfo(skb)->gso_type; 373 struct sw_flow_key later_key; 374 struct sk_buff *segs, *nskb; 375 int err; 376 377 BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET); 378 segs = __skb_gso_segment(skb, NETIF_F_SG, false); 379 if (IS_ERR(segs)) 380 return PTR_ERR(segs); 381 if (segs == NULL) 382 return -EINVAL; 383 384 if (gso_type & SKB_GSO_UDP) { 385 /* The initial flow key extracted by ovs_flow_key_extract() 386 * in this case is for a first fragment, so we need to 387 * properly mark later fragments. 388 */ 389 later_key = *key; 390 later_key.ip.frag = OVS_FRAG_TYPE_LATER; 391 } 392 393 /* Queue all of the segments. */ 394 skb_list_walk_safe(segs, skb, nskb) { 395 if (gso_type & SKB_GSO_UDP && skb != segs) 396 key = &later_key; 397 398 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen); 399 if (err) 400 break; 401 402 } 403 404 /* Free all of the segments. */ 405 skb_list_walk_safe(segs, skb, nskb) { 406 if (err) 407 kfree_skb(skb); 408 else 409 consume_skb(skb); 410 } 411 return err; 412 } 413 414 static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info, 415 unsigned int hdrlen, int actions_attrlen) 416 { 417 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header)) 418 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */ 419 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */ 420 + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */ 421 + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */ 422 423 /* OVS_PACKET_ATTR_USERDATA */ 424 if (upcall_info->userdata) 425 size += NLA_ALIGN(upcall_info->userdata->nla_len); 426 427 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */ 428 if (upcall_info->egress_tun_info) 429 size += nla_total_size(ovs_tun_key_attr_size()); 430 431 /* OVS_PACKET_ATTR_ACTIONS */ 432 if (upcall_info->actions_len) 433 size += nla_total_size(actions_attrlen); 434 435 /* OVS_PACKET_ATTR_MRU */ 436 if (upcall_info->mru) 437 size += nla_total_size(sizeof(upcall_info->mru)); 438 439 return size; 440 } 441 442 static void pad_packet(struct datapath *dp, struct sk_buff *skb) 443 { 444 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) { 445 size_t plen = NLA_ALIGN(skb->len) - skb->len; 446 447 if (plen > 0) 448 skb_put_zero(skb, plen); 449 } 450 } 451 452 static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, 453 const struct sw_flow_key *key, 454 const struct dp_upcall_info *upcall_info, 455 uint32_t cutlen) 456 { 457 struct ovs_header *upcall; 458 struct sk_buff *nskb = NULL; 459 struct sk_buff *user_skb = NULL; /* to be queued to userspace */ 460 struct nlattr *nla; 461 size_t msg_size; 462 size_t skb_len; 463 unsigned int hlen; 464 int err, dp_ifindex; 465 u64 hash; 466 467 dp_ifindex = get_dpifindex(dp); 468 if (!dp_ifindex) 469 return -ENODEV; 470 471 if (!skb_frags_readable(skb)) 472 return -EFAULT; 473 474 if (skb_vlan_tag_present(skb)) { 475 nskb = skb_clone(skb, GFP_ATOMIC); 476 if (!nskb) 477 return -ENOMEM; 478 479 nskb = __vlan_hwaccel_push_inside(nskb); 480 if (!nskb) 481 return -ENOMEM; 482 483 skb = nskb; 484 } 485 486 skb_len = min(skb->len, cutlen); 487 if (nla_attr_size(skb_len) > USHRT_MAX) { 488 err = -EFBIG; 489 goto out; 490 } 491 492 /* Complete checksum if needed */ 493 if (skb->ip_summed == CHECKSUM_PARTIAL && 494 (err = skb_csum_hwoffload_help(skb, 0))) 495 goto out; 496 497 /* Older versions of OVS user space enforce alignment of the last 498 * Netlink attribute to NLA_ALIGNTO which would require extensive 499 * padding logic. Only perform zerocopy if padding is not required. 500 */ 501 if (dp->user_features & OVS_DP_F_UNALIGNED) 502 hlen = min(skb_zerocopy_headlen(skb), cutlen); 503 else 504 hlen = skb_len; 505 506 msg_size = upcall_msg_size(upcall_info, hlen, 507 OVS_CB(skb)->acts_origlen); 508 user_skb = genlmsg_new(msg_size, GFP_ATOMIC); 509 if (!user_skb) { 510 err = -ENOMEM; 511 goto out; 512 } 513 514 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family, 515 0, upcall_info->cmd); 516 if (!upcall) { 517 err = -EINVAL; 518 goto out; 519 } 520 upcall->dp_ifindex = dp_ifindex; 521 522 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb); 523 if (err) 524 goto out; 525 526 if (upcall_info->userdata) 527 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA, 528 nla_len(upcall_info->userdata), 529 nla_data(upcall_info->userdata)); 530 531 if (upcall_info->egress_tun_info) { 532 nla = nla_nest_start_noflag(user_skb, 533 OVS_PACKET_ATTR_EGRESS_TUN_KEY); 534 if (!nla) { 535 err = -EMSGSIZE; 536 goto out; 537 } 538 err = ovs_nla_put_tunnel_info(user_skb, 539 upcall_info->egress_tun_info); 540 if (err) 541 goto out; 542 543 nla_nest_end(user_skb, nla); 544 } 545 546 if (upcall_info->actions_len) { 547 nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS); 548 if (!nla) { 549 err = -EMSGSIZE; 550 goto out; 551 } 552 err = ovs_nla_put_actions(upcall_info->actions, 553 upcall_info->actions_len, 554 user_skb); 555 if (!err) 556 nla_nest_end(user_skb, nla); 557 else 558 nla_nest_cancel(user_skb, nla); 559 } 560 561 /* Add OVS_PACKET_ATTR_MRU */ 562 if (upcall_info->mru && 563 nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) { 564 err = -ENOBUFS; 565 goto out; 566 } 567 568 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */ 569 if (skb_len < skb->len && 570 nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) { 571 err = -ENOBUFS; 572 goto out; 573 } 574 575 /* Add OVS_PACKET_ATTR_HASH */ 576 hash = skb_get_hash_raw(skb); 577 if (skb->sw_hash) 578 hash |= OVS_PACKET_HASH_SW_BIT; 579 580 if (skb->l4_hash) 581 hash |= OVS_PACKET_HASH_L4_BIT; 582 583 if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) { 584 err = -ENOBUFS; 585 goto out; 586 } 587 588 /* Only reserve room for attribute header, packet data is added 589 * in skb_zerocopy() */ 590 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) { 591 err = -ENOBUFS; 592 goto out; 593 } 594 nla->nla_len = nla_attr_size(skb_len); 595 596 err = skb_zerocopy(user_skb, skb, skb_len, hlen); 597 if (err) 598 goto out; 599 600 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */ 601 pad_packet(dp, user_skb); 602 603 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len; 604 605 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid); 606 user_skb = NULL; 607 out: 608 consume_skb(user_skb); 609 consume_skb(nskb); 610 611 return err; 612 } 613 614 static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info) 615 { 616 struct ovs_header *ovs_header = genl_info_userhdr(info); 617 struct net *net = sock_net(skb->sk); 618 struct nlattr **a = info->attrs; 619 struct sw_flow_actions *acts; 620 struct sk_buff *packet; 621 struct sw_flow *flow; 622 struct sw_flow_actions *sf_acts; 623 struct datapath *dp; 624 struct vport *input_vport; 625 u16 mru = 0; 626 u64 hash; 627 int len; 628 int err; 629 bool log = !a[OVS_PACKET_ATTR_PROBE]; 630 631 err = -EINVAL; 632 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] || 633 !a[OVS_PACKET_ATTR_ACTIONS]) 634 goto err; 635 636 len = nla_len(a[OVS_PACKET_ATTR_PACKET]); 637 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL); 638 err = -ENOMEM; 639 if (!packet) 640 goto err; 641 skb_reserve(packet, NET_IP_ALIGN); 642 643 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len); 644 645 /* Set packet's mru */ 646 if (a[OVS_PACKET_ATTR_MRU]) { 647 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]); 648 packet->ignore_df = 1; 649 } 650 OVS_CB(packet)->mru = mru; 651 OVS_CB(packet)->cutlen = U32_MAX; 652 653 if (a[OVS_PACKET_ATTR_HASH]) { 654 hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]); 655 656 __skb_set_hash(packet, hash & 0xFFFFFFFFULL, 657 !!(hash & OVS_PACKET_HASH_SW_BIT), 658 !!(hash & OVS_PACKET_HASH_L4_BIT)); 659 } 660 661 OVS_CB(packet)->upcall_pid = 662 nla_get_u32_default(a[OVS_PACKET_ATTR_UPCALL_PID], 0); 663 664 /* Build an sw_flow for sending this packet. */ 665 flow = ovs_flow_alloc(); 666 err = PTR_ERR(flow); 667 if (IS_ERR(flow)) 668 goto err_kfree_skb; 669 670 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY], 671 packet, &flow->key, log); 672 if (err) 673 goto err_flow_free; 674 675 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS], 676 &flow->key, &acts, log); 677 if (err) 678 goto err_flow_free; 679 680 rcu_assign_pointer(flow->sf_acts, acts); 681 packet->priority = flow->key.phy.priority; 682 packet->mark = flow->key.phy.skb_mark; 683 684 rcu_read_lock(); 685 dp = get_dp_rcu(net, ovs_header->dp_ifindex); 686 err = -ENODEV; 687 if (!dp) 688 goto err_unlock; 689 690 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port); 691 if (!input_vport) 692 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL); 693 694 if (!input_vport) 695 goto err_unlock; 696 697 packet->dev = input_vport->dev; 698 OVS_CB(packet)->input_vport = input_vport; 699 sf_acts = rcu_dereference(flow->sf_acts); 700 701 local_bh_disable(); 702 local_lock_nested_bh(&ovs_pcpu_storage->bh_lock); 703 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 704 this_cpu_write(ovs_pcpu_storage->owner, current); 705 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key); 706 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 707 this_cpu_write(ovs_pcpu_storage->owner, NULL); 708 local_unlock_nested_bh(&ovs_pcpu_storage->bh_lock); 709 local_bh_enable(); 710 rcu_read_unlock(); 711 712 ovs_flow_free(flow, false); 713 return err; 714 715 err_unlock: 716 rcu_read_unlock(); 717 err_flow_free: 718 ovs_flow_free(flow, false); 719 err_kfree_skb: 720 kfree_skb(packet); 721 err: 722 return err; 723 } 724 725 static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = { 726 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN }, 727 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED }, 728 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED }, 729 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG }, 730 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 }, 731 [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 }, 732 [OVS_PACKET_ATTR_UPCALL_PID] = { .type = NLA_U32 }, 733 }; 734 735 static const struct genl_small_ops dp_packet_genl_ops[] = { 736 { .cmd = OVS_PACKET_CMD_EXECUTE, 737 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 738 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 739 .doit = ovs_packet_cmd_execute 740 } 741 }; 742 743 static struct genl_family dp_packet_genl_family __ro_after_init = { 744 .hdrsize = sizeof(struct ovs_header), 745 .name = OVS_PACKET_FAMILY, 746 .version = OVS_PACKET_VERSION, 747 .maxattr = OVS_PACKET_ATTR_MAX, 748 .policy = packet_policy, 749 .netnsok = true, 750 .parallel_ops = true, 751 .small_ops = dp_packet_genl_ops, 752 .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops), 753 .resv_start_op = OVS_PACKET_CMD_EXECUTE + 1, 754 .module = THIS_MODULE, 755 }; 756 757 static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats, 758 struct ovs_dp_megaflow_stats *mega_stats) 759 { 760 int i; 761 762 memset(mega_stats, 0, sizeof(*mega_stats)); 763 764 stats->n_flows = ovs_flow_tbl_count(&dp->table); 765 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table); 766 767 stats->n_hit = stats->n_missed = stats->n_lost = 0; 768 769 for_each_possible_cpu(i) { 770 const struct dp_stats_percpu *percpu_stats; 771 struct dp_stats_percpu local_stats; 772 unsigned int start; 773 774 percpu_stats = per_cpu_ptr(dp->stats_percpu, i); 775 776 do { 777 start = u64_stats_fetch_begin(&percpu_stats->syncp); 778 local_stats = *percpu_stats; 779 } while (u64_stats_fetch_retry(&percpu_stats->syncp, start)); 780 781 stats->n_hit += local_stats.n_hit; 782 stats->n_missed += local_stats.n_missed; 783 stats->n_lost += local_stats.n_lost; 784 mega_stats->n_mask_hit += local_stats.n_mask_hit; 785 mega_stats->n_cache_hit += local_stats.n_cache_hit; 786 } 787 } 788 789 static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags) 790 { 791 return ovs_identifier_is_ufid(sfid) && 792 !(ufid_flags & OVS_UFID_F_OMIT_KEY); 793 } 794 795 static bool should_fill_mask(uint32_t ufid_flags) 796 { 797 return !(ufid_flags & OVS_UFID_F_OMIT_MASK); 798 } 799 800 static bool should_fill_actions(uint32_t ufid_flags) 801 { 802 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS); 803 } 804 805 static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts, 806 const struct sw_flow_id *sfid, 807 uint32_t ufid_flags) 808 { 809 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header)); 810 811 /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback 812 * see ovs_nla_put_identifier() 813 */ 814 if (sfid && ovs_identifier_is_ufid(sfid)) 815 len += nla_total_size(sfid->ufid_len); 816 else 817 len += nla_total_size(ovs_key_attr_size()); 818 819 /* OVS_FLOW_ATTR_KEY */ 820 if (!sfid || should_fill_key(sfid, ufid_flags)) 821 len += nla_total_size(ovs_key_attr_size()); 822 823 /* OVS_FLOW_ATTR_MASK */ 824 if (should_fill_mask(ufid_flags)) 825 len += nla_total_size(ovs_key_attr_size()); 826 827 /* OVS_FLOW_ATTR_ACTIONS */ 828 if (should_fill_actions(ufid_flags)) 829 len += nla_total_size(acts->orig_len); 830 831 return len 832 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */ 833 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */ 834 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */ 835 } 836 837 /* Called with ovs_mutex or RCU read lock. */ 838 static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow, 839 struct sk_buff *skb) 840 { 841 struct ovs_flow_stats stats; 842 __be16 tcp_flags; 843 unsigned long used; 844 845 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags); 846 847 if (used && 848 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used), 849 OVS_FLOW_ATTR_PAD)) 850 return -EMSGSIZE; 851 852 if (stats.n_packets && 853 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS, 854 sizeof(struct ovs_flow_stats), &stats, 855 OVS_FLOW_ATTR_PAD)) 856 return -EMSGSIZE; 857 858 if ((u8)ntohs(tcp_flags) && 859 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags))) 860 return -EMSGSIZE; 861 862 return 0; 863 } 864 865 /* Called with ovs_mutex or RCU read lock. */ 866 static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow, 867 struct sk_buff *skb, int skb_orig_len) 868 { 869 struct nlattr *start; 870 int err; 871 872 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if 873 * this is the first flow to be dumped into 'skb'. This is unusual for 874 * Netlink but individual action lists can be longer than 875 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this. 876 * The userspace caller can always fetch the actions separately if it 877 * really wants them. (Most userspace callers in fact don't care.) 878 * 879 * This can only fail for dump operations because the skb is always 880 * properly sized for single flows. 881 */ 882 start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS); 883 if (start) { 884 const struct sw_flow_actions *sf_acts; 885 886 sf_acts = rcu_dereference_ovsl(flow->sf_acts); 887 err = ovs_nla_put_actions(sf_acts->actions, 888 sf_acts->actions_len, skb); 889 890 if (!err) 891 nla_nest_end(skb, start); 892 else { 893 if (skb_orig_len) 894 return err; 895 896 nla_nest_cancel(skb, start); 897 } 898 } else if (skb_orig_len) { 899 return -EMSGSIZE; 900 } 901 902 return 0; 903 } 904 905 /* Called with ovs_mutex or RCU read lock. */ 906 static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex, 907 struct sk_buff *skb, u32 portid, 908 u32 seq, u32 flags, u8 cmd, u32 ufid_flags) 909 { 910 const int skb_orig_len = skb->len; 911 struct ovs_header *ovs_header; 912 int err; 913 914 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family, 915 flags, cmd); 916 if (!ovs_header) 917 return -EMSGSIZE; 918 919 ovs_header->dp_ifindex = dp_ifindex; 920 921 err = ovs_nla_put_identifier(flow, skb); 922 if (err) 923 goto error; 924 925 if (should_fill_key(&flow->id, ufid_flags)) { 926 err = ovs_nla_put_masked_key(flow, skb); 927 if (err) 928 goto error; 929 } 930 931 if (should_fill_mask(ufid_flags)) { 932 err = ovs_nla_put_mask(flow, skb); 933 if (err) 934 goto error; 935 } 936 937 err = ovs_flow_cmd_fill_stats(flow, skb); 938 if (err) 939 goto error; 940 941 if (should_fill_actions(ufid_flags)) { 942 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len); 943 if (err) 944 goto error; 945 } 946 947 genlmsg_end(skb, ovs_header); 948 return 0; 949 950 error: 951 genlmsg_cancel(skb, ovs_header); 952 return err; 953 } 954 955 /* May not be called with RCU read lock. */ 956 static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts, 957 const struct sw_flow_id *sfid, 958 struct genl_info *info, 959 bool always, 960 uint32_t ufid_flags) 961 { 962 struct sk_buff *skb; 963 size_t len; 964 965 if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0)) 966 return NULL; 967 968 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags); 969 skb = genlmsg_new(len, GFP_KERNEL); 970 if (!skb) 971 return ERR_PTR(-ENOMEM); 972 973 return skb; 974 } 975 976 /* Called with ovs_mutex. */ 977 static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow, 978 int dp_ifindex, 979 struct genl_info *info, u8 cmd, 980 bool always, u32 ufid_flags) 981 { 982 struct sk_buff *skb; 983 int retval; 984 985 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), 986 &flow->id, info, always, ufid_flags); 987 if (IS_ERR_OR_NULL(skb)) 988 return skb; 989 990 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb, 991 info->snd_portid, info->snd_seq, 0, 992 cmd, ufid_flags); 993 if (WARN_ON_ONCE(retval < 0)) { 994 kfree_skb(skb); 995 skb = ERR_PTR(retval); 996 } 997 return skb; 998 } 999 1000 static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) 1001 { 1002 struct net *net = sock_net(skb->sk); 1003 struct nlattr **a = info->attrs; 1004 struct ovs_header *ovs_header = genl_info_userhdr(info); 1005 struct sw_flow *flow = NULL, *new_flow; 1006 struct sw_flow_mask mask; 1007 struct sk_buff *reply; 1008 struct datapath *dp; 1009 struct sw_flow_key *key; 1010 struct sw_flow_actions *acts; 1011 struct sw_flow_match match; 1012 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1013 int error; 1014 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1015 1016 /* Must have key and actions. */ 1017 error = -EINVAL; 1018 if (!a[OVS_FLOW_ATTR_KEY]) { 1019 OVS_NLERR(log, "Flow key attr not present in new flow."); 1020 goto error; 1021 } 1022 if (!a[OVS_FLOW_ATTR_ACTIONS]) { 1023 OVS_NLERR(log, "Flow actions attr not present in new flow."); 1024 goto error; 1025 } 1026 1027 /* Most of the time we need to allocate a new flow, do it before 1028 * locking. 1029 */ 1030 new_flow = ovs_flow_alloc(); 1031 if (IS_ERR(new_flow)) { 1032 error = PTR_ERR(new_flow); 1033 goto error; 1034 } 1035 1036 /* Extract key. */ 1037 key = kzalloc_obj(*key); 1038 if (!key) { 1039 error = -ENOMEM; 1040 goto err_kfree_flow; 1041 } 1042 1043 ovs_match_init(&match, key, false, &mask); 1044 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], 1045 a[OVS_FLOW_ATTR_MASK], log); 1046 if (error) 1047 goto err_kfree_key; 1048 1049 ovs_flow_mask_key(&new_flow->key, key, true, &mask); 1050 1051 /* Extract flow identifier. */ 1052 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID], 1053 key, log); 1054 if (error) 1055 goto err_kfree_key; 1056 1057 /* Validate actions. */ 1058 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS], 1059 &new_flow->key, &acts, log); 1060 if (error) { 1061 OVS_NLERR(log, "Flow actions may not be safe on all matching packets."); 1062 goto err_kfree_key; 1063 } 1064 1065 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false, 1066 ufid_flags); 1067 if (IS_ERR(reply)) { 1068 error = PTR_ERR(reply); 1069 goto err_kfree_acts; 1070 } 1071 1072 ovs_lock(); 1073 dp = get_dp(net, ovs_header->dp_ifindex); 1074 if (unlikely(!dp)) { 1075 error = -ENODEV; 1076 goto err_unlock_ovs; 1077 } 1078 1079 /* Check if this is a duplicate flow */ 1080 if (ovs_identifier_is_ufid(&new_flow->id)) 1081 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id); 1082 if (!flow) 1083 flow = ovs_flow_tbl_lookup(&dp->table, key); 1084 if (likely(!flow)) { 1085 rcu_assign_pointer(new_flow->sf_acts, acts); 1086 1087 /* Put flow in bucket. */ 1088 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask); 1089 if (unlikely(error)) { 1090 acts = NULL; 1091 goto err_unlock_ovs; 1092 } 1093 1094 if (unlikely(reply)) { 1095 error = ovs_flow_cmd_fill_info(new_flow, 1096 ovs_header->dp_ifindex, 1097 reply, info->snd_portid, 1098 info->snd_seq, 0, 1099 OVS_FLOW_CMD_NEW, 1100 ufid_flags); 1101 BUG_ON(error < 0); 1102 } 1103 ovs_unlock(); 1104 } else { 1105 struct sw_flow_actions *old_acts; 1106 1107 /* Bail out if we're not allowed to modify an existing flow. 1108 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL 1109 * because Generic Netlink treats the latter as a dump 1110 * request. We also accept NLM_F_EXCL in case that bug ever 1111 * gets fixed. 1112 */ 1113 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE 1114 | NLM_F_EXCL))) { 1115 error = -EEXIST; 1116 goto err_unlock_ovs; 1117 } 1118 1119 /* Look for any overlapping flow. */ 1120 if (unlikely(!ovs_flow_cmp(flow, &match))) { 1121 if (ovs_identifier_is_key(&flow->id)) 1122 flow = ovs_flow_tbl_lookup_exact(&dp->table, 1123 &match); 1124 else /* UFID matches but key is different */ 1125 flow = NULL; 1126 if (!flow) { 1127 error = -ENOENT; 1128 goto err_unlock_ovs; 1129 } 1130 } 1131 1132 if (unlikely(reply)) { 1133 size_t cur, req; 1134 1135 cur = ovs_flow_cmd_msg_size(acts, &new_flow->id, 1136 ufid_flags); 1137 req = ovs_flow_cmd_msg_size(acts, &flow->id, 1138 ufid_flags); 1139 if (cur < req) { 1140 struct sk_buff *resized; 1141 1142 resized = ovs_flow_cmd_alloc_info(acts, 1143 &flow->id, 1144 info, false, 1145 ufid_flags); 1146 if (IS_ERR(resized)) { 1147 error = PTR_ERR(resized); 1148 goto err_unlock_ovs; 1149 } 1150 kfree_skb(reply); 1151 reply = resized; 1152 } 1153 } 1154 1155 /* Update actions. */ 1156 old_acts = ovsl_dereference(flow->sf_acts); 1157 rcu_assign_pointer(flow->sf_acts, acts); 1158 1159 if (unlikely(reply)) { 1160 error = ovs_flow_cmd_fill_info(flow, 1161 ovs_header->dp_ifindex, 1162 reply, info->snd_portid, 1163 info->snd_seq, 0, 1164 OVS_FLOW_CMD_NEW, 1165 ufid_flags); 1166 BUG_ON(error < 0); 1167 } 1168 ovs_unlock(); 1169 1170 ovs_nla_free_flow_actions_rcu(old_acts); 1171 ovs_flow_free(new_flow, false); 1172 } 1173 1174 if (reply) 1175 ovs_notify(&dp_flow_genl_family, reply, info); 1176 1177 kfree(key); 1178 return 0; 1179 1180 err_unlock_ovs: 1181 ovs_unlock(); 1182 kfree_skb(reply); 1183 err_kfree_acts: 1184 ovs_nla_free_flow_actions(acts); 1185 err_kfree_key: 1186 kfree(key); 1187 err_kfree_flow: 1188 ovs_flow_free(new_flow, false); 1189 error: 1190 return error; 1191 } 1192 1193 /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */ 1194 static noinline_for_stack 1195 struct sw_flow_actions *get_flow_actions(struct net *net, 1196 const struct nlattr *a, 1197 const struct sw_flow_key *key, 1198 const struct sw_flow_mask *mask, 1199 bool log) 1200 { 1201 struct sw_flow_actions *acts; 1202 struct sw_flow_key masked_key; 1203 int error; 1204 1205 ovs_flow_mask_key(&masked_key, key, true, mask); 1206 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log); 1207 if (error) { 1208 OVS_NLERR(log, 1209 "Actions may not be safe on all matching packets"); 1210 return ERR_PTR(error); 1211 } 1212 1213 return acts; 1214 } 1215 1216 /* Factor out match-init and action-copy to avoid 1217 * "Wframe-larger-than=1024" warning. Because mask is only 1218 * used to get actions, we new a function to save some 1219 * stack space. 1220 * 1221 * If there are not key and action attrs, we return 0 1222 * directly. In the case, the caller will also not use the 1223 * match as before. If there is action attr, we try to get 1224 * actions and save them to *acts. Before returning from 1225 * the function, we reset the match->mask pointer. Because 1226 * we should not to return match object with dangling reference 1227 * to mask. 1228 * */ 1229 static noinline_for_stack int 1230 ovs_nla_init_match_and_action(struct net *net, 1231 struct sw_flow_match *match, 1232 struct sw_flow_key *key, 1233 struct nlattr **a, 1234 struct sw_flow_actions **acts, 1235 bool log) 1236 { 1237 struct sw_flow_mask mask; 1238 int error = 0; 1239 1240 if (a[OVS_FLOW_ATTR_KEY]) { 1241 ovs_match_init(match, key, true, &mask); 1242 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY], 1243 a[OVS_FLOW_ATTR_MASK], log); 1244 if (error) 1245 goto error; 1246 } 1247 1248 if (a[OVS_FLOW_ATTR_ACTIONS]) { 1249 if (!a[OVS_FLOW_ATTR_KEY]) { 1250 OVS_NLERR(log, 1251 "Flow key attribute not present in set flow."); 1252 error = -EINVAL; 1253 goto error; 1254 } 1255 1256 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key, 1257 &mask, log); 1258 if (IS_ERR(*acts)) { 1259 error = PTR_ERR(*acts); 1260 goto error; 1261 } 1262 } 1263 1264 /* On success, error is 0. */ 1265 error: 1266 match->mask = NULL; 1267 return error; 1268 } 1269 1270 static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info) 1271 { 1272 struct net *net = sock_net(skb->sk); 1273 struct nlattr **a = info->attrs; 1274 struct ovs_header *ovs_header = genl_info_userhdr(info); 1275 struct sw_flow_key key; 1276 struct sw_flow *flow; 1277 struct sk_buff *reply = NULL; 1278 struct datapath *dp; 1279 struct sw_flow_actions *old_acts = NULL, *acts = NULL; 1280 struct sw_flow_match match; 1281 struct sw_flow_id sfid; 1282 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1283 int error = 0; 1284 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1285 bool ufid_present; 1286 1287 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log); 1288 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) { 1289 OVS_NLERR(log, 1290 "Flow set message rejected, Key attribute missing."); 1291 return -EINVAL; 1292 } 1293 1294 error = ovs_nla_init_match_and_action(net, &match, &key, a, 1295 &acts, log); 1296 if (error) 1297 goto error; 1298 1299 if (acts) { 1300 /* Can allocate before locking if have acts. */ 1301 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false, 1302 ufid_flags); 1303 if (IS_ERR(reply)) { 1304 error = PTR_ERR(reply); 1305 goto err_kfree_acts; 1306 } 1307 } 1308 1309 ovs_lock(); 1310 dp = get_dp(net, ovs_header->dp_ifindex); 1311 if (unlikely(!dp)) { 1312 error = -ENODEV; 1313 goto err_unlock_ovs; 1314 } 1315 /* Check that the flow exists. */ 1316 if (ufid_present) 1317 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid); 1318 else 1319 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1320 if (unlikely(!flow)) { 1321 error = -ENOENT; 1322 goto err_unlock_ovs; 1323 } 1324 1325 /* Update actions, if present. */ 1326 if (likely(acts)) { 1327 old_acts = ovsl_dereference(flow->sf_acts); 1328 rcu_assign_pointer(flow->sf_acts, acts); 1329 1330 if (unlikely(reply)) { 1331 error = ovs_flow_cmd_fill_info(flow, 1332 ovs_header->dp_ifindex, 1333 reply, info->snd_portid, 1334 info->snd_seq, 0, 1335 OVS_FLOW_CMD_SET, 1336 ufid_flags); 1337 BUG_ON(error < 0); 1338 } 1339 } else { 1340 /* Could not alloc without acts before locking. */ 1341 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, 1342 info, OVS_FLOW_CMD_SET, false, 1343 ufid_flags); 1344 1345 if (IS_ERR(reply)) { 1346 error = PTR_ERR(reply); 1347 reply = NULL; 1348 goto err_unlock_ovs; 1349 } 1350 } 1351 1352 /* Clear stats. */ 1353 if (a[OVS_FLOW_ATTR_CLEAR]) 1354 ovs_flow_stats_clear(flow); 1355 ovs_unlock(); 1356 1357 if (reply) 1358 ovs_notify(&dp_flow_genl_family, reply, info); 1359 if (old_acts) 1360 ovs_nla_free_flow_actions_rcu(old_acts); 1361 1362 return 0; 1363 1364 err_unlock_ovs: 1365 ovs_unlock(); 1366 kfree_skb(reply); 1367 err_kfree_acts: 1368 ovs_nla_free_flow_actions(acts); 1369 error: 1370 return error; 1371 } 1372 1373 static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info) 1374 { 1375 struct nlattr **a = info->attrs; 1376 struct ovs_header *ovs_header = genl_info_userhdr(info); 1377 struct net *net = sock_net(skb->sk); 1378 struct sw_flow_key key; 1379 struct sk_buff *reply; 1380 struct sw_flow *flow; 1381 struct datapath *dp; 1382 struct sw_flow_match match; 1383 struct sw_flow_id ufid; 1384 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1385 int err = 0; 1386 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1387 bool ufid_present; 1388 1389 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log); 1390 if (a[OVS_FLOW_ATTR_KEY]) { 1391 ovs_match_init(&match, &key, true, NULL); 1392 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL, 1393 log); 1394 } else if (!ufid_present) { 1395 OVS_NLERR(log, 1396 "Flow get message rejected, Key attribute missing."); 1397 err = -EINVAL; 1398 } 1399 if (err) 1400 return err; 1401 1402 ovs_lock(); 1403 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 1404 if (!dp) { 1405 err = -ENODEV; 1406 goto unlock; 1407 } 1408 1409 if (ufid_present) 1410 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid); 1411 else 1412 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1413 if (!flow) { 1414 err = -ENOENT; 1415 goto unlock; 1416 } 1417 1418 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info, 1419 OVS_FLOW_CMD_GET, true, ufid_flags); 1420 if (IS_ERR(reply)) { 1421 err = PTR_ERR(reply); 1422 goto unlock; 1423 } 1424 1425 ovs_unlock(); 1426 return genlmsg_reply(reply, info); 1427 unlock: 1428 ovs_unlock(); 1429 return err; 1430 } 1431 1432 static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) 1433 { 1434 struct nlattr **a = info->attrs; 1435 struct ovs_header *ovs_header = genl_info_userhdr(info); 1436 struct net *net = sock_net(skb->sk); 1437 struct sw_flow_key key; 1438 struct sk_buff *reply; 1439 struct sw_flow *flow = NULL; 1440 struct datapath *dp; 1441 struct sw_flow_match match; 1442 struct sw_flow_id ufid; 1443 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1444 int err; 1445 bool log = !a[OVS_FLOW_ATTR_PROBE]; 1446 bool ufid_present; 1447 1448 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log); 1449 if (a[OVS_FLOW_ATTR_KEY]) { 1450 ovs_match_init(&match, &key, true, NULL); 1451 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], 1452 NULL, log); 1453 if (unlikely(err)) 1454 return err; 1455 } 1456 1457 ovs_lock(); 1458 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 1459 if (unlikely(!dp)) { 1460 err = -ENODEV; 1461 goto unlock; 1462 } 1463 1464 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) { 1465 err = ovs_flow_tbl_flush(&dp->table); 1466 goto unlock; 1467 } 1468 1469 if (ufid_present) 1470 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid); 1471 else 1472 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match); 1473 if (unlikely(!flow)) { 1474 err = -ENOENT; 1475 goto unlock; 1476 } 1477 1478 reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), 1479 &flow->id, info, false, ufid_flags); 1480 if (IS_ERR(reply)) { 1481 netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, 1482 PTR_ERR(reply)); 1483 reply = NULL; 1484 } 1485 1486 if (likely(reply)) { 1487 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, 1488 reply, info->snd_portid, 1489 info->snd_seq, 0, 1490 OVS_FLOW_CMD_DEL, ufid_flags); 1491 if (WARN_ON_ONCE(err < 0)) { 1492 kfree_skb(reply); 1493 reply = NULL; 1494 } 1495 } 1496 /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses 1497 * the flow->mask that can be scheduled to be freed by the 1498 * ovs_flow_tbl_remove() and we're not holding the RCU read lock. 1499 */ 1500 ovs_flow_tbl_remove(&dp->table, flow); 1501 ovs_unlock(); 1502 1503 if (likely(reply)) 1504 ovs_notify(&dp_flow_genl_family, reply, info); 1505 1506 ovs_flow_free(flow, true); 1507 return 0; 1508 unlock: 1509 ovs_unlock(); 1510 return err; 1511 } 1512 1513 static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 1514 { 1515 struct nlattr *a[__OVS_FLOW_ATTR_MAX]; 1516 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh)); 1517 struct table_instance *ti; 1518 struct datapath *dp; 1519 u32 ufid_flags; 1520 int err; 1521 1522 err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a, 1523 OVS_FLOW_ATTR_MAX, flow_policy, NULL); 1524 if (err) 1525 return err; 1526 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]); 1527 1528 rcu_read_lock(); 1529 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex); 1530 if (!dp) { 1531 rcu_read_unlock(); 1532 return -ENODEV; 1533 } 1534 1535 ti = rcu_dereference(dp->table.ti); 1536 for (;;) { 1537 struct sw_flow *flow; 1538 u32 bucket, obj; 1539 1540 bucket = cb->args[0]; 1541 obj = cb->args[1]; 1542 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj); 1543 if (!flow) 1544 break; 1545 1546 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb, 1547 NETLINK_CB(cb->skb).portid, 1548 cb->nlh->nlmsg_seq, NLM_F_MULTI, 1549 OVS_FLOW_CMD_GET, ufid_flags) < 0) 1550 break; 1551 1552 cb->args[0] = bucket; 1553 cb->args[1] = obj; 1554 } 1555 rcu_read_unlock(); 1556 return skb->len; 1557 } 1558 1559 static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = { 1560 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED }, 1561 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED }, 1562 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED }, 1563 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG }, 1564 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG }, 1565 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 }, 1566 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 }, 1567 }; 1568 1569 static const struct genl_small_ops dp_flow_genl_ops[] = { 1570 { .cmd = OVS_FLOW_CMD_NEW, 1571 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1572 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1573 .doit = ovs_flow_cmd_new 1574 }, 1575 { .cmd = OVS_FLOW_CMD_DEL, 1576 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1577 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1578 .doit = ovs_flow_cmd_del 1579 }, 1580 { .cmd = OVS_FLOW_CMD_GET, 1581 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1582 .flags = 0, /* OK for unprivileged users. */ 1583 .doit = ovs_flow_cmd_get, 1584 .dumpit = ovs_flow_cmd_dump 1585 }, 1586 { .cmd = OVS_FLOW_CMD_SET, 1587 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 1588 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 1589 .doit = ovs_flow_cmd_set, 1590 }, 1591 }; 1592 1593 static struct genl_family dp_flow_genl_family __ro_after_init = { 1594 .hdrsize = sizeof(struct ovs_header), 1595 .name = OVS_FLOW_FAMILY, 1596 .version = OVS_FLOW_VERSION, 1597 .maxattr = OVS_FLOW_ATTR_MAX, 1598 .policy = flow_policy, 1599 .netnsok = true, 1600 .parallel_ops = true, 1601 .small_ops = dp_flow_genl_ops, 1602 .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops), 1603 .resv_start_op = OVS_FLOW_CMD_SET + 1, 1604 .mcgrps = &ovs_dp_flow_multicast_group, 1605 .n_mcgrps = 1, 1606 .module = THIS_MODULE, 1607 }; 1608 1609 static size_t ovs_dp_cmd_msg_size(void) 1610 { 1611 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header)); 1612 1613 msgsize += nla_total_size(IFNAMSIZ); 1614 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats)); 1615 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats)); 1616 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */ 1617 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */ 1618 msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */ 1619 1620 return msgsize; 1621 } 1622 1623 /* Called with ovs_mutex. */ 1624 static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, 1625 u32 portid, u32 seq, u32 flags, u8 cmd) 1626 { 1627 struct ovs_header *ovs_header; 1628 struct ovs_dp_stats dp_stats; 1629 struct ovs_dp_megaflow_stats dp_megaflow_stats; 1630 struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids); 1631 int err, pids_len; 1632 1633 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family, 1634 flags, cmd); 1635 if (!ovs_header) 1636 goto error; 1637 1638 ovs_header->dp_ifindex = get_dpifindex(dp); 1639 1640 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp)); 1641 if (err) 1642 goto nla_put_failure; 1643 1644 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats); 1645 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats), 1646 &dp_stats, OVS_DP_ATTR_PAD)) 1647 goto nla_put_failure; 1648 1649 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS, 1650 sizeof(struct ovs_dp_megaflow_stats), 1651 &dp_megaflow_stats, OVS_DP_ATTR_PAD)) 1652 goto nla_put_failure; 1653 1654 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features)) 1655 goto nla_put_failure; 1656 1657 if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE, 1658 ovs_flow_tbl_masks_cache_size(&dp->table))) 1659 goto nla_put_failure; 1660 1661 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) { 1662 pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32); 1663 if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids)) 1664 goto nla_put_failure; 1665 } 1666 1667 genlmsg_end(skb, ovs_header); 1668 return 0; 1669 1670 nla_put_failure: 1671 genlmsg_cancel(skb, ovs_header); 1672 error: 1673 return -EMSGSIZE; 1674 } 1675 1676 static struct sk_buff *ovs_dp_cmd_alloc_info(void) 1677 { 1678 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL); 1679 } 1680 1681 /* Called with rcu_read_lock or ovs_mutex. */ 1682 static struct datapath *lookup_datapath(struct net *net, 1683 const struct ovs_header *ovs_header, 1684 struct nlattr *a[OVS_DP_ATTR_MAX + 1]) 1685 { 1686 struct datapath *dp; 1687 1688 if (!a[OVS_DP_ATTR_NAME]) 1689 dp = get_dp(net, ovs_header->dp_ifindex); 1690 else { 1691 struct vport *vport; 1692 1693 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME])); 1694 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL; 1695 } 1696 return dp ? dp : ERR_PTR(-ENODEV); 1697 } 1698 1699 static void ovs_dp_reset_user_features(struct sk_buff *skb, 1700 struct genl_info *info) 1701 { 1702 struct datapath *dp; 1703 1704 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 1705 info->attrs); 1706 if (IS_ERR(dp)) 1707 return; 1708 1709 pr_warn("%s: Dropping previously announced user features\n", 1710 ovs_dp_name(dp)); 1711 dp->user_features = 0; 1712 } 1713 1714 static int ovs_dp_set_upcall_portids(struct datapath *dp, 1715 const struct nlattr *ids) 1716 { 1717 struct dp_nlsk_pids *old, *dp_nlsk_pids; 1718 1719 if (!nla_len(ids) || nla_len(ids) % sizeof(u32)) 1720 return -EINVAL; 1721 1722 old = ovsl_dereference(dp->upcall_portids); 1723 1724 dp_nlsk_pids = kmalloc(sizeof(*dp_nlsk_pids) + nla_len(ids), 1725 GFP_KERNEL); 1726 if (!dp_nlsk_pids) 1727 return -ENOMEM; 1728 1729 dp_nlsk_pids->n_pids = nla_len(ids) / sizeof(u32); 1730 nla_memcpy(dp_nlsk_pids->pids, ids, nla_len(ids)); 1731 1732 rcu_assign_pointer(dp->upcall_portids, dp_nlsk_pids); 1733 1734 kfree_rcu(old, rcu); 1735 1736 return 0; 1737 } 1738 1739 u32 ovs_dp_get_upcall_portid(const struct datapath *dp, uint32_t cpu_id) 1740 { 1741 struct dp_nlsk_pids *dp_nlsk_pids; 1742 1743 dp_nlsk_pids = rcu_dereference(dp->upcall_portids); 1744 1745 if (dp_nlsk_pids) { 1746 if (cpu_id < dp_nlsk_pids->n_pids) { 1747 return dp_nlsk_pids->pids[cpu_id]; 1748 } else if (dp_nlsk_pids->n_pids > 0 && 1749 cpu_id >= dp_nlsk_pids->n_pids) { 1750 /* If the number of netlink PIDs is mismatched with 1751 * the number of CPUs as seen by the kernel, log this 1752 * and send the upcall to an arbitrary socket (0) in 1753 * order to not drop packets 1754 */ 1755 pr_info_ratelimited("cpu_id mismatch with handler threads"); 1756 return dp_nlsk_pids->pids[cpu_id % 1757 dp_nlsk_pids->n_pids]; 1758 } else { 1759 return 0; 1760 } 1761 } else { 1762 return 0; 1763 } 1764 } 1765 1766 static int ovs_dp_change(struct datapath *dp, struct nlattr *a[]) 1767 { 1768 u32 user_features = 0, old_features = dp->user_features; 1769 int err; 1770 1771 if (a[OVS_DP_ATTR_USER_FEATURES]) { 1772 user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]); 1773 1774 if (user_features & ~(OVS_DP_F_VPORT_PIDS | 1775 OVS_DP_F_UNALIGNED | 1776 OVS_DP_F_TC_RECIRC_SHARING | 1777 OVS_DP_F_DISPATCH_UPCALL_PER_CPU)) 1778 return -EOPNOTSUPP; 1779 1780 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 1781 if (user_features & OVS_DP_F_TC_RECIRC_SHARING) 1782 return -EOPNOTSUPP; 1783 #endif 1784 } 1785 1786 if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) { 1787 int err; 1788 u32 cache_size; 1789 1790 cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]); 1791 err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size); 1792 if (err) 1793 return err; 1794 } 1795 1796 dp->user_features = user_features; 1797 1798 if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && 1799 a[OVS_DP_ATTR_PER_CPU_PIDS]) { 1800 /* Upcall Netlink Port IDs have been updated */ 1801 err = ovs_dp_set_upcall_portids(dp, 1802 a[OVS_DP_ATTR_PER_CPU_PIDS]); 1803 if (err) 1804 return err; 1805 } 1806 1807 if ((dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) && 1808 !(old_features & OVS_DP_F_TC_RECIRC_SHARING)) 1809 tc_skb_ext_tc_enable(); 1810 else if (!(dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) && 1811 (old_features & OVS_DP_F_TC_RECIRC_SHARING)) 1812 tc_skb_ext_tc_disable(); 1813 1814 return 0; 1815 } 1816 1817 static int ovs_dp_stats_init(struct datapath *dp) 1818 { 1819 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu); 1820 if (!dp->stats_percpu) 1821 return -ENOMEM; 1822 1823 return 0; 1824 } 1825 1826 static int ovs_dp_vport_init(struct datapath *dp) 1827 { 1828 int i; 1829 1830 dp->ports = kmalloc_objs(struct hlist_head, DP_VPORT_HASH_BUCKETS); 1831 if (!dp->ports) 1832 return -ENOMEM; 1833 1834 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) 1835 INIT_HLIST_HEAD(&dp->ports[i]); 1836 1837 return 0; 1838 } 1839 1840 static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) 1841 { 1842 struct nlattr **a = info->attrs; 1843 struct vport_parms parms; 1844 struct sk_buff *reply; 1845 struct datapath *dp; 1846 struct vport *vport; 1847 struct ovs_net *ovs_net; 1848 int err; 1849 1850 err = -EINVAL; 1851 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID]) 1852 goto err; 1853 1854 reply = ovs_dp_cmd_alloc_info(); 1855 if (!reply) 1856 return -ENOMEM; 1857 1858 err = -ENOMEM; 1859 dp = kzalloc_obj(*dp); 1860 if (dp == NULL) 1861 goto err_destroy_reply; 1862 1863 ovs_dp_set_net(dp, sock_net(skb->sk)); 1864 1865 /* Allocate table. */ 1866 err = ovs_flow_tbl_init(&dp->table); 1867 if (err) 1868 goto err_destroy_dp; 1869 1870 err = ovs_dp_stats_init(dp); 1871 if (err) 1872 goto err_destroy_table; 1873 1874 err = ovs_dp_vport_init(dp); 1875 if (err) 1876 goto err_destroy_stats; 1877 1878 err = ovs_meters_init(dp); 1879 if (err) 1880 goto err_destroy_ports; 1881 1882 /* Set up our datapath device. */ 1883 parms.name = nla_data(a[OVS_DP_ATTR_NAME]); 1884 parms.type = OVS_VPORT_TYPE_INTERNAL; 1885 parms.dp = dp; 1886 parms.port_no = OVSP_LOCAL; 1887 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID]; 1888 parms.desired_ifindex = nla_get_s32_default(a[OVS_DP_ATTR_IFINDEX], 0); 1889 1890 /* So far only local changes have been made, now need the lock. */ 1891 ovs_lock(); 1892 1893 err = ovs_dp_change(dp, a); 1894 if (err) 1895 goto err_unlock_and_destroy_meters; 1896 1897 vport = new_vport(&parms); 1898 if (IS_ERR(vport)) { 1899 err = PTR_ERR(vport); 1900 if (err == -EBUSY) 1901 err = -EEXIST; 1902 1903 if (err == -EEXIST) { 1904 /* An outdated user space instance that does not understand 1905 * the concept of user_features has attempted to create a new 1906 * datapath and is likely to reuse it. Drop all user features. 1907 */ 1908 if (info->genlhdr->version < OVS_DP_VER_FEATURES) 1909 ovs_dp_reset_user_features(skb, info); 1910 } 1911 1912 goto err_destroy_portids; 1913 } 1914 1915 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 1916 info->snd_seq, 0, OVS_DP_CMD_NEW); 1917 BUG_ON(err < 0); 1918 1919 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id); 1920 list_add_tail_rcu(&dp->list_node, &ovs_net->dps); 1921 1922 ovs_unlock(); 1923 1924 ovs_notify(&dp_datapath_genl_family, reply, info); 1925 return 0; 1926 1927 err_destroy_portids: 1928 kfree(rcu_dereference_raw(dp->upcall_portids)); 1929 err_unlock_and_destroy_meters: 1930 ovs_unlock(); 1931 ovs_meters_exit(dp); 1932 err_destroy_ports: 1933 kfree(dp->ports); 1934 err_destroy_stats: 1935 free_percpu(dp->stats_percpu); 1936 err_destroy_table: 1937 ovs_flow_tbl_destroy(&dp->table); 1938 err_destroy_dp: 1939 kfree(dp); 1940 err_destroy_reply: 1941 kfree_skb(reply); 1942 err: 1943 return err; 1944 } 1945 1946 /* Called with ovs_mutex. */ 1947 static void __dp_destroy(struct datapath *dp) 1948 { 1949 struct flow_table *table = &dp->table; 1950 int i; 1951 1952 if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING) 1953 tc_skb_ext_tc_disable(); 1954 1955 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 1956 struct vport *vport; 1957 struct hlist_node *n; 1958 1959 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node) 1960 if (vport->port_no != OVSP_LOCAL) 1961 ovs_dp_detach_port(vport); 1962 } 1963 1964 list_del_rcu(&dp->list_node); 1965 1966 /* OVSP_LOCAL is datapath internal port. We need to make sure that 1967 * all ports in datapath are destroyed first before freeing datapath. 1968 */ 1969 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL)); 1970 1971 /* Flush sw_flow in the tables. RCU cb only releases resource 1972 * such as dp, ports and tables. That may avoid some issues 1973 * such as RCU usage warning. 1974 */ 1975 table_instance_flow_flush(table, ovsl_dereference(table->ti), 1976 ovsl_dereference(table->ufid_ti)); 1977 1978 /* RCU destroy the ports, meters and flow tables. */ 1979 call_rcu(&dp->rcu, destroy_dp_rcu); 1980 } 1981 1982 static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info) 1983 { 1984 struct sk_buff *reply; 1985 struct datapath *dp; 1986 int err; 1987 1988 reply = ovs_dp_cmd_alloc_info(); 1989 if (!reply) 1990 return -ENOMEM; 1991 1992 ovs_lock(); 1993 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 1994 info->attrs); 1995 err = PTR_ERR(dp); 1996 if (IS_ERR(dp)) 1997 goto err_unlock_free; 1998 1999 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2000 info->snd_seq, 0, OVS_DP_CMD_DEL); 2001 BUG_ON(err < 0); 2002 2003 __dp_destroy(dp); 2004 ovs_unlock(); 2005 2006 ovs_notify(&dp_datapath_genl_family, reply, info); 2007 2008 return 0; 2009 2010 err_unlock_free: 2011 ovs_unlock(); 2012 kfree_skb(reply); 2013 return err; 2014 } 2015 2016 static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info) 2017 { 2018 struct sk_buff *reply; 2019 struct datapath *dp; 2020 int err; 2021 2022 reply = ovs_dp_cmd_alloc_info(); 2023 if (!reply) 2024 return -ENOMEM; 2025 2026 ovs_lock(); 2027 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 2028 info->attrs); 2029 err = PTR_ERR(dp); 2030 if (IS_ERR(dp)) 2031 goto err_unlock_free; 2032 2033 err = ovs_dp_change(dp, info->attrs); 2034 if (err) 2035 goto err_unlock_free; 2036 2037 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2038 info->snd_seq, 0, OVS_DP_CMD_SET); 2039 BUG_ON(err < 0); 2040 2041 ovs_unlock(); 2042 ovs_notify(&dp_datapath_genl_family, reply, info); 2043 2044 return 0; 2045 2046 err_unlock_free: 2047 ovs_unlock(); 2048 kfree_skb(reply); 2049 return err; 2050 } 2051 2052 static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info) 2053 { 2054 struct sk_buff *reply; 2055 struct datapath *dp; 2056 int err; 2057 2058 reply = ovs_dp_cmd_alloc_info(); 2059 if (!reply) 2060 return -ENOMEM; 2061 2062 ovs_lock(); 2063 dp = lookup_datapath(sock_net(skb->sk), genl_info_userhdr(info), 2064 info->attrs); 2065 if (IS_ERR(dp)) { 2066 err = PTR_ERR(dp); 2067 goto err_unlock_free; 2068 } 2069 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, 2070 info->snd_seq, 0, OVS_DP_CMD_GET); 2071 BUG_ON(err < 0); 2072 ovs_unlock(); 2073 2074 return genlmsg_reply(reply, info); 2075 2076 err_unlock_free: 2077 ovs_unlock(); 2078 kfree_skb(reply); 2079 return err; 2080 } 2081 2082 static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 2083 { 2084 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); 2085 struct datapath *dp; 2086 int skip = cb->args[0]; 2087 int i = 0; 2088 2089 ovs_lock(); 2090 list_for_each_entry(dp, &ovs_net->dps, list_node) { 2091 if (i >= skip && 2092 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid, 2093 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2094 OVS_DP_CMD_GET) < 0) 2095 break; 2096 i++; 2097 } 2098 ovs_unlock(); 2099 2100 cb->args[0] = i; 2101 2102 return skb->len; 2103 } 2104 2105 static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = { 2106 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, 2107 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 }, 2108 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 }, 2109 [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0, 2110 PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)), 2111 [OVS_DP_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0), 2112 }; 2113 2114 static const struct genl_small_ops dp_datapath_genl_ops[] = { 2115 { .cmd = OVS_DP_CMD_NEW, 2116 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2117 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2118 .doit = ovs_dp_cmd_new 2119 }, 2120 { .cmd = OVS_DP_CMD_DEL, 2121 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2122 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2123 .doit = ovs_dp_cmd_del 2124 }, 2125 { .cmd = OVS_DP_CMD_GET, 2126 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2127 .flags = 0, /* OK for unprivileged users. */ 2128 .doit = ovs_dp_cmd_get, 2129 .dumpit = ovs_dp_cmd_dump 2130 }, 2131 { .cmd = OVS_DP_CMD_SET, 2132 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2133 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2134 .doit = ovs_dp_cmd_set, 2135 }, 2136 }; 2137 2138 static struct genl_family dp_datapath_genl_family __ro_after_init = { 2139 .hdrsize = sizeof(struct ovs_header), 2140 .name = OVS_DATAPATH_FAMILY, 2141 .version = OVS_DATAPATH_VERSION, 2142 .maxattr = OVS_DP_ATTR_MAX, 2143 .policy = datapath_policy, 2144 .netnsok = true, 2145 .parallel_ops = true, 2146 .small_ops = dp_datapath_genl_ops, 2147 .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops), 2148 .resv_start_op = OVS_DP_CMD_SET + 1, 2149 .mcgrps = &ovs_dp_datapath_multicast_group, 2150 .n_mcgrps = 1, 2151 .module = THIS_MODULE, 2152 }; 2153 2154 /* Called with ovs_mutex or RCU read lock. */ 2155 static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb, 2156 struct net *net, u32 portid, u32 seq, 2157 u32 flags, u8 cmd, gfp_t gfp) 2158 { 2159 struct ovs_header *ovs_header; 2160 struct ovs_vport_stats vport_stats; 2161 struct net *net_vport; 2162 int err; 2163 2164 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family, 2165 flags, cmd); 2166 if (!ovs_header) 2167 return -EMSGSIZE; 2168 2169 ovs_header->dp_ifindex = get_dpifindex(vport->dp); 2170 2171 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) || 2172 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) || 2173 nla_put_string(skb, OVS_VPORT_ATTR_NAME, 2174 ovs_vport_name(vport)) || 2175 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex)) 2176 goto nla_put_failure; 2177 2178 rcu_read_lock(); 2179 net_vport = dev_net_rcu(vport->dev); 2180 if (!net_eq(net, net_vport)) { 2181 int id = peernet2id_alloc(net, net_vport, GFP_ATOMIC); 2182 2183 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id)) 2184 goto nla_put_failure_unlock; 2185 } 2186 rcu_read_unlock(); 2187 2188 ovs_vport_get_stats(vport, &vport_stats); 2189 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS, 2190 sizeof(struct ovs_vport_stats), &vport_stats, 2191 OVS_VPORT_ATTR_PAD)) 2192 goto nla_put_failure; 2193 2194 if (ovs_vport_get_upcall_stats(vport, skb)) 2195 goto nla_put_failure; 2196 2197 if (ovs_vport_get_upcall_portids(vport, skb)) 2198 goto nla_put_failure; 2199 2200 genlmsg_end(skb, ovs_header); 2201 return 0; 2202 2203 nla_put_failure_unlock: 2204 rcu_read_unlock(); 2205 nla_put_failure: 2206 err = -EMSGSIZE; 2207 genlmsg_cancel(skb, ovs_header); 2208 return err; 2209 } 2210 2211 static size_t ovs_vport_cmd_msg_size(void) 2212 { 2213 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header)); 2214 2215 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_PORT_NO */ 2216 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_TYPE */ 2217 msgsize += nla_total_size(IFNAMSIZ); /* OVS_VPORT_ATTR_NAME */ 2218 msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_IFINDEX */ 2219 msgsize += nla_total_size(sizeof(s32)); /* OVS_VPORT_ATTR_NETNSID */ 2220 2221 /* OVS_VPORT_ATTR_STATS */ 2222 msgsize += nla_total_size_64bit(sizeof(struct ovs_vport_stats)); 2223 2224 /* OVS_VPORT_ATTR_UPCALL_STATS(OVS_VPORT_UPCALL_ATTR_SUCCESS + 2225 * OVS_VPORT_UPCALL_ATTR_FAIL) 2226 */ 2227 msgsize += nla_total_size(nla_total_size_64bit(sizeof(u64)) + 2228 nla_total_size_64bit(sizeof(u64))); 2229 2230 /* OVS_VPORT_ATTR_UPCALL_PID */ 2231 msgsize += nla_total_size(nr_cpu_ids * sizeof(u32)); 2232 2233 return msgsize; 2234 } 2235 2236 static struct sk_buff *ovs_vport_cmd_alloc_info(void) 2237 { 2238 return genlmsg_new(ovs_vport_cmd_msg_size(), GFP_KERNEL); 2239 } 2240 2241 /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */ 2242 struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net, 2243 u32 portid, u32 seq, u8 cmd) 2244 { 2245 struct sk_buff *skb; 2246 int retval; 2247 2248 skb = ovs_vport_cmd_alloc_info(); 2249 if (!skb) 2250 return ERR_PTR(-ENOMEM); 2251 2252 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd, 2253 GFP_KERNEL); 2254 BUG_ON(retval < 0); 2255 2256 return skb; 2257 } 2258 2259 /* Called with ovs_mutex or RCU read lock. */ 2260 static struct vport *lookup_vport(struct net *net, 2261 const struct ovs_header *ovs_header, 2262 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1]) 2263 { 2264 struct datapath *dp; 2265 struct vport *vport; 2266 2267 if (a[OVS_VPORT_ATTR_IFINDEX]) 2268 return ERR_PTR(-EOPNOTSUPP); 2269 if (a[OVS_VPORT_ATTR_NAME]) { 2270 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME])); 2271 if (!vport) 2272 return ERR_PTR(-ENODEV); 2273 if (ovs_header->dp_ifindex && 2274 ovs_header->dp_ifindex != get_dpifindex(vport->dp)) 2275 return ERR_PTR(-ENODEV); 2276 return vport; 2277 } else if (a[OVS_VPORT_ATTR_PORT_NO]) { 2278 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]); 2279 2280 if (port_no >= DP_MAX_PORTS) 2281 return ERR_PTR(-EFBIG); 2282 2283 dp = get_dp(net, ovs_header->dp_ifindex); 2284 if (!dp) 2285 return ERR_PTR(-ENODEV); 2286 2287 vport = ovs_vport_ovsl_rcu(dp, port_no); 2288 if (!vport) 2289 return ERR_PTR(-ENODEV); 2290 return vport; 2291 } else 2292 return ERR_PTR(-EINVAL); 2293 2294 } 2295 2296 static unsigned int ovs_get_max_headroom(struct datapath *dp) 2297 { 2298 unsigned int dev_headroom, max_headroom = 0; 2299 struct net_device *dev; 2300 struct vport *vport; 2301 int i; 2302 2303 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2304 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node, 2305 lockdep_ovsl_is_held()) { 2306 dev = vport->dev; 2307 dev_headroom = netdev_get_fwd_headroom(dev); 2308 if (dev_headroom > max_headroom) 2309 max_headroom = dev_headroom; 2310 } 2311 } 2312 2313 return max_headroom; 2314 } 2315 2316 /* Called with ovs_mutex */ 2317 static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom) 2318 { 2319 struct vport *vport; 2320 int i; 2321 2322 dp->max_headroom = new_headroom; 2323 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2324 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node, 2325 lockdep_ovsl_is_held()) 2326 netdev_set_rx_headroom(vport->dev, new_headroom); 2327 } 2328 } 2329 2330 static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info) 2331 { 2332 struct nlattr **a = info->attrs; 2333 struct ovs_header *ovs_header = genl_info_userhdr(info); 2334 struct vport_parms parms; 2335 struct sk_buff *reply; 2336 struct vport *vport; 2337 struct datapath *dp; 2338 unsigned int new_headroom; 2339 u32 port_no; 2340 int err; 2341 2342 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] || 2343 !a[OVS_VPORT_ATTR_UPCALL_PID]) 2344 return -EINVAL; 2345 2346 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]); 2347 2348 if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL) 2349 return -EOPNOTSUPP; 2350 2351 port_no = nla_get_u32_default(a[OVS_VPORT_ATTR_PORT_NO], 0); 2352 if (port_no >= DP_MAX_PORTS) 2353 return -EFBIG; 2354 2355 reply = ovs_vport_cmd_alloc_info(); 2356 if (!reply) 2357 return -ENOMEM; 2358 2359 ovs_lock(); 2360 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex); 2361 err = -ENODEV; 2362 if (!dp) 2363 goto exit_unlock_free; 2364 2365 if (port_no) { 2366 vport = ovs_vport_ovsl(dp, port_no); 2367 err = -EBUSY; 2368 if (vport) 2369 goto exit_unlock_free; 2370 } else { 2371 for (port_no = 1; ; port_no++) { 2372 if (port_no >= DP_MAX_PORTS) { 2373 err = -EFBIG; 2374 goto exit_unlock_free; 2375 } 2376 vport = ovs_vport_ovsl(dp, port_no); 2377 if (!vport) 2378 break; 2379 } 2380 } 2381 2382 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]); 2383 parms.dp = dp; 2384 parms.port_no = port_no; 2385 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID]; 2386 parms.desired_ifindex = nla_get_s32_default(a[OVS_VPORT_ATTR_IFINDEX], 2387 0); 2388 2389 vport = new_vport(&parms); 2390 err = PTR_ERR(vport); 2391 if (IS_ERR(vport)) 2392 goto exit_unlock_free; 2393 2394 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2395 info->snd_portid, info->snd_seq, 0, 2396 OVS_VPORT_CMD_NEW, GFP_KERNEL); 2397 2398 new_headroom = netdev_get_fwd_headroom(vport->dev); 2399 2400 if (new_headroom > dp->max_headroom) 2401 ovs_update_headroom(dp, new_headroom); 2402 else 2403 netdev_set_rx_headroom(vport->dev, dp->max_headroom); 2404 2405 BUG_ON(err < 0); 2406 ovs_unlock(); 2407 2408 ovs_notify(&dp_vport_genl_family, reply, info); 2409 return 0; 2410 2411 exit_unlock_free: 2412 ovs_unlock(); 2413 kfree_skb(reply); 2414 return err; 2415 } 2416 2417 static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info) 2418 { 2419 struct nlattr **a = info->attrs; 2420 struct sk_buff *reply; 2421 struct vport *vport; 2422 int err; 2423 2424 reply = ovs_vport_cmd_alloc_info(); 2425 if (!reply) 2426 return -ENOMEM; 2427 2428 ovs_lock(); 2429 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a); 2430 err = PTR_ERR(vport); 2431 if (IS_ERR(vport)) 2432 goto exit_unlock_free; 2433 2434 if (a[OVS_VPORT_ATTR_TYPE] && 2435 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) { 2436 err = -EINVAL; 2437 goto exit_unlock_free; 2438 } 2439 2440 if (a[OVS_VPORT_ATTR_OPTIONS]) { 2441 /* There are no vport types that support legacy options. */ 2442 err = -EOPNOTSUPP; 2443 goto exit_unlock_free; 2444 } 2445 2446 if (a[OVS_VPORT_ATTR_UPCALL_PID]) { 2447 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID]; 2448 2449 err = ovs_vport_set_upcall_portids(vport, ids); 2450 if (err) 2451 goto exit_unlock_free; 2452 } 2453 2454 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2455 info->snd_portid, info->snd_seq, 0, 2456 OVS_VPORT_CMD_SET, GFP_KERNEL); 2457 BUG_ON(err < 0); 2458 2459 ovs_unlock(); 2460 ovs_notify(&dp_vport_genl_family, reply, info); 2461 return 0; 2462 2463 exit_unlock_free: 2464 ovs_unlock(); 2465 kfree_skb(reply); 2466 return err; 2467 } 2468 2469 static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info) 2470 { 2471 bool update_headroom = false; 2472 struct nlattr **a = info->attrs; 2473 struct sk_buff *reply; 2474 struct datapath *dp; 2475 struct vport *vport; 2476 unsigned int new_headroom; 2477 int err; 2478 2479 reply = ovs_vport_cmd_alloc_info(); 2480 if (!reply) 2481 return -ENOMEM; 2482 2483 ovs_lock(); 2484 vport = lookup_vport(sock_net(skb->sk), genl_info_userhdr(info), a); 2485 err = PTR_ERR(vport); 2486 if (IS_ERR(vport)) 2487 goto exit_unlock_free; 2488 2489 if (vport->port_no == OVSP_LOCAL) { 2490 err = -EINVAL; 2491 goto exit_unlock_free; 2492 } 2493 2494 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2495 info->snd_portid, info->snd_seq, 0, 2496 OVS_VPORT_CMD_DEL, GFP_KERNEL); 2497 BUG_ON(err < 0); 2498 2499 /* the vport deletion may trigger dp headroom update */ 2500 dp = vport->dp; 2501 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom) 2502 update_headroom = true; 2503 2504 netdev_reset_rx_headroom(vport->dev); 2505 ovs_dp_detach_port(vport); 2506 2507 if (update_headroom) { 2508 new_headroom = ovs_get_max_headroom(dp); 2509 2510 if (new_headroom < dp->max_headroom) 2511 ovs_update_headroom(dp, new_headroom); 2512 } 2513 ovs_unlock(); 2514 2515 ovs_notify(&dp_vport_genl_family, reply, info); 2516 return 0; 2517 2518 exit_unlock_free: 2519 ovs_unlock(); 2520 kfree_skb(reply); 2521 return err; 2522 } 2523 2524 static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info) 2525 { 2526 struct nlattr **a = info->attrs; 2527 struct ovs_header *ovs_header = genl_info_userhdr(info); 2528 struct sk_buff *reply; 2529 struct vport *vport; 2530 int err; 2531 2532 reply = ovs_vport_cmd_alloc_info(); 2533 if (!reply) 2534 return -ENOMEM; 2535 2536 rcu_read_lock(); 2537 vport = lookup_vport(sock_net(skb->sk), ovs_header, a); 2538 err = PTR_ERR(vport); 2539 if (IS_ERR(vport)) 2540 goto exit_unlock_free; 2541 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info), 2542 info->snd_portid, info->snd_seq, 0, 2543 OVS_VPORT_CMD_GET, GFP_ATOMIC); 2544 BUG_ON(err < 0); 2545 rcu_read_unlock(); 2546 2547 return genlmsg_reply(reply, info); 2548 2549 exit_unlock_free: 2550 rcu_read_unlock(); 2551 kfree_skb(reply); 2552 return err; 2553 } 2554 2555 static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) 2556 { 2557 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh)); 2558 struct datapath *dp; 2559 int bucket = cb->args[0], skip = cb->args[1]; 2560 int i, j = 0; 2561 2562 rcu_read_lock(); 2563 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex); 2564 if (!dp) { 2565 rcu_read_unlock(); 2566 return -ENODEV; 2567 } 2568 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) { 2569 struct vport *vport; 2570 2571 j = 0; 2572 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) { 2573 if (j >= skip && 2574 ovs_vport_cmd_fill_info(vport, skb, 2575 sock_net(skb->sk), 2576 NETLINK_CB(cb->skb).portid, 2577 cb->nlh->nlmsg_seq, 2578 NLM_F_MULTI, 2579 OVS_VPORT_CMD_GET, 2580 GFP_ATOMIC) < 0) 2581 goto out; 2582 2583 j++; 2584 } 2585 skip = 0; 2586 } 2587 out: 2588 rcu_read_unlock(); 2589 2590 cb->args[0] = i; 2591 cb->args[1] = j; 2592 2593 return skb->len; 2594 } 2595 2596 static void ovs_dp_masks_rebalance(struct work_struct *work) 2597 { 2598 struct ovs_net *ovs_net = container_of(work, struct ovs_net, 2599 masks_rebalance.work); 2600 struct datapath *dp; 2601 2602 ovs_lock(); 2603 2604 list_for_each_entry(dp, &ovs_net->dps, list_node) 2605 ovs_flow_masks_rebalance(&dp->table); 2606 2607 ovs_unlock(); 2608 2609 schedule_delayed_work(&ovs_net->masks_rebalance, 2610 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL)); 2611 } 2612 2613 static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = { 2614 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, 2615 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) }, 2616 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 }, 2617 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 }, 2618 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC }, 2619 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED }, /* Unused. */ 2620 [OVS_VPORT_ATTR_IFINDEX] = NLA_POLICY_MIN(NLA_S32, 0), 2621 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 }, 2622 [OVS_VPORT_ATTR_UPCALL_STATS] = { .type = NLA_NESTED }, 2623 }; 2624 2625 static const struct genl_small_ops dp_vport_genl_ops[] = { 2626 { .cmd = OVS_VPORT_CMD_NEW, 2627 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2628 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2629 .doit = ovs_vport_cmd_new 2630 }, 2631 { .cmd = OVS_VPORT_CMD_DEL, 2632 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2633 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2634 .doit = ovs_vport_cmd_del 2635 }, 2636 { .cmd = OVS_VPORT_CMD_GET, 2637 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2638 .flags = 0, /* OK for unprivileged users. */ 2639 .doit = ovs_vport_cmd_get, 2640 .dumpit = ovs_vport_cmd_dump 2641 }, 2642 { .cmd = OVS_VPORT_CMD_SET, 2643 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, 2644 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */ 2645 .doit = ovs_vport_cmd_set, 2646 }, 2647 }; 2648 2649 struct genl_family dp_vport_genl_family __ro_after_init = { 2650 .hdrsize = sizeof(struct ovs_header), 2651 .name = OVS_VPORT_FAMILY, 2652 .version = OVS_VPORT_VERSION, 2653 .maxattr = OVS_VPORT_ATTR_MAX, 2654 .policy = vport_policy, 2655 .netnsok = true, 2656 .parallel_ops = true, 2657 .small_ops = dp_vport_genl_ops, 2658 .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops), 2659 .resv_start_op = OVS_VPORT_CMD_SET + 1, 2660 .mcgrps = &ovs_dp_vport_multicast_group, 2661 .n_mcgrps = 1, 2662 .module = THIS_MODULE, 2663 }; 2664 2665 static struct genl_family * const dp_genl_families[] = { 2666 &dp_datapath_genl_family, 2667 &dp_vport_genl_family, 2668 &dp_flow_genl_family, 2669 &dp_packet_genl_family, 2670 &dp_meter_genl_family, 2671 #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) 2672 &dp_ct_limit_genl_family, 2673 #endif 2674 }; 2675 2676 static void dp_unregister_genl(int n_families) 2677 { 2678 int i; 2679 2680 for (i = 0; i < n_families; i++) 2681 genl_unregister_family(dp_genl_families[i]); 2682 } 2683 2684 static int __init dp_register_genl(void) 2685 { 2686 int err; 2687 int i; 2688 2689 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) { 2690 2691 err = genl_register_family(dp_genl_families[i]); 2692 if (err) 2693 goto error; 2694 } 2695 2696 return 0; 2697 2698 error: 2699 dp_unregister_genl(i); 2700 return err; 2701 } 2702 2703 static int __net_init ovs_init_net(struct net *net) 2704 { 2705 struct ovs_net *ovs_net = net_generic(net, ovs_net_id); 2706 int err; 2707 2708 INIT_LIST_HEAD(&ovs_net->dps); 2709 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq); 2710 INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance); 2711 2712 err = ovs_ct_init(net); 2713 if (err) 2714 return err; 2715 2716 schedule_delayed_work(&ovs_net->masks_rebalance, 2717 msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL)); 2718 return 0; 2719 } 2720 2721 static void __net_exit list_vports_from_net(struct net *net, struct net *dnet, 2722 struct list_head *head) 2723 { 2724 struct ovs_net *ovs_net = net_generic(net, ovs_net_id); 2725 struct datapath *dp; 2726 2727 list_for_each_entry(dp, &ovs_net->dps, list_node) { 2728 int i; 2729 2730 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { 2731 struct vport *vport; 2732 2733 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) { 2734 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL) 2735 continue; 2736 2737 if (dev_net(vport->dev) == dnet) 2738 list_add(&vport->detach_list, head); 2739 } 2740 } 2741 } 2742 } 2743 2744 static void __net_exit ovs_pre_exit_net(struct net *dnet) 2745 { 2746 ovs_lock(); 2747 ovs_ct_exit_start(dnet); 2748 ovs_unlock(); 2749 } 2750 2751 static void __net_exit ovs_exit_net(struct net *dnet) 2752 { 2753 struct datapath *dp, *dp_next; 2754 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id); 2755 struct vport *vport, *vport_next; 2756 struct net *net; 2757 LIST_HEAD(head); 2758 2759 ovs_lock(); 2760 2761 ovs_ct_exit_finish(dnet); 2762 2763 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node) 2764 __dp_destroy(dp); 2765 2766 down_read(&net_rwsem); 2767 for_each_net(net) 2768 list_vports_from_net(net, dnet, &head); 2769 up_read(&net_rwsem); 2770 2771 /* Detach all vports from given namespace. */ 2772 list_for_each_entry_safe(vport, vport_next, &head, detach_list) { 2773 list_del(&vport->detach_list); 2774 ovs_dp_detach_port(vport); 2775 } 2776 2777 ovs_unlock(); 2778 2779 cancel_delayed_work_sync(&ovs_net->masks_rebalance); 2780 cancel_work_sync(&ovs_net->dp_notify_work); 2781 } 2782 2783 static struct pernet_operations ovs_net_ops = { 2784 .init = ovs_init_net, 2785 .pre_exit = ovs_pre_exit_net, 2786 .exit = ovs_exit_net, 2787 .id = &ovs_net_id, 2788 .size = sizeof(struct ovs_net), 2789 }; 2790 2791 static const char * const ovs_drop_reasons[] = { 2792 #define S(x) [(x) & ~SKB_DROP_REASON_SUBSYS_MASK] = (#x), 2793 OVS_DROP_REASONS(S) 2794 #undef S 2795 }; 2796 2797 static struct drop_reason_list drop_reason_list_ovs = { 2798 .reasons = ovs_drop_reasons, 2799 .n_reasons = ARRAY_SIZE(ovs_drop_reasons), 2800 }; 2801 2802 static int __init ovs_alloc_percpu_storage(void) 2803 { 2804 unsigned int cpu; 2805 2806 ovs_pcpu_storage = alloc_percpu(*ovs_pcpu_storage); 2807 if (!ovs_pcpu_storage) 2808 return -ENOMEM; 2809 2810 for_each_possible_cpu(cpu) { 2811 struct ovs_pcpu_storage *ovs_pcpu; 2812 2813 ovs_pcpu = per_cpu_ptr(ovs_pcpu_storage, cpu); 2814 local_lock_init(&ovs_pcpu->bh_lock); 2815 } 2816 return 0; 2817 } 2818 2819 static void ovs_free_percpu_storage(void) 2820 { 2821 free_percpu(ovs_pcpu_storage); 2822 } 2823 2824 static int __init dp_init(void) 2825 { 2826 int err; 2827 2828 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > 2829 sizeof_field(struct sk_buff, cb)); 2830 2831 pr_info("Open vSwitch switching datapath\n"); 2832 2833 err = ovs_alloc_percpu_storage(); 2834 if (err) 2835 goto error; 2836 2837 err = ovs_internal_dev_rtnl_link_register(); 2838 if (err) 2839 goto error; 2840 2841 err = ovs_flow_init(); 2842 if (err) 2843 goto error_unreg_rtnl_link; 2844 2845 err = ovs_vport_init(); 2846 if (err) 2847 goto error_flow_exit; 2848 2849 err = register_pernet_device(&ovs_net_ops); 2850 if (err) 2851 goto error_vport_exit; 2852 2853 err = register_netdevice_notifier(&ovs_dp_device_notifier); 2854 if (err) 2855 goto error_netns_exit; 2856 2857 err = ovs_netdev_init(); 2858 if (err) 2859 goto error_unreg_notifier; 2860 2861 err = dp_register_genl(); 2862 if (err < 0) 2863 goto error_unreg_netdev; 2864 2865 drop_reasons_register_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH, 2866 &drop_reason_list_ovs); 2867 2868 return 0; 2869 2870 error_unreg_netdev: 2871 ovs_netdev_exit(); 2872 error_unreg_notifier: 2873 unregister_netdevice_notifier(&ovs_dp_device_notifier); 2874 error_netns_exit: 2875 unregister_pernet_device(&ovs_net_ops); 2876 error_vport_exit: 2877 ovs_vport_exit(); 2878 error_flow_exit: 2879 ovs_flow_exit(); 2880 error_unreg_rtnl_link: 2881 ovs_internal_dev_rtnl_link_unregister(); 2882 error: 2883 ovs_free_percpu_storage(); 2884 return err; 2885 } 2886 2887 static void dp_cleanup(void) 2888 { 2889 dp_unregister_genl(ARRAY_SIZE(dp_genl_families)); 2890 ovs_netdev_exit(); 2891 unregister_netdevice_notifier(&ovs_dp_device_notifier); 2892 unregister_pernet_device(&ovs_net_ops); 2893 drop_reasons_unregister_subsys(SKB_DROP_REASON_SUBSYS_OPENVSWITCH); 2894 rcu_barrier(); 2895 ovs_vport_exit(); 2896 ovs_flow_exit(); 2897 ovs_internal_dev_rtnl_link_unregister(); 2898 ovs_free_percpu_storage(); 2899 } 2900 2901 module_init(dp_init); 2902 module_exit(dp_cleanup); 2903 2904 MODULE_DESCRIPTION("Open vSwitch switching datapath"); 2905 MODULE_LICENSE("GPL"); 2906 MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY); 2907 MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY); 2908 MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY); 2909 MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY); 2910 MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY); 2911 MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY); 2912