12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 277b9900eSJiri Pirko /* 377b9900eSJiri Pirko * net/sched/cls_flower.c Flower classifier 477b9900eSJiri Pirko * 577b9900eSJiri Pirko * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 677b9900eSJiri Pirko */ 777b9900eSJiri Pirko 877b9900eSJiri Pirko #include <linux/kernel.h> 977b9900eSJiri Pirko #include <linux/init.h> 1077b9900eSJiri Pirko #include <linux/module.h> 1177b9900eSJiri Pirko #include <linux/rhashtable.h> 12d9363774SDaniel Borkmann #include <linux/workqueue.h> 1306177558SVlad Buslov #include <linux/refcount.h> 1477b9900eSJiri Pirko 1577b9900eSJiri Pirko #include <linux/if_ether.h> 1677b9900eSJiri Pirko #include <linux/in6.h> 1777b9900eSJiri Pirko #include <linux/ip.h> 18a577d8f7SBenjamin LaHaise #include <linux/mpls.h> 1977b9900eSJiri Pirko 2077b9900eSJiri Pirko #include <net/sch_generic.h> 2177b9900eSJiri Pirko #include <net/pkt_cls.h> 2277b9900eSJiri Pirko #include <net/ip.h> 2377b9900eSJiri Pirko #include <net/flow_dissector.h> 240a6e7778SPieter Jansen van Vuuren #include <net/geneve.h> 25d8f9dfaeSXin Long #include <net/vxlan.h> 2679b1011cSXin Long #include <net/erspan.h> 2777b9900eSJiri Pirko 28bc3103f1SAmir Vadai #include <net/dst.h> 29bc3103f1SAmir Vadai #include <net/dst_metadata.h> 30bc3103f1SAmir Vadai 31e0ace68aSPaul Blakey #include <uapi/linux/netfilter/nf_conntrack_common.h> 32e0ace68aSPaul Blakey 3377b9900eSJiri Pirko struct fl_flow_key { 348212ed77SJiri Pirko struct flow_dissector_key_meta meta; 3542aecaa9STom Herbert struct flow_dissector_key_control control; 36bc3103f1SAmir Vadai struct flow_dissector_key_control enc_control; 3777b9900eSJiri Pirko struct flow_dissector_key_basic basic; 3877b9900eSJiri Pirko struct flow_dissector_key_eth_addrs eth; 399399ae9aSHadar Hen Zion struct flow_dissector_key_vlan vlan; 40d64efd09SJianbo Liu struct flow_dissector_key_vlan cvlan; 4177b9900eSJiri Pirko union { 42c3f83241STom Herbert struct flow_dissector_key_ipv4_addrs ipv4; 4377b9900eSJiri Pirko struct flow_dissector_key_ipv6_addrs ipv6; 4477b9900eSJiri Pirko }; 4577b9900eSJiri Pirko struct flow_dissector_key_ports tp; 467b684884SSimon Horman struct flow_dissector_key_icmp icmp; 4799d31326SSimon Horman struct flow_dissector_key_arp arp; 48bc3103f1SAmir Vadai struct flow_dissector_key_keyid enc_key_id; 49bc3103f1SAmir Vadai union { 50bc3103f1SAmir Vadai struct flow_dissector_key_ipv4_addrs enc_ipv4; 51bc3103f1SAmir Vadai struct flow_dissector_key_ipv6_addrs enc_ipv6; 52bc3103f1SAmir Vadai }; 53f4d997fdSHadar Hen Zion struct flow_dissector_key_ports enc_tp; 54a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls mpls; 55fdfc7dd6SJiri Pirko struct flow_dissector_key_tcp tcp; 564d80cc0aSOr Gerlitz struct flow_dissector_key_ip ip; 570e2c17b6SOr Gerlitz struct flow_dissector_key_ip enc_ip; 580a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts enc_opts; 598ffb055bSYoshiki Komachi union { 608ffb055bSYoshiki Komachi struct flow_dissector_key_ports tp; 618ffb055bSYoshiki Komachi struct { 625c72299fSAmritha Nambiar struct flow_dissector_key_ports tp_min; 635c72299fSAmritha Nambiar struct flow_dissector_key_ports tp_max; 648ffb055bSYoshiki Komachi }; 658ffb055bSYoshiki Komachi } tp_range; 66e0ace68aSPaul Blakey struct flow_dissector_key_ct ct; 6777b9900eSJiri Pirko } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 6877b9900eSJiri Pirko 6977b9900eSJiri Pirko struct fl_flow_mask_range { 7077b9900eSJiri Pirko unsigned short int start; 7177b9900eSJiri Pirko unsigned short int end; 7277b9900eSJiri Pirko }; 7377b9900eSJiri Pirko 7477b9900eSJiri Pirko struct fl_flow_mask { 7577b9900eSJiri Pirko struct fl_flow_key key; 7677b9900eSJiri Pirko struct fl_flow_mask_range range; 775c72299fSAmritha Nambiar u32 flags; 7805cd271fSPaul Blakey struct rhash_head ht_node; 7905cd271fSPaul Blakey struct rhashtable ht; 8005cd271fSPaul Blakey struct rhashtable_params filter_ht_params; 8105cd271fSPaul Blakey struct flow_dissector dissector; 8205cd271fSPaul Blakey struct list_head filters; 8344a5cd43SPaolo Abeni struct rcu_work rwork; 8405cd271fSPaul Blakey struct list_head list; 85f48ef4d5SVlad Buslov refcount_t refcnt; 8677b9900eSJiri Pirko }; 8777b9900eSJiri Pirko 88b95ec7ebSJiri Pirko struct fl_flow_tmplt { 89b95ec7ebSJiri Pirko struct fl_flow_key dummy_key; 90b95ec7ebSJiri Pirko struct fl_flow_key mask; 91b95ec7ebSJiri Pirko struct flow_dissector dissector; 92b95ec7ebSJiri Pirko struct tcf_chain *chain; 93b95ec7ebSJiri Pirko }; 94b95ec7ebSJiri Pirko 9577b9900eSJiri Pirko struct cls_fl_head { 9677b9900eSJiri Pirko struct rhashtable ht; 97259e60f9SVlad Buslov spinlock_t masks_lock; /* Protect masks list */ 9805cd271fSPaul Blakey struct list_head masks; 99c049d56eSVlad Buslov struct list_head hw_filters; 100aaa908ffSCong Wang struct rcu_work rwork; 101c15ab236SChris Mi struct idr handle_idr; 102d9363774SDaniel Borkmann }; 10377b9900eSJiri Pirko 10477b9900eSJiri Pirko struct cls_fl_filter { 10505cd271fSPaul Blakey struct fl_flow_mask *mask; 10677b9900eSJiri Pirko struct rhash_head ht_node; 10777b9900eSJiri Pirko struct fl_flow_key mkey; 10877b9900eSJiri Pirko struct tcf_exts exts; 10977b9900eSJiri Pirko struct tcf_result res; 11077b9900eSJiri Pirko struct fl_flow_key key; 11177b9900eSJiri Pirko struct list_head list; 112c049d56eSVlad Buslov struct list_head hw_list; 11377b9900eSJiri Pirko u32 handle; 114e69985c6SAmir Vadai u32 flags; 11586c55361SVlad Buslov u32 in_hw_count; 116aaa908ffSCong Wang struct rcu_work rwork; 1177091d8c7SHadar Hen Zion struct net_device *hw_dev; 11806177558SVlad Buslov /* Flower classifier is unlocked, which means that its reference counter 11906177558SVlad Buslov * can be changed concurrently without any kind of external 12006177558SVlad Buslov * synchronization. Use atomic reference counter to be concurrency-safe. 12106177558SVlad Buslov */ 12206177558SVlad Buslov refcount_t refcnt; 123b2552b8cSVlad Buslov bool deleted; 12477b9900eSJiri Pirko }; 12577b9900eSJiri Pirko 12605cd271fSPaul Blakey static const struct rhashtable_params mask_ht_params = { 12705cd271fSPaul Blakey .key_offset = offsetof(struct fl_flow_mask, key), 12805cd271fSPaul Blakey .key_len = sizeof(struct fl_flow_key), 12905cd271fSPaul Blakey .head_offset = offsetof(struct fl_flow_mask, ht_node), 13005cd271fSPaul Blakey .automatic_shrinking = true, 13105cd271fSPaul Blakey }; 13205cd271fSPaul Blakey 13377b9900eSJiri Pirko static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 13477b9900eSJiri Pirko { 13577b9900eSJiri Pirko return mask->range.end - mask->range.start; 13677b9900eSJiri Pirko } 13777b9900eSJiri Pirko 13877b9900eSJiri Pirko static void fl_mask_update_range(struct fl_flow_mask *mask) 13977b9900eSJiri Pirko { 14077b9900eSJiri Pirko const u8 *bytes = (const u8 *) &mask->key; 14177b9900eSJiri Pirko size_t size = sizeof(mask->key); 14205cd271fSPaul Blakey size_t i, first = 0, last; 14377b9900eSJiri Pirko 14405cd271fSPaul Blakey for (i = 0; i < size; i++) { 14577b9900eSJiri Pirko if (bytes[i]) { 14677b9900eSJiri Pirko first = i; 14705cd271fSPaul Blakey break; 14805cd271fSPaul Blakey } 14905cd271fSPaul Blakey } 15005cd271fSPaul Blakey last = first; 15105cd271fSPaul Blakey for (i = size - 1; i != first; i--) { 15205cd271fSPaul Blakey if (bytes[i]) { 15377b9900eSJiri Pirko last = i; 15405cd271fSPaul Blakey break; 15577b9900eSJiri Pirko } 15677b9900eSJiri Pirko } 15777b9900eSJiri Pirko mask->range.start = rounddown(first, sizeof(long)); 15877b9900eSJiri Pirko mask->range.end = roundup(last + 1, sizeof(long)); 15977b9900eSJiri Pirko } 16077b9900eSJiri Pirko 16177b9900eSJiri Pirko static void *fl_key_get_start(struct fl_flow_key *key, 16277b9900eSJiri Pirko const struct fl_flow_mask *mask) 16377b9900eSJiri Pirko { 16477b9900eSJiri Pirko return (u8 *) key + mask->range.start; 16577b9900eSJiri Pirko } 16677b9900eSJiri Pirko 16777b9900eSJiri Pirko static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 16877b9900eSJiri Pirko struct fl_flow_mask *mask) 16977b9900eSJiri Pirko { 17077b9900eSJiri Pirko const long *lkey = fl_key_get_start(key, mask); 17177b9900eSJiri Pirko const long *lmask = fl_key_get_start(&mask->key, mask); 17277b9900eSJiri Pirko long *lmkey = fl_key_get_start(mkey, mask); 17377b9900eSJiri Pirko int i; 17477b9900eSJiri Pirko 17577b9900eSJiri Pirko for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 17677b9900eSJiri Pirko *lmkey++ = *lkey++ & *lmask++; 17777b9900eSJiri Pirko } 17877b9900eSJiri Pirko 179b95ec7ebSJiri Pirko static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt, 180b95ec7ebSJiri Pirko struct fl_flow_mask *mask) 181b95ec7ebSJiri Pirko { 182b95ec7ebSJiri Pirko const long *lmask = fl_key_get_start(&mask->key, mask); 183b95ec7ebSJiri Pirko const long *ltmplt; 184b95ec7ebSJiri Pirko int i; 185b95ec7ebSJiri Pirko 186b95ec7ebSJiri Pirko if (!tmplt) 187b95ec7ebSJiri Pirko return true; 188b95ec7ebSJiri Pirko ltmplt = fl_key_get_start(&tmplt->mask, mask); 189b95ec7ebSJiri Pirko for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) { 190b95ec7ebSJiri Pirko if (~*ltmplt++ & *lmask++) 191b95ec7ebSJiri Pirko return false; 192b95ec7ebSJiri Pirko } 193b95ec7ebSJiri Pirko return true; 194b95ec7ebSJiri Pirko } 195b95ec7ebSJiri Pirko 19677b9900eSJiri Pirko static void fl_clear_masked_range(struct fl_flow_key *key, 19777b9900eSJiri Pirko struct fl_flow_mask *mask) 19877b9900eSJiri Pirko { 19977b9900eSJiri Pirko memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 20077b9900eSJiri Pirko } 20177b9900eSJiri Pirko 2025c72299fSAmritha Nambiar static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter, 2035c72299fSAmritha Nambiar struct fl_flow_key *key, 2045c72299fSAmritha Nambiar struct fl_flow_key *mkey) 2055c72299fSAmritha Nambiar { 2065c72299fSAmritha Nambiar __be16 min_mask, max_mask, min_val, max_val; 2075c72299fSAmritha Nambiar 2088ffb055bSYoshiki Komachi min_mask = htons(filter->mask->key.tp_range.tp_min.dst); 2098ffb055bSYoshiki Komachi max_mask = htons(filter->mask->key.tp_range.tp_max.dst); 2108ffb055bSYoshiki Komachi min_val = htons(filter->key.tp_range.tp_min.dst); 2118ffb055bSYoshiki Komachi max_val = htons(filter->key.tp_range.tp_max.dst); 2125c72299fSAmritha Nambiar 2135c72299fSAmritha Nambiar if (min_mask && max_mask) { 2148ffb055bSYoshiki Komachi if (htons(key->tp_range.tp.dst) < min_val || 2158ffb055bSYoshiki Komachi htons(key->tp_range.tp.dst) > max_val) 2165c72299fSAmritha Nambiar return false; 2175c72299fSAmritha Nambiar 2185c72299fSAmritha Nambiar /* skb does not have min and max values */ 2198ffb055bSYoshiki Komachi mkey->tp_range.tp_min.dst = filter->mkey.tp_range.tp_min.dst; 2208ffb055bSYoshiki Komachi mkey->tp_range.tp_max.dst = filter->mkey.tp_range.tp_max.dst; 2215c72299fSAmritha Nambiar } 2225c72299fSAmritha Nambiar return true; 2235c72299fSAmritha Nambiar } 2245c72299fSAmritha Nambiar 2255c72299fSAmritha Nambiar static bool fl_range_port_src_cmp(struct cls_fl_filter *filter, 2265c72299fSAmritha Nambiar struct fl_flow_key *key, 2275c72299fSAmritha Nambiar struct fl_flow_key *mkey) 2285c72299fSAmritha Nambiar { 2295c72299fSAmritha Nambiar __be16 min_mask, max_mask, min_val, max_val; 2305c72299fSAmritha Nambiar 2318ffb055bSYoshiki Komachi min_mask = htons(filter->mask->key.tp_range.tp_min.src); 2328ffb055bSYoshiki Komachi max_mask = htons(filter->mask->key.tp_range.tp_max.src); 2338ffb055bSYoshiki Komachi min_val = htons(filter->key.tp_range.tp_min.src); 2348ffb055bSYoshiki Komachi max_val = htons(filter->key.tp_range.tp_max.src); 2355c72299fSAmritha Nambiar 2365c72299fSAmritha Nambiar if (min_mask && max_mask) { 2378ffb055bSYoshiki Komachi if (htons(key->tp_range.tp.src) < min_val || 2388ffb055bSYoshiki Komachi htons(key->tp_range.tp.src) > max_val) 2395c72299fSAmritha Nambiar return false; 2405c72299fSAmritha Nambiar 2415c72299fSAmritha Nambiar /* skb does not have min and max values */ 2428ffb055bSYoshiki Komachi mkey->tp_range.tp_min.src = filter->mkey.tp_range.tp_min.src; 2438ffb055bSYoshiki Komachi mkey->tp_range.tp_max.src = filter->mkey.tp_range.tp_max.src; 2445c72299fSAmritha Nambiar } 2455c72299fSAmritha Nambiar return true; 2465c72299fSAmritha Nambiar } 2475c72299fSAmritha Nambiar 2485c72299fSAmritha Nambiar static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask, 249a3308d8fSPaul Blakey struct fl_flow_key *mkey) 250a3308d8fSPaul Blakey { 25105cd271fSPaul Blakey return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask), 25205cd271fSPaul Blakey mask->filter_ht_params); 253a3308d8fSPaul Blakey } 254a3308d8fSPaul Blakey 2555c72299fSAmritha Nambiar static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask, 2565c72299fSAmritha Nambiar struct fl_flow_key *mkey, 2575c72299fSAmritha Nambiar struct fl_flow_key *key) 2585c72299fSAmritha Nambiar { 2595c72299fSAmritha Nambiar struct cls_fl_filter *filter, *f; 2605c72299fSAmritha Nambiar 2615c72299fSAmritha Nambiar list_for_each_entry_rcu(filter, &mask->filters, list) { 2625c72299fSAmritha Nambiar if (!fl_range_port_dst_cmp(filter, key, mkey)) 2635c72299fSAmritha Nambiar continue; 2645c72299fSAmritha Nambiar 2655c72299fSAmritha Nambiar if (!fl_range_port_src_cmp(filter, key, mkey)) 2665c72299fSAmritha Nambiar continue; 2675c72299fSAmritha Nambiar 2685c72299fSAmritha Nambiar f = __fl_lookup(mask, mkey); 2695c72299fSAmritha Nambiar if (f) 2705c72299fSAmritha Nambiar return f; 2715c72299fSAmritha Nambiar } 2725c72299fSAmritha Nambiar return NULL; 2735c72299fSAmritha Nambiar } 2745c72299fSAmritha Nambiar 2755c72299fSAmritha Nambiar static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask, 2765c72299fSAmritha Nambiar struct fl_flow_key *mkey, 2775c72299fSAmritha Nambiar struct fl_flow_key *key) 2785c72299fSAmritha Nambiar { 2795c72299fSAmritha Nambiar if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE)) 2805c72299fSAmritha Nambiar return fl_lookup_range(mask, mkey, key); 2815c72299fSAmritha Nambiar 2825c72299fSAmritha Nambiar return __fl_lookup(mask, mkey); 2835c72299fSAmritha Nambiar } 2845c72299fSAmritha Nambiar 285e0ace68aSPaul Blakey static u16 fl_ct_info_to_flower_map[] = { 286e0ace68aSPaul Blakey [IP_CT_ESTABLISHED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 287e0ace68aSPaul Blakey TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED, 288e0ace68aSPaul Blakey [IP_CT_RELATED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 289e0ace68aSPaul Blakey TCA_FLOWER_KEY_CT_FLAGS_RELATED, 290e0ace68aSPaul Blakey [IP_CT_ESTABLISHED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 291e0ace68aSPaul Blakey TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED, 292e0ace68aSPaul Blakey [IP_CT_RELATED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 293e0ace68aSPaul Blakey TCA_FLOWER_KEY_CT_FLAGS_RELATED, 294e0ace68aSPaul Blakey [IP_CT_NEW] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 295e0ace68aSPaul Blakey TCA_FLOWER_KEY_CT_FLAGS_NEW, 296e0ace68aSPaul Blakey }; 297e0ace68aSPaul Blakey 29877b9900eSJiri Pirko static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 29977b9900eSJiri Pirko struct tcf_result *res) 30077b9900eSJiri Pirko { 30177b9900eSJiri Pirko struct cls_fl_head *head = rcu_dereference_bh(tp->root); 30277b9900eSJiri Pirko struct fl_flow_key skb_mkey; 303e0ace68aSPaul Blakey struct fl_flow_key skb_key; 304e0ace68aSPaul Blakey struct fl_flow_mask *mask; 305e0ace68aSPaul Blakey struct cls_fl_filter *f; 30677b9900eSJiri Pirko 30705cd271fSPaul Blakey list_for_each_entry_rcu(mask, &head->masks, list) { 30805cd271fSPaul Blakey fl_clear_masked_range(&skb_key, mask); 309bc3103f1SAmir Vadai 3108212ed77SJiri Pirko skb_flow_dissect_meta(skb, &mask->dissector, &skb_key); 31105cd271fSPaul Blakey /* skb_flow_dissect() does not set n_proto in case an unknown 31205cd271fSPaul Blakey * protocol, so do it rather here. 31377b9900eSJiri Pirko */ 31477b9900eSJiri Pirko skb_key.basic.n_proto = skb->protocol; 31505cd271fSPaul Blakey skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key); 316e0ace68aSPaul Blakey skb_flow_dissect_ct(skb, &mask->dissector, &skb_key, 317e0ace68aSPaul Blakey fl_ct_info_to_flower_map, 318e0ace68aSPaul Blakey ARRAY_SIZE(fl_ct_info_to_flower_map)); 31905cd271fSPaul Blakey skb_flow_dissect(skb, &mask->dissector, &skb_key, 0); 32077b9900eSJiri Pirko 32105cd271fSPaul Blakey fl_set_masked_key(&skb_mkey, &skb_key, mask); 32277b9900eSJiri Pirko 3235c72299fSAmritha Nambiar f = fl_lookup(mask, &skb_mkey, &skb_key); 324e8eb36cdSAmir Vadai if (f && !tc_skip_sw(f->flags)) { 32577b9900eSJiri Pirko *res = f->res; 32677b9900eSJiri Pirko return tcf_exts_exec(skb, &f->exts, res); 32777b9900eSJiri Pirko } 32805cd271fSPaul Blakey } 32977b9900eSJiri Pirko return -1; 33077b9900eSJiri Pirko } 33177b9900eSJiri Pirko 33277b9900eSJiri Pirko static int fl_init(struct tcf_proto *tp) 33377b9900eSJiri Pirko { 33477b9900eSJiri Pirko struct cls_fl_head *head; 33577b9900eSJiri Pirko 33677b9900eSJiri Pirko head = kzalloc(sizeof(*head), GFP_KERNEL); 33777b9900eSJiri Pirko if (!head) 33877b9900eSJiri Pirko return -ENOBUFS; 33977b9900eSJiri Pirko 340259e60f9SVlad Buslov spin_lock_init(&head->masks_lock); 34105cd271fSPaul Blakey INIT_LIST_HEAD_RCU(&head->masks); 342c049d56eSVlad Buslov INIT_LIST_HEAD(&head->hw_filters); 34377b9900eSJiri Pirko rcu_assign_pointer(tp->root, head); 344c15ab236SChris Mi idr_init(&head->handle_idr); 34577b9900eSJiri Pirko 34605cd271fSPaul Blakey return rhashtable_init(&head->ht, &mask_ht_params); 34705cd271fSPaul Blakey } 34805cd271fSPaul Blakey 34999815f50SVlad Buslov static void fl_mask_free(struct fl_flow_mask *mask, bool mask_init_done) 35044a5cd43SPaolo Abeni { 35199815f50SVlad Buslov /* temporary masks don't have their filters list and ht initialized */ 35299815f50SVlad Buslov if (mask_init_done) { 353f48ef4d5SVlad Buslov WARN_ON(!list_empty(&mask->filters)); 35444a5cd43SPaolo Abeni rhashtable_destroy(&mask->ht); 35599815f50SVlad Buslov } 35644a5cd43SPaolo Abeni kfree(mask); 35744a5cd43SPaolo Abeni } 35844a5cd43SPaolo Abeni 35944a5cd43SPaolo Abeni static void fl_mask_free_work(struct work_struct *work) 36044a5cd43SPaolo Abeni { 36144a5cd43SPaolo Abeni struct fl_flow_mask *mask = container_of(to_rcu_work(work), 36244a5cd43SPaolo Abeni struct fl_flow_mask, rwork); 36344a5cd43SPaolo Abeni 36499815f50SVlad Buslov fl_mask_free(mask, true); 36599815f50SVlad Buslov } 36699815f50SVlad Buslov 36799815f50SVlad Buslov static void fl_uninit_mask_free_work(struct work_struct *work) 36899815f50SVlad Buslov { 36999815f50SVlad Buslov struct fl_flow_mask *mask = container_of(to_rcu_work(work), 37099815f50SVlad Buslov struct fl_flow_mask, rwork); 37199815f50SVlad Buslov 37299815f50SVlad Buslov fl_mask_free(mask, false); 37344a5cd43SPaolo Abeni } 37444a5cd43SPaolo Abeni 3759994677cSVlad Buslov static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask) 37605cd271fSPaul Blakey { 377f48ef4d5SVlad Buslov if (!refcount_dec_and_test(&mask->refcnt)) 37805cd271fSPaul Blakey return false; 37905cd271fSPaul Blakey 38005cd271fSPaul Blakey rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params); 381259e60f9SVlad Buslov 382259e60f9SVlad Buslov spin_lock(&head->masks_lock); 38305cd271fSPaul Blakey list_del_rcu(&mask->list); 384259e60f9SVlad Buslov spin_unlock(&head->masks_lock); 385259e60f9SVlad Buslov 38644a5cd43SPaolo Abeni tcf_queue_work(&mask->rwork, fl_mask_free_work); 38705cd271fSPaul Blakey 38805cd271fSPaul Blakey return true; 38977b9900eSJiri Pirko } 39077b9900eSJiri Pirko 391c049d56eSVlad Buslov static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp) 392c049d56eSVlad Buslov { 393c049d56eSVlad Buslov /* Flower classifier only changes root pointer during init and destroy. 394c049d56eSVlad Buslov * Users must obtain reference to tcf_proto instance before calling its 395c049d56eSVlad Buslov * API, so tp->root pointer is protected from concurrent call to 396c049d56eSVlad Buslov * fl_destroy() by reference counting. 397c049d56eSVlad Buslov */ 398c049d56eSVlad Buslov return rcu_dereference_raw(tp->root); 399c049d56eSVlad Buslov } 400c049d56eSVlad Buslov 4010dadc117SCong Wang static void __fl_destroy_filter(struct cls_fl_filter *f) 4020dadc117SCong Wang { 4030dadc117SCong Wang tcf_exts_destroy(&f->exts); 4040dadc117SCong Wang tcf_exts_put_net(&f->exts); 4050dadc117SCong Wang kfree(f); 4060dadc117SCong Wang } 4070dadc117SCong Wang 4080552c8afSCong Wang static void fl_destroy_filter_work(struct work_struct *work) 4090552c8afSCong Wang { 410aaa908ffSCong Wang struct cls_fl_filter *f = container_of(to_rcu_work(work), 411aaa908ffSCong Wang struct cls_fl_filter, rwork); 4120552c8afSCong Wang 4130dadc117SCong Wang __fl_destroy_filter(f); 4140552c8afSCong Wang } 4150552c8afSCong Wang 4161b0f8037SJakub Kicinski static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f, 417c24e43d8SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack) 4185b33f488SAmir Vadai { 419208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 420f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 4215b33f488SAmir Vadai 422d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 423f9e30088SPablo Neira Ayuso cls_flower.command = FLOW_CLS_DESTROY; 424de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 4255b33f488SAmir Vadai 42640119211SVlad Buslov tc_setup_cb_destroy(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, false, 427918190f5SVlad Buslov &f->flags, &f->in_hw_count, rtnl_held); 428c24e43d8SVlad Buslov 4295b33f488SAmir Vadai } 4305b33f488SAmir Vadai 431e8eb36cdSAmir Vadai static int fl_hw_replace_filter(struct tcf_proto *tp, 432c24e43d8SVlad Buslov struct cls_fl_filter *f, bool rtnl_held, 43341002038SQuentin Monnet struct netlink_ext_ack *extack) 4345b33f488SAmir Vadai { 435208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 436f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 437717503b9SJiri Pirko bool skip_sw = tc_skip_sw(f->flags); 438c24e43d8SVlad Buslov int err = 0; 439c24e43d8SVlad Buslov 440e3ab786bSPablo Neira Ayuso cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 441918190f5SVlad Buslov if (!cls_flower.rule) 442918190f5SVlad Buslov return -ENOMEM; 4438f256622SPablo Neira Ayuso 444d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 445f9e30088SPablo Neira Ayuso cls_flower.command = FLOW_CLS_REPLACE; 446de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 4478f256622SPablo Neira Ayuso cls_flower.rule->match.dissector = &f->mask->dissector; 4488f256622SPablo Neira Ayuso cls_flower.rule->match.mask = &f->mask->key; 4498f256622SPablo Neira Ayuso cls_flower.rule->match.key = &f->mkey; 450384c181eSAmritha Nambiar cls_flower.classid = f->res.classid; 4515b33f488SAmir Vadai 4529838b20aSVlad Buslov err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts, 453918190f5SVlad Buslov rtnl_held); 4543a7b6861SPablo Neira Ayuso if (err) { 4553a7b6861SPablo Neira Ayuso kfree(cls_flower.rule); 456918190f5SVlad Buslov if (skip_sw) { 4571f15bb4fSVlad Buslov NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 458918190f5SVlad Buslov return err; 459918190f5SVlad Buslov } 460918190f5SVlad Buslov return 0; 4611f15bb4fSVlad Buslov } 4623a7b6861SPablo Neira Ayuso 46340119211SVlad Buslov err = tc_setup_cb_add(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, 464918190f5SVlad Buslov skip_sw, &f->flags, &f->in_hw_count, rtnl_held); 4655a6ff4b1SVlad Buslov tc_cleanup_flow_action(&cls_flower.rule->action); 4668f256622SPablo Neira Ayuso kfree(cls_flower.rule); 4678f256622SPablo Neira Ayuso 46840119211SVlad Buslov if (err) { 469918190f5SVlad Buslov fl_hw_destroy_filter(tp, f, rtnl_held, NULL); 470c24e43d8SVlad Buslov return err; 471c24e43d8SVlad Buslov } 472c24e43d8SVlad Buslov 473918190f5SVlad Buslov if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW)) 474918190f5SVlad Buslov return -EINVAL; 475918190f5SVlad Buslov 476918190f5SVlad Buslov return 0; 477918190f5SVlad Buslov } 478918190f5SVlad Buslov 479c24e43d8SVlad Buslov static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f, 480c24e43d8SVlad Buslov bool rtnl_held) 48110cbc684SAmir Vadai { 482208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 483f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 48410cbc684SAmir Vadai 485d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL); 486f9e30088SPablo Neira Ayuso cls_flower.command = FLOW_CLS_STATS; 487de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 488384c181eSAmritha Nambiar cls_flower.classid = f->res.classid; 48910cbc684SAmir Vadai 490918190f5SVlad Buslov tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, 491918190f5SVlad Buslov rtnl_held); 4923b1903efSPablo Neira Ayuso 4933b1903efSPablo Neira Ayuso tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes, 4943b1903efSPablo Neira Ayuso cls_flower.stats.pkts, 4953b1903efSPablo Neira Ayuso cls_flower.stats.lastused); 49610cbc684SAmir Vadai } 49710cbc684SAmir Vadai 49806177558SVlad Buslov static void __fl_put(struct cls_fl_filter *f) 49906177558SVlad Buslov { 50006177558SVlad Buslov if (!refcount_dec_and_test(&f->refcnt)) 50106177558SVlad Buslov return; 50206177558SVlad Buslov 50306177558SVlad Buslov if (tcf_exts_get_net(&f->exts)) 50406177558SVlad Buslov tcf_queue_work(&f->rwork, fl_destroy_filter_work); 50506177558SVlad Buslov else 50606177558SVlad Buslov __fl_destroy_filter(f); 50706177558SVlad Buslov } 50806177558SVlad Buslov 50906177558SVlad Buslov static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle) 51006177558SVlad Buslov { 51106177558SVlad Buslov struct cls_fl_filter *f; 51206177558SVlad Buslov 51306177558SVlad Buslov rcu_read_lock(); 51406177558SVlad Buslov f = idr_find(&head->handle_idr, handle); 51506177558SVlad Buslov if (f && !refcount_inc_not_zero(&f->refcnt)) 51606177558SVlad Buslov f = NULL; 51706177558SVlad Buslov rcu_read_unlock(); 51806177558SVlad Buslov 51906177558SVlad Buslov return f; 52006177558SVlad Buslov } 52106177558SVlad Buslov 522b2552b8cSVlad Buslov static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f, 523c24e43d8SVlad Buslov bool *last, bool rtnl_held, 524c24e43d8SVlad Buslov struct netlink_ext_ack *extack) 52513fa876eSRoi Dayan { 526e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 527c15ab236SChris Mi 528b2552b8cSVlad Buslov *last = false; 529b2552b8cSVlad Buslov 5303d81e711SVlad Buslov spin_lock(&tp->lock); 5313d81e711SVlad Buslov if (f->deleted) { 5323d81e711SVlad Buslov spin_unlock(&tp->lock); 533b2552b8cSVlad Buslov return -ENOENT; 5343d81e711SVlad Buslov } 535b2552b8cSVlad Buslov 536b2552b8cSVlad Buslov f->deleted = true; 537b2552b8cSVlad Buslov rhashtable_remove_fast(&f->mask->ht, &f->ht_node, 538b2552b8cSVlad Buslov f->mask->filter_ht_params); 5399c160941SMatthew Wilcox idr_remove(&head->handle_idr, f->handle); 54013fa876eSRoi Dayan list_del_rcu(&f->list); 5413d81e711SVlad Buslov spin_unlock(&tp->lock); 5423d81e711SVlad Buslov 5439994677cSVlad Buslov *last = fl_mask_put(head, f->mask); 54479685219SHadar Hen Zion if (!tc_skip_hw(f->flags)) 545c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, f, rtnl_held, extack); 54613fa876eSRoi Dayan tcf_unbind_filter(tp, &f->res); 54706177558SVlad Buslov __fl_put(f); 54805cd271fSPaul Blakey 549b2552b8cSVlad Buslov return 0; 55013fa876eSRoi Dayan } 55113fa876eSRoi Dayan 552d9363774SDaniel Borkmann static void fl_destroy_sleepable(struct work_struct *work) 553d9363774SDaniel Borkmann { 554aaa908ffSCong Wang struct cls_fl_head *head = container_of(to_rcu_work(work), 555aaa908ffSCong Wang struct cls_fl_head, 556aaa908ffSCong Wang rwork); 557de9dc650SPaul Blakey 558de9dc650SPaul Blakey rhashtable_destroy(&head->ht); 559d9363774SDaniel Borkmann kfree(head); 560d9363774SDaniel Borkmann module_put(THIS_MODULE); 561d9363774SDaniel Borkmann } 562d9363774SDaniel Borkmann 56312db03b6SVlad Buslov static void fl_destroy(struct tcf_proto *tp, bool rtnl_held, 56412db03b6SVlad Buslov struct netlink_ext_ack *extack) 56577b9900eSJiri Pirko { 566e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 56705cd271fSPaul Blakey struct fl_flow_mask *mask, *next_mask; 56877b9900eSJiri Pirko struct cls_fl_filter *f, *next; 569b2552b8cSVlad Buslov bool last; 57077b9900eSJiri Pirko 57105cd271fSPaul Blakey list_for_each_entry_safe(mask, next_mask, &head->masks, list) { 57205cd271fSPaul Blakey list_for_each_entry_safe(f, next, &mask->filters, list) { 573c24e43d8SVlad Buslov __fl_delete(tp, f, &last, rtnl_held, extack); 574b2552b8cSVlad Buslov if (last) 57505cd271fSPaul Blakey break; 57605cd271fSPaul Blakey } 57705cd271fSPaul Blakey } 578c15ab236SChris Mi idr_destroy(&head->handle_idr); 579d9363774SDaniel Borkmann 580d9363774SDaniel Borkmann __module_get(THIS_MODULE); 581aaa908ffSCong Wang tcf_queue_work(&head->rwork, fl_destroy_sleepable); 58277b9900eSJiri Pirko } 58377b9900eSJiri Pirko 58406177558SVlad Buslov static void fl_put(struct tcf_proto *tp, void *arg) 58506177558SVlad Buslov { 58606177558SVlad Buslov struct cls_fl_filter *f = arg; 58706177558SVlad Buslov 58806177558SVlad Buslov __fl_put(f); 58906177558SVlad Buslov } 59006177558SVlad Buslov 5918113c095SWANG Cong static void *fl_get(struct tcf_proto *tp, u32 handle) 59277b9900eSJiri Pirko { 593e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 59477b9900eSJiri Pirko 59506177558SVlad Buslov return __fl_get(head, handle); 59677b9900eSJiri Pirko } 59777b9900eSJiri Pirko 59877b9900eSJiri Pirko static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 59977b9900eSJiri Pirko [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 60077b9900eSJiri Pirko [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 60177b9900eSJiri Pirko [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 60277b9900eSJiri Pirko .len = IFNAMSIZ }, 60377b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 60477b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 60577b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 60677b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 60777b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 60877b9900eSJiri Pirko [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 60977b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 61077b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 61177b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 61277b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 61377b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 61477b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 61577b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 61677b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 61777b9900eSJiri Pirko [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 61877b9900eSJiri Pirko [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 619b175c3a4SJamal Hadi Salim [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 620b175c3a4SJamal Hadi Salim [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 6219399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 6229399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 6239399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 624bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 625bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 626bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 627bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 628bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 629bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 630bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 631bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 632bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 633aa72d708SOr Gerlitz [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, 634aa72d708SOr Gerlitz [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, 635aa72d708SOr Gerlitz [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, 636aa72d708SOr Gerlitz [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, 6375976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, 6385976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, 6395976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, 6405976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, 641f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, 642f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, 643f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, 644f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, 645faa3ffceSOr Gerlitz [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 }, 646faa3ffceSOr Gerlitz [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 }, 6477b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 }, 6487b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 }, 6497b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 }, 6507b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 }, 6517b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 }, 6527b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 }, 6537b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 }, 6547b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 }, 65599d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 }, 65699d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 }, 65799d31326SSimon Horman [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 }, 65899d31326SSimon Horman [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 }, 65999d31326SSimon Horman [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 }, 66099d31326SSimon Horman [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 }, 66199d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN }, 66299d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN }, 66399d31326SSimon Horman [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN }, 66499d31326SSimon Horman [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN }, 665a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 }, 666a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 }, 667a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 }, 668a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 }, 669fdfc7dd6SJiri Pirko [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 }, 670fdfc7dd6SJiri Pirko [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 }, 6714d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 }, 6724d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 }, 6734d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 }, 6744d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 }, 675d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 }, 676d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 }, 677d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 }, 6780e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 }, 6790e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 }, 6800e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 }, 6810e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 }, 6820a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED }, 6830a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED }, 684e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_STATE] = { .type = NLA_U16 }, 685e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_STATE_MASK] = { .type = NLA_U16 }, 686e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_ZONE] = { .type = NLA_U16 }, 687e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_ZONE_MASK] = { .type = NLA_U16 }, 688e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_MARK] = { .type = NLA_U32 }, 689e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_MARK_MASK] = { .type = NLA_U32 }, 690e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_LABELS] = { .type = NLA_BINARY, 691e0ace68aSPaul Blakey .len = 128 / BITS_PER_BYTE }, 692e0ace68aSPaul Blakey [TCA_FLOWER_KEY_CT_LABELS_MASK] = { .type = NLA_BINARY, 693e0ace68aSPaul Blakey .len = 128 / BITS_PER_BYTE }, 694*e2debf08SDavide Caratti [TCA_FLOWER_FLAGS] = { .type = NLA_U32 }, 6950a6e7778SPieter Jansen van Vuuren }; 6960a6e7778SPieter Jansen van Vuuren 6970a6e7778SPieter Jansen van Vuuren static const struct nla_policy 6980a6e7778SPieter Jansen van Vuuren enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = { 699d8f9dfaeSXin Long [TCA_FLOWER_KEY_ENC_OPTS_UNSPEC] = { 700d8f9dfaeSXin Long .strict_start_type = TCA_FLOWER_KEY_ENC_OPTS_VXLAN }, 7010a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED }, 702d8f9dfaeSXin Long [TCA_FLOWER_KEY_ENC_OPTS_VXLAN] = { .type = NLA_NESTED }, 70379b1011cSXin Long [TCA_FLOWER_KEY_ENC_OPTS_ERSPAN] = { .type = NLA_NESTED }, 7040a6e7778SPieter Jansen van Vuuren }; 7050a6e7778SPieter Jansen van Vuuren 7060a6e7778SPieter Jansen van Vuuren static const struct nla_policy 7070a6e7778SPieter Jansen van Vuuren geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = { 7080a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 }, 7090a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 }, 7100a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY, 7110a6e7778SPieter Jansen van Vuuren .len = 128 }, 71277b9900eSJiri Pirko }; 71377b9900eSJiri Pirko 714d8f9dfaeSXin Long static const struct nla_policy 715d8f9dfaeSXin Long vxlan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1] = { 716d8f9dfaeSXin Long [TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP] = { .type = NLA_U32 }, 717d8f9dfaeSXin Long }; 718d8f9dfaeSXin Long 71979b1011cSXin Long static const struct nla_policy 72079b1011cSXin Long erspan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1] = { 72179b1011cSXin Long [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER] = { .type = NLA_U8 }, 72279b1011cSXin Long [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX] = { .type = NLA_U32 }, 72379b1011cSXin Long [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] = { .type = NLA_U8 }, 72479b1011cSXin Long [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID] = { .type = NLA_U8 }, 72579b1011cSXin Long }; 72679b1011cSXin Long 72777b9900eSJiri Pirko static void fl_set_key_val(struct nlattr **tb, 72877b9900eSJiri Pirko void *val, int val_type, 72977b9900eSJiri Pirko void *mask, int mask_type, int len) 73077b9900eSJiri Pirko { 73177b9900eSJiri Pirko if (!tb[val_type]) 73277b9900eSJiri Pirko return; 733e0ace68aSPaul Blakey nla_memcpy(val, tb[val_type], len); 73477b9900eSJiri Pirko if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 73577b9900eSJiri Pirko memset(mask, 0xff, len); 73677b9900eSJiri Pirko else 737e0ace68aSPaul Blakey nla_memcpy(mask, tb[mask_type], len); 73877b9900eSJiri Pirko } 73977b9900eSJiri Pirko 7405c72299fSAmritha Nambiar static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, 7415c72299fSAmritha Nambiar struct fl_flow_key *mask) 7425c72299fSAmritha Nambiar { 7438ffb055bSYoshiki Komachi fl_set_key_val(tb, &key->tp_range.tp_min.dst, 7448ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_range.tp_min.dst, 7458ffb055bSYoshiki Komachi TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.dst)); 7468ffb055bSYoshiki Komachi fl_set_key_val(tb, &key->tp_range.tp_max.dst, 7478ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_range.tp_max.dst, 7488ffb055bSYoshiki Komachi TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.dst)); 7498ffb055bSYoshiki Komachi fl_set_key_val(tb, &key->tp_range.tp_min.src, 7508ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_range.tp_min.src, 7518ffb055bSYoshiki Komachi TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.src)); 7528ffb055bSYoshiki Komachi fl_set_key_val(tb, &key->tp_range.tp_max.src, 7538ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_range.tp_max.src, 7548ffb055bSYoshiki Komachi TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.src)); 7555c72299fSAmritha Nambiar 7568ffb055bSYoshiki Komachi if ((mask->tp_range.tp_min.dst && mask->tp_range.tp_max.dst && 7578ffb055bSYoshiki Komachi htons(key->tp_range.tp_max.dst) <= 7588ffb055bSYoshiki Komachi htons(key->tp_range.tp_min.dst)) || 7598ffb055bSYoshiki Komachi (mask->tp_range.tp_min.src && mask->tp_range.tp_max.src && 7608ffb055bSYoshiki Komachi htons(key->tp_range.tp_max.src) <= 7618ffb055bSYoshiki Komachi htons(key->tp_range.tp_min.src))) 7625c72299fSAmritha Nambiar return -EINVAL; 7635c72299fSAmritha Nambiar 7645c72299fSAmritha Nambiar return 0; 7655c72299fSAmritha Nambiar } 7665c72299fSAmritha Nambiar 7671a7fca63SBenjamin LaHaise static int fl_set_key_mpls(struct nlattr **tb, 768a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *key_val, 769a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *key_mask) 770a577d8f7SBenjamin LaHaise { 771a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_TTL]) { 772a577d8f7SBenjamin LaHaise key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]); 773a577d8f7SBenjamin LaHaise key_mask->mpls_ttl = MPLS_TTL_MASK; 774a577d8f7SBenjamin LaHaise } 775a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_BOS]) { 7761a7fca63SBenjamin LaHaise u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]); 7771a7fca63SBenjamin LaHaise 7781a7fca63SBenjamin LaHaise if (bos & ~MPLS_BOS_MASK) 7791a7fca63SBenjamin LaHaise return -EINVAL; 7801a7fca63SBenjamin LaHaise key_val->mpls_bos = bos; 781a577d8f7SBenjamin LaHaise key_mask->mpls_bos = MPLS_BOS_MASK; 782a577d8f7SBenjamin LaHaise } 783a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_TC]) { 7841a7fca63SBenjamin LaHaise u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]); 7851a7fca63SBenjamin LaHaise 7861a7fca63SBenjamin LaHaise if (tc & ~MPLS_TC_MASK) 7871a7fca63SBenjamin LaHaise return -EINVAL; 7881a7fca63SBenjamin LaHaise key_val->mpls_tc = tc; 789a577d8f7SBenjamin LaHaise key_mask->mpls_tc = MPLS_TC_MASK; 790a577d8f7SBenjamin LaHaise } 791a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) { 7921a7fca63SBenjamin LaHaise u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]); 7931a7fca63SBenjamin LaHaise 7941a7fca63SBenjamin LaHaise if (label & ~MPLS_LABEL_MASK) 7951a7fca63SBenjamin LaHaise return -EINVAL; 7961a7fca63SBenjamin LaHaise key_val->mpls_label = label; 797a577d8f7SBenjamin LaHaise key_mask->mpls_label = MPLS_LABEL_MASK; 798a577d8f7SBenjamin LaHaise } 7991a7fca63SBenjamin LaHaise return 0; 800a577d8f7SBenjamin LaHaise } 801a577d8f7SBenjamin LaHaise 8029399ae9aSHadar Hen Zion static void fl_set_key_vlan(struct nlattr **tb, 803aaab0834SJianbo Liu __be16 ethertype, 804d64efd09SJianbo Liu int vlan_id_key, int vlan_prio_key, 8059399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *key_val, 8069399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *key_mask) 8079399ae9aSHadar Hen Zion { 8089399ae9aSHadar Hen Zion #define VLAN_PRIORITY_MASK 0x7 8099399ae9aSHadar Hen Zion 810d64efd09SJianbo Liu if (tb[vlan_id_key]) { 8119399ae9aSHadar Hen Zion key_val->vlan_id = 812d64efd09SJianbo Liu nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK; 8139399ae9aSHadar Hen Zion key_mask->vlan_id = VLAN_VID_MASK; 8149399ae9aSHadar Hen Zion } 815d64efd09SJianbo Liu if (tb[vlan_prio_key]) { 8169399ae9aSHadar Hen Zion key_val->vlan_priority = 817d64efd09SJianbo Liu nla_get_u8(tb[vlan_prio_key]) & 8189399ae9aSHadar Hen Zion VLAN_PRIORITY_MASK; 8199399ae9aSHadar Hen Zion key_mask->vlan_priority = VLAN_PRIORITY_MASK; 8209399ae9aSHadar Hen Zion } 821aaab0834SJianbo Liu key_val->vlan_tpid = ethertype; 822aaab0834SJianbo Liu key_mask->vlan_tpid = cpu_to_be16(~0); 8239399ae9aSHadar Hen Zion } 8249399ae9aSHadar Hen Zion 825faa3ffceSOr Gerlitz static void fl_set_key_flag(u32 flower_key, u32 flower_mask, 826faa3ffceSOr Gerlitz u32 *dissector_key, u32 *dissector_mask, 827faa3ffceSOr Gerlitz u32 flower_flag_bit, u32 dissector_flag_bit) 828faa3ffceSOr Gerlitz { 829faa3ffceSOr Gerlitz if (flower_mask & flower_flag_bit) { 830faa3ffceSOr Gerlitz *dissector_mask |= dissector_flag_bit; 831faa3ffceSOr Gerlitz if (flower_key & flower_flag_bit) 832faa3ffceSOr Gerlitz *dissector_key |= dissector_flag_bit; 833faa3ffceSOr Gerlitz } 834faa3ffceSOr Gerlitz } 835faa3ffceSOr Gerlitz 836d9724772SOr Gerlitz static int fl_set_key_flags(struct nlattr **tb, 837faa3ffceSOr Gerlitz u32 *flags_key, u32 *flags_mask) 838faa3ffceSOr Gerlitz { 839faa3ffceSOr Gerlitz u32 key, mask; 840faa3ffceSOr Gerlitz 841d9724772SOr Gerlitz /* mask is mandatory for flags */ 842d9724772SOr Gerlitz if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) 843d9724772SOr Gerlitz return -EINVAL; 844faa3ffceSOr Gerlitz 845faa3ffceSOr Gerlitz key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS])); 846faa3ffceSOr Gerlitz mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK])); 847faa3ffceSOr Gerlitz 848faa3ffceSOr Gerlitz *flags_key = 0; 849faa3ffceSOr Gerlitz *flags_mask = 0; 850faa3ffceSOr Gerlitz 851faa3ffceSOr Gerlitz fl_set_key_flag(key, mask, flags_key, flags_mask, 852faa3ffceSOr Gerlitz TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 853459d153dSPieter Jansen van Vuuren fl_set_key_flag(key, mask, flags_key, flags_mask, 854459d153dSPieter Jansen van Vuuren TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 855459d153dSPieter Jansen van Vuuren FLOW_DIS_FIRST_FRAG); 856d9724772SOr Gerlitz 857d9724772SOr Gerlitz return 0; 858faa3ffceSOr Gerlitz } 859faa3ffceSOr Gerlitz 8600e2c17b6SOr Gerlitz static void fl_set_key_ip(struct nlattr **tb, bool encap, 8614d80cc0aSOr Gerlitz struct flow_dissector_key_ip *key, 8624d80cc0aSOr Gerlitz struct flow_dissector_key_ip *mask) 8634d80cc0aSOr Gerlitz { 8640e2c17b6SOr Gerlitz int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 8650e2c17b6SOr Gerlitz int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 8660e2c17b6SOr Gerlitz int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 8670e2c17b6SOr Gerlitz int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 8684d80cc0aSOr Gerlitz 8690e2c17b6SOr Gerlitz fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)); 8700e2c17b6SOr Gerlitz fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)); 8714d80cc0aSOr Gerlitz } 8724d80cc0aSOr Gerlitz 8730a6e7778SPieter Jansen van Vuuren static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key, 8740a6e7778SPieter Jansen van Vuuren int depth, int option_len, 8750a6e7778SPieter Jansen van Vuuren struct netlink_ext_ack *extack) 8760a6e7778SPieter Jansen van Vuuren { 8770a6e7778SPieter Jansen van Vuuren struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1]; 8780a6e7778SPieter Jansen van Vuuren struct nlattr *class = NULL, *type = NULL, *data = NULL; 8790a6e7778SPieter Jansen van Vuuren struct geneve_opt *opt; 8800a6e7778SPieter Jansen van Vuuren int err, data_len = 0; 8810a6e7778SPieter Jansen van Vuuren 8820a6e7778SPieter Jansen van Vuuren if (option_len > sizeof(struct geneve_opt)) 8830a6e7778SPieter Jansen van Vuuren data_len = option_len - sizeof(struct geneve_opt); 8840a6e7778SPieter Jansen van Vuuren 8850a6e7778SPieter Jansen van Vuuren opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len]; 8860a6e7778SPieter Jansen van Vuuren memset(opt, 0xff, option_len); 8870a6e7778SPieter Jansen van Vuuren opt->length = data_len / 4; 8880a6e7778SPieter Jansen van Vuuren opt->r1 = 0; 8890a6e7778SPieter Jansen van Vuuren opt->r2 = 0; 8900a6e7778SPieter Jansen van Vuuren opt->r3 = 0; 8910a6e7778SPieter Jansen van Vuuren 8920a6e7778SPieter Jansen van Vuuren /* If no mask has been prodived we assume an exact match. */ 8930a6e7778SPieter Jansen van Vuuren if (!depth) 8940a6e7778SPieter Jansen van Vuuren return sizeof(struct geneve_opt) + data_len; 8950a6e7778SPieter Jansen van Vuuren 8960a6e7778SPieter Jansen van Vuuren if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) { 8970a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Non-geneve option type for mask"); 8980a6e7778SPieter Jansen van Vuuren return -EINVAL; 8990a6e7778SPieter Jansen van Vuuren } 9000a6e7778SPieter Jansen van Vuuren 9018cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, 9028cb08174SJohannes Berg TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX, 9030a6e7778SPieter Jansen van Vuuren nla, geneve_opt_policy, extack); 9040a6e7778SPieter Jansen van Vuuren if (err < 0) 9050a6e7778SPieter Jansen van Vuuren return err; 9060a6e7778SPieter Jansen van Vuuren 9070a6e7778SPieter Jansen van Vuuren /* We are not allowed to omit any of CLASS, TYPE or DATA 9080a6e7778SPieter Jansen van Vuuren * fields from the key. 9090a6e7778SPieter Jansen van Vuuren */ 9100a6e7778SPieter Jansen van Vuuren if (!option_len && 9110a6e7778SPieter Jansen van Vuuren (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] || 9120a6e7778SPieter Jansen van Vuuren !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] || 9130a6e7778SPieter Jansen van Vuuren !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) { 9140a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data"); 9150a6e7778SPieter Jansen van Vuuren return -EINVAL; 9160a6e7778SPieter Jansen van Vuuren } 9170a6e7778SPieter Jansen van Vuuren 9180a6e7778SPieter Jansen van Vuuren /* Omitting any of CLASS, TYPE or DATA fields is allowed 9190a6e7778SPieter Jansen van Vuuren * for the mask. 9200a6e7778SPieter Jansen van Vuuren */ 9210a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) { 9220a6e7778SPieter Jansen van Vuuren int new_len = key->enc_opts.len; 9230a6e7778SPieter Jansen van Vuuren 9240a6e7778SPieter Jansen van Vuuren data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]; 9250a6e7778SPieter Jansen van Vuuren data_len = nla_len(data); 9260a6e7778SPieter Jansen van Vuuren if (data_len < 4) { 9270a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long"); 9280a6e7778SPieter Jansen van Vuuren return -ERANGE; 9290a6e7778SPieter Jansen van Vuuren } 9300a6e7778SPieter Jansen van Vuuren if (data_len % 4) { 9310a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long"); 9320a6e7778SPieter Jansen van Vuuren return -ERANGE; 9330a6e7778SPieter Jansen van Vuuren } 9340a6e7778SPieter Jansen van Vuuren 9350a6e7778SPieter Jansen van Vuuren new_len += sizeof(struct geneve_opt) + data_len; 9360a6e7778SPieter Jansen van Vuuren BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX); 9370a6e7778SPieter Jansen van Vuuren if (new_len > FLOW_DIS_TUN_OPTS_MAX) { 9380a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size"); 9390a6e7778SPieter Jansen van Vuuren return -ERANGE; 9400a6e7778SPieter Jansen van Vuuren } 9410a6e7778SPieter Jansen van Vuuren opt->length = data_len / 4; 9420a6e7778SPieter Jansen van Vuuren memcpy(opt->opt_data, nla_data(data), data_len); 9430a6e7778SPieter Jansen van Vuuren } 9440a6e7778SPieter Jansen van Vuuren 9450a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) { 9460a6e7778SPieter Jansen van Vuuren class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]; 9470a6e7778SPieter Jansen van Vuuren opt->opt_class = nla_get_be16(class); 9480a6e7778SPieter Jansen van Vuuren } 9490a6e7778SPieter Jansen van Vuuren 9500a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) { 9510a6e7778SPieter Jansen van Vuuren type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]; 9520a6e7778SPieter Jansen van Vuuren opt->type = nla_get_u8(type); 9530a6e7778SPieter Jansen van Vuuren } 9540a6e7778SPieter Jansen van Vuuren 9550a6e7778SPieter Jansen van Vuuren return sizeof(struct geneve_opt) + data_len; 9560a6e7778SPieter Jansen van Vuuren } 9570a6e7778SPieter Jansen van Vuuren 958d8f9dfaeSXin Long static int fl_set_vxlan_opt(const struct nlattr *nla, struct fl_flow_key *key, 959d8f9dfaeSXin Long int depth, int option_len, 960d8f9dfaeSXin Long struct netlink_ext_ack *extack) 961d8f9dfaeSXin Long { 962d8f9dfaeSXin Long struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1]; 963d8f9dfaeSXin Long struct vxlan_metadata *md; 964d8f9dfaeSXin Long int err; 965d8f9dfaeSXin Long 966d8f9dfaeSXin Long md = (struct vxlan_metadata *)&key->enc_opts.data[key->enc_opts.len]; 967d8f9dfaeSXin Long memset(md, 0xff, sizeof(*md)); 968d8f9dfaeSXin Long 969d8f9dfaeSXin Long if (!depth) 970d8f9dfaeSXin Long return sizeof(*md); 971d8f9dfaeSXin Long 972d8f9dfaeSXin Long if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_VXLAN) { 973d8f9dfaeSXin Long NL_SET_ERR_MSG(extack, "Non-vxlan option type for mask"); 974d8f9dfaeSXin Long return -EINVAL; 975d8f9dfaeSXin Long } 976d8f9dfaeSXin Long 977d8f9dfaeSXin Long err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX, nla, 978d8f9dfaeSXin Long vxlan_opt_policy, extack); 979d8f9dfaeSXin Long if (err < 0) 980d8f9dfaeSXin Long return err; 981d8f9dfaeSXin Long 982d8f9dfaeSXin Long if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) { 983d8f9dfaeSXin Long NL_SET_ERR_MSG(extack, "Missing tunnel key vxlan option gbp"); 984d8f9dfaeSXin Long return -EINVAL; 985d8f9dfaeSXin Long } 986d8f9dfaeSXin Long 987d8f9dfaeSXin Long if (tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) 988d8f9dfaeSXin Long md->gbp = nla_get_u32(tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]); 989d8f9dfaeSXin Long 990d8f9dfaeSXin Long return sizeof(*md); 991d8f9dfaeSXin Long } 992d8f9dfaeSXin Long 99379b1011cSXin Long static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key, 99479b1011cSXin Long int depth, int option_len, 99579b1011cSXin Long struct netlink_ext_ack *extack) 99679b1011cSXin Long { 99779b1011cSXin Long struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1]; 99879b1011cSXin Long struct erspan_metadata *md; 99979b1011cSXin Long int err; 100079b1011cSXin Long 100179b1011cSXin Long md = (struct erspan_metadata *)&key->enc_opts.data[key->enc_opts.len]; 100279b1011cSXin Long memset(md, 0xff, sizeof(*md)); 100379b1011cSXin Long md->version = 1; 100479b1011cSXin Long 100579b1011cSXin Long if (!depth) 100679b1011cSXin Long return sizeof(*md); 100779b1011cSXin Long 100879b1011cSXin Long if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_ERSPAN) { 100979b1011cSXin Long NL_SET_ERR_MSG(extack, "Non-erspan option type for mask"); 101079b1011cSXin Long return -EINVAL; 101179b1011cSXin Long } 101279b1011cSXin Long 101379b1011cSXin Long err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX, nla, 101479b1011cSXin Long erspan_opt_policy, extack); 101579b1011cSXin Long if (err < 0) 101679b1011cSXin Long return err; 101779b1011cSXin Long 101879b1011cSXin Long if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) { 101979b1011cSXin Long NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option ver"); 102079b1011cSXin Long return -EINVAL; 102179b1011cSXin Long } 102279b1011cSXin Long 102379b1011cSXin Long if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) 102479b1011cSXin Long md->version = nla_get_u8(tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]); 102579b1011cSXin Long 102679b1011cSXin Long if (md->version == 1) { 102779b1011cSXin Long if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) { 102879b1011cSXin Long NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option index"); 102979b1011cSXin Long return -EINVAL; 103079b1011cSXin Long } 103179b1011cSXin Long if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) { 103279b1011cSXin Long nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]; 103379b1011cSXin Long md->u.index = nla_get_be32(nla); 103479b1011cSXin Long } 103579b1011cSXin Long } else if (md->version == 2) { 103679b1011cSXin Long if (!option_len && (!tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] || 103779b1011cSXin Long !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID])) { 103879b1011cSXin Long NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option dir or hwid"); 103979b1011cSXin Long return -EINVAL; 104079b1011cSXin Long } 104179b1011cSXin Long if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]) { 104279b1011cSXin Long nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]; 104379b1011cSXin Long md->u.md2.dir = nla_get_u8(nla); 104479b1011cSXin Long } 104579b1011cSXin Long if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]) { 104679b1011cSXin Long nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]; 104779b1011cSXin Long set_hwid(&md->u.md2, nla_get_u8(nla)); 104879b1011cSXin Long } 104979b1011cSXin Long } else { 105079b1011cSXin Long NL_SET_ERR_MSG(extack, "Tunnel key erspan option ver is incorrect"); 105179b1011cSXin Long return -EINVAL; 105279b1011cSXin Long } 105379b1011cSXin Long 105479b1011cSXin Long return sizeof(*md); 105579b1011cSXin Long } 105679b1011cSXin Long 10570a6e7778SPieter Jansen van Vuuren static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key, 10580a6e7778SPieter Jansen van Vuuren struct fl_flow_key *mask, 10590a6e7778SPieter Jansen van Vuuren struct netlink_ext_ack *extack) 10600a6e7778SPieter Jansen van Vuuren { 10610a6e7778SPieter Jansen van Vuuren const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL; 106263c82997SJakub Kicinski int err, option_len, key_depth, msk_depth = 0; 106363c82997SJakub Kicinski 10648cb08174SJohannes Berg err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS], 106563c82997SJakub Kicinski TCA_FLOWER_KEY_ENC_OPTS_MAX, 106663c82997SJakub Kicinski enc_opts_policy, extack); 106763c82997SJakub Kicinski if (err) 106863c82997SJakub Kicinski return err; 10690a6e7778SPieter Jansen van Vuuren 10700a6e7778SPieter Jansen van Vuuren nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]); 10710a6e7778SPieter Jansen van Vuuren 10720a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) { 10738cb08174SJohannes Berg err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK], 107463c82997SJakub Kicinski TCA_FLOWER_KEY_ENC_OPTS_MAX, 107563c82997SJakub Kicinski enc_opts_policy, extack); 107663c82997SJakub Kicinski if (err) 107763c82997SJakub Kicinski return err; 107863c82997SJakub Kicinski 10790a6e7778SPieter Jansen van Vuuren nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 10800a6e7778SPieter Jansen van Vuuren msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 10810a6e7778SPieter Jansen van Vuuren } 10820a6e7778SPieter Jansen van Vuuren 10830a6e7778SPieter Jansen van Vuuren nla_for_each_attr(nla_opt_key, nla_enc_key, 10840a6e7778SPieter Jansen van Vuuren nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) { 10850a6e7778SPieter Jansen van Vuuren switch (nla_type(nla_opt_key)) { 10860a6e7778SPieter Jansen van Vuuren case TCA_FLOWER_KEY_ENC_OPTS_GENEVE: 1087d8f9dfaeSXin Long if (key->enc_opts.dst_opt_type && 1088d8f9dfaeSXin Long key->enc_opts.dst_opt_type != TUNNEL_GENEVE_OPT) { 1089d8f9dfaeSXin Long NL_SET_ERR_MSG(extack, "Duplicate type for geneve options"); 1090d8f9dfaeSXin Long return -EINVAL; 1091d8f9dfaeSXin Long } 10920a6e7778SPieter Jansen van Vuuren option_len = 0; 10930a6e7778SPieter Jansen van Vuuren key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 10940a6e7778SPieter Jansen van Vuuren option_len = fl_set_geneve_opt(nla_opt_key, key, 10950a6e7778SPieter Jansen van Vuuren key_depth, option_len, 10960a6e7778SPieter Jansen van Vuuren extack); 10970a6e7778SPieter Jansen van Vuuren if (option_len < 0) 10980a6e7778SPieter Jansen van Vuuren return option_len; 10990a6e7778SPieter Jansen van Vuuren 11000a6e7778SPieter Jansen van Vuuren key->enc_opts.len += option_len; 11010a6e7778SPieter Jansen van Vuuren /* At the same time we need to parse through the mask 11020a6e7778SPieter Jansen van Vuuren * in order to verify exact and mask attribute lengths. 11030a6e7778SPieter Jansen van Vuuren */ 11040a6e7778SPieter Jansen van Vuuren mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 11050a6e7778SPieter Jansen van Vuuren option_len = fl_set_geneve_opt(nla_opt_msk, mask, 11060a6e7778SPieter Jansen van Vuuren msk_depth, option_len, 11070a6e7778SPieter Jansen van Vuuren extack); 11080a6e7778SPieter Jansen van Vuuren if (option_len < 0) 11090a6e7778SPieter Jansen van Vuuren return option_len; 11100a6e7778SPieter Jansen van Vuuren 11110a6e7778SPieter Jansen van Vuuren mask->enc_opts.len += option_len; 11120a6e7778SPieter Jansen van Vuuren if (key->enc_opts.len != mask->enc_opts.len) { 11130a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 11140a6e7778SPieter Jansen van Vuuren return -EINVAL; 11150a6e7778SPieter Jansen van Vuuren } 11160a6e7778SPieter Jansen van Vuuren 11170a6e7778SPieter Jansen van Vuuren if (msk_depth) 11180a6e7778SPieter Jansen van Vuuren nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 11190a6e7778SPieter Jansen van Vuuren break; 1120d8f9dfaeSXin Long case TCA_FLOWER_KEY_ENC_OPTS_VXLAN: 1121d8f9dfaeSXin Long if (key->enc_opts.dst_opt_type) { 1122d8f9dfaeSXin Long NL_SET_ERR_MSG(extack, "Duplicate type for vxlan options"); 1123d8f9dfaeSXin Long return -EINVAL; 1124d8f9dfaeSXin Long } 1125d8f9dfaeSXin Long option_len = 0; 1126d8f9dfaeSXin Long key->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT; 1127d8f9dfaeSXin Long option_len = fl_set_vxlan_opt(nla_opt_key, key, 1128d8f9dfaeSXin Long key_depth, option_len, 1129d8f9dfaeSXin Long extack); 1130d8f9dfaeSXin Long if (option_len < 0) 1131d8f9dfaeSXin Long return option_len; 1132d8f9dfaeSXin Long 1133d8f9dfaeSXin Long key->enc_opts.len += option_len; 1134d8f9dfaeSXin Long /* At the same time we need to parse through the mask 1135d8f9dfaeSXin Long * in order to verify exact and mask attribute lengths. 1136d8f9dfaeSXin Long */ 1137d8f9dfaeSXin Long mask->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT; 1138d8f9dfaeSXin Long option_len = fl_set_vxlan_opt(nla_opt_msk, mask, 1139d8f9dfaeSXin Long msk_depth, option_len, 1140d8f9dfaeSXin Long extack); 1141d8f9dfaeSXin Long if (option_len < 0) 1142d8f9dfaeSXin Long return option_len; 1143d8f9dfaeSXin Long 1144d8f9dfaeSXin Long mask->enc_opts.len += option_len; 1145d8f9dfaeSXin Long if (key->enc_opts.len != mask->enc_opts.len) { 1146d8f9dfaeSXin Long NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 1147d8f9dfaeSXin Long return -EINVAL; 1148d8f9dfaeSXin Long } 1149d8f9dfaeSXin Long 1150d8f9dfaeSXin Long if (msk_depth) 1151d8f9dfaeSXin Long nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 1152d8f9dfaeSXin Long break; 115379b1011cSXin Long case TCA_FLOWER_KEY_ENC_OPTS_ERSPAN: 115479b1011cSXin Long if (key->enc_opts.dst_opt_type) { 115579b1011cSXin Long NL_SET_ERR_MSG(extack, "Duplicate type for erspan options"); 115679b1011cSXin Long return -EINVAL; 115779b1011cSXin Long } 115879b1011cSXin Long option_len = 0; 115979b1011cSXin Long key->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT; 116079b1011cSXin Long option_len = fl_set_erspan_opt(nla_opt_key, key, 116179b1011cSXin Long key_depth, option_len, 116279b1011cSXin Long extack); 116379b1011cSXin Long if (option_len < 0) 116479b1011cSXin Long return option_len; 116579b1011cSXin Long 116679b1011cSXin Long key->enc_opts.len += option_len; 116779b1011cSXin Long /* At the same time we need to parse through the mask 116879b1011cSXin Long * in order to verify exact and mask attribute lengths. 116979b1011cSXin Long */ 117079b1011cSXin Long mask->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT; 117179b1011cSXin Long option_len = fl_set_erspan_opt(nla_opt_msk, mask, 117279b1011cSXin Long msk_depth, option_len, 117379b1011cSXin Long extack); 117479b1011cSXin Long if (option_len < 0) 117579b1011cSXin Long return option_len; 117679b1011cSXin Long 117779b1011cSXin Long mask->enc_opts.len += option_len; 117879b1011cSXin Long if (key->enc_opts.len != mask->enc_opts.len) { 117979b1011cSXin Long NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 118079b1011cSXin Long return -EINVAL; 118179b1011cSXin Long } 118279b1011cSXin Long 118379b1011cSXin Long if (msk_depth) 118479b1011cSXin Long nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 118579b1011cSXin Long break; 11860a6e7778SPieter Jansen van Vuuren default: 11870a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Unknown tunnel option type"); 11880a6e7778SPieter Jansen van Vuuren return -EINVAL; 11890a6e7778SPieter Jansen van Vuuren } 11900a6e7778SPieter Jansen van Vuuren } 11910a6e7778SPieter Jansen van Vuuren 11920a6e7778SPieter Jansen van Vuuren return 0; 11930a6e7778SPieter Jansen van Vuuren } 11940a6e7778SPieter Jansen van Vuuren 1195e0ace68aSPaul Blakey static int fl_set_key_ct(struct nlattr **tb, 1196e0ace68aSPaul Blakey struct flow_dissector_key_ct *key, 1197e0ace68aSPaul Blakey struct flow_dissector_key_ct *mask, 1198e0ace68aSPaul Blakey struct netlink_ext_ack *extack) 1199e0ace68aSPaul Blakey { 1200e0ace68aSPaul Blakey if (tb[TCA_FLOWER_KEY_CT_STATE]) { 1201e0ace68aSPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK)) { 1202e0ace68aSPaul Blakey NL_SET_ERR_MSG(extack, "Conntrack isn't enabled"); 1203e0ace68aSPaul Blakey return -EOPNOTSUPP; 1204e0ace68aSPaul Blakey } 1205e0ace68aSPaul Blakey fl_set_key_val(tb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE, 1206e0ace68aSPaul Blakey &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK, 1207e0ace68aSPaul Blakey sizeof(key->ct_state)); 1208e0ace68aSPaul Blakey } 1209e0ace68aSPaul Blakey if (tb[TCA_FLOWER_KEY_CT_ZONE]) { 1210e0ace68aSPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) { 1211e0ace68aSPaul Blakey NL_SET_ERR_MSG(extack, "Conntrack zones isn't enabled"); 1212e0ace68aSPaul Blakey return -EOPNOTSUPP; 1213e0ace68aSPaul Blakey } 1214e0ace68aSPaul Blakey fl_set_key_val(tb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE, 1215e0ace68aSPaul Blakey &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK, 1216e0ace68aSPaul Blakey sizeof(key->ct_zone)); 1217e0ace68aSPaul Blakey } 1218e0ace68aSPaul Blakey if (tb[TCA_FLOWER_KEY_CT_MARK]) { 1219e0ace68aSPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) { 1220e0ace68aSPaul Blakey NL_SET_ERR_MSG(extack, "Conntrack mark isn't enabled"); 1221e0ace68aSPaul Blakey return -EOPNOTSUPP; 1222e0ace68aSPaul Blakey } 1223e0ace68aSPaul Blakey fl_set_key_val(tb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK, 1224e0ace68aSPaul Blakey &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK, 1225e0ace68aSPaul Blakey sizeof(key->ct_mark)); 1226e0ace68aSPaul Blakey } 1227e0ace68aSPaul Blakey if (tb[TCA_FLOWER_KEY_CT_LABELS]) { 1228e0ace68aSPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) { 1229e0ace68aSPaul Blakey NL_SET_ERR_MSG(extack, "Conntrack labels aren't enabled"); 1230e0ace68aSPaul Blakey return -EOPNOTSUPP; 1231e0ace68aSPaul Blakey } 1232e0ace68aSPaul Blakey fl_set_key_val(tb, key->ct_labels, TCA_FLOWER_KEY_CT_LABELS, 1233e0ace68aSPaul Blakey mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK, 1234e0ace68aSPaul Blakey sizeof(key->ct_labels)); 1235e0ace68aSPaul Blakey } 1236e0ace68aSPaul Blakey 1237e0ace68aSPaul Blakey return 0; 1238e0ace68aSPaul Blakey } 1239e0ace68aSPaul Blakey 124077b9900eSJiri Pirko static int fl_set_key(struct net *net, struct nlattr **tb, 12411057c55fSAlexander Aring struct fl_flow_key *key, struct fl_flow_key *mask, 12421057c55fSAlexander Aring struct netlink_ext_ack *extack) 124377b9900eSJiri Pirko { 12449399ae9aSHadar Hen Zion __be16 ethertype; 1245d9724772SOr Gerlitz int ret = 0; 1246a5148626SJiri Pirko 124777b9900eSJiri Pirko if (tb[TCA_FLOWER_INDEV]) { 12481057c55fSAlexander Aring int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack); 124977b9900eSJiri Pirko if (err < 0) 125077b9900eSJiri Pirko return err; 12518212ed77SJiri Pirko key->meta.ingress_ifindex = err; 12528212ed77SJiri Pirko mask->meta.ingress_ifindex = 0xffffffff; 125377b9900eSJiri Pirko } 125477b9900eSJiri Pirko 125577b9900eSJiri Pirko fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 125677b9900eSJiri Pirko mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 125777b9900eSJiri Pirko sizeof(key->eth.dst)); 125877b9900eSJiri Pirko fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 125977b9900eSJiri Pirko mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 126077b9900eSJiri Pirko sizeof(key->eth.src)); 126166530bdfSJamal Hadi Salim 12620b498a52SArnd Bergmann if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 12639399ae9aSHadar Hen Zion ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 12649399ae9aSHadar Hen Zion 1265aaab0834SJianbo Liu if (eth_type_vlan(ethertype)) { 1266d64efd09SJianbo Liu fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID, 1267d64efd09SJianbo Liu TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, 1268d64efd09SJianbo Liu &mask->vlan); 1269d64efd09SJianbo Liu 12705e9a0fe4SJianbo Liu if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) { 1271d64efd09SJianbo Liu ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]); 1272d64efd09SJianbo Liu if (eth_type_vlan(ethertype)) { 1273d64efd09SJianbo Liu fl_set_key_vlan(tb, ethertype, 1274d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_ID, 1275d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_PRIO, 1276d64efd09SJianbo Liu &key->cvlan, &mask->cvlan); 12779399ae9aSHadar Hen Zion fl_set_key_val(tb, &key->basic.n_proto, 1278d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 1279d64efd09SJianbo Liu &mask->basic.n_proto, 1280d64efd09SJianbo Liu TCA_FLOWER_UNSPEC, 128177b9900eSJiri Pirko sizeof(key->basic.n_proto)); 12829399ae9aSHadar Hen Zion } else { 12839399ae9aSHadar Hen Zion key->basic.n_proto = ethertype; 12849399ae9aSHadar Hen Zion mask->basic.n_proto = cpu_to_be16(~0); 12859399ae9aSHadar Hen Zion } 12865e9a0fe4SJianbo Liu } 1287d64efd09SJianbo Liu } else { 1288d64efd09SJianbo Liu key->basic.n_proto = ethertype; 1289d64efd09SJianbo Liu mask->basic.n_proto = cpu_to_be16(~0); 1290d64efd09SJianbo Liu } 12910b498a52SArnd Bergmann } 129266530bdfSJamal Hadi Salim 129377b9900eSJiri Pirko if (key->basic.n_proto == htons(ETH_P_IP) || 129477b9900eSJiri Pirko key->basic.n_proto == htons(ETH_P_IPV6)) { 129577b9900eSJiri Pirko fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 129677b9900eSJiri Pirko &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 129777b9900eSJiri Pirko sizeof(key->basic.ip_proto)); 12980e2c17b6SOr Gerlitz fl_set_key_ip(tb, false, &key->ip, &mask->ip); 129977b9900eSJiri Pirko } 130066530bdfSJamal Hadi Salim 130166530bdfSJamal Hadi Salim if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 130266530bdfSJamal Hadi Salim key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1303970bfcd0SPaul Blakey mask->control.addr_type = ~0; 130477b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 130577b9900eSJiri Pirko &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 130677b9900eSJiri Pirko sizeof(key->ipv4.src)); 130777b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 130877b9900eSJiri Pirko &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 130977b9900eSJiri Pirko sizeof(key->ipv4.dst)); 131066530bdfSJamal Hadi Salim } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 131166530bdfSJamal Hadi Salim key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1312970bfcd0SPaul Blakey mask->control.addr_type = ~0; 131377b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 131477b9900eSJiri Pirko &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 131577b9900eSJiri Pirko sizeof(key->ipv6.src)); 131677b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 131777b9900eSJiri Pirko &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 131877b9900eSJiri Pirko sizeof(key->ipv6.dst)); 131977b9900eSJiri Pirko } 132066530bdfSJamal Hadi Salim 132177b9900eSJiri Pirko if (key->basic.ip_proto == IPPROTO_TCP) { 132277b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 1323aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 132477b9900eSJiri Pirko sizeof(key->tp.src)); 132577b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 1326aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 132777b9900eSJiri Pirko sizeof(key->tp.dst)); 1328fdfc7dd6SJiri Pirko fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 1329fdfc7dd6SJiri Pirko &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 1330fdfc7dd6SJiri Pirko sizeof(key->tcp.flags)); 133177b9900eSJiri Pirko } else if (key->basic.ip_proto == IPPROTO_UDP) { 133277b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 1333aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 133477b9900eSJiri Pirko sizeof(key->tp.src)); 133577b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 1336aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 133777b9900eSJiri Pirko sizeof(key->tp.dst)); 13385976c5f4SSimon Horman } else if (key->basic.ip_proto == IPPROTO_SCTP) { 13395976c5f4SSimon Horman fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 13405976c5f4SSimon Horman &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 13415976c5f4SSimon Horman sizeof(key->tp.src)); 13425976c5f4SSimon Horman fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 13435976c5f4SSimon Horman &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 13445976c5f4SSimon Horman sizeof(key->tp.dst)); 13457b684884SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_IP) && 13467b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMP) { 13477b684884SSimon Horman fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE, 13487b684884SSimon Horman &mask->icmp.type, 13497b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 13507b684884SSimon Horman sizeof(key->icmp.type)); 13517b684884SSimon Horman fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE, 13527b684884SSimon Horman &mask->icmp.code, 13537b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 13547b684884SSimon Horman sizeof(key->icmp.code)); 13557b684884SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_IPV6) && 13567b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMPV6) { 13577b684884SSimon Horman fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE, 13587b684884SSimon Horman &mask->icmp.type, 13597b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 13607b684884SSimon Horman sizeof(key->icmp.type)); 1361040587afSSimon Horman fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE, 13627b684884SSimon Horman &mask->icmp.code, 1363040587afSSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 13647b684884SSimon Horman sizeof(key->icmp.code)); 1365a577d8f7SBenjamin LaHaise } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) || 1366a577d8f7SBenjamin LaHaise key->basic.n_proto == htons(ETH_P_MPLS_MC)) { 13671a7fca63SBenjamin LaHaise ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls); 13681a7fca63SBenjamin LaHaise if (ret) 13691a7fca63SBenjamin LaHaise return ret; 137099d31326SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_ARP) || 137199d31326SSimon Horman key->basic.n_proto == htons(ETH_P_RARP)) { 137299d31326SSimon Horman fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP, 137399d31326SSimon Horman &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK, 137499d31326SSimon Horman sizeof(key->arp.sip)); 137599d31326SSimon Horman fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP, 137699d31326SSimon Horman &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK, 137799d31326SSimon Horman sizeof(key->arp.tip)); 137899d31326SSimon Horman fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP, 137999d31326SSimon Horman &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK, 138099d31326SSimon Horman sizeof(key->arp.op)); 138199d31326SSimon Horman fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 138299d31326SSimon Horman mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 138399d31326SSimon Horman sizeof(key->arp.sha)); 138499d31326SSimon Horman fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 138599d31326SSimon Horman mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 138699d31326SSimon Horman sizeof(key->arp.tha)); 138777b9900eSJiri Pirko } 138877b9900eSJiri Pirko 13895c72299fSAmritha Nambiar if (key->basic.ip_proto == IPPROTO_TCP || 13905c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_UDP || 13915c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_SCTP) { 13925c72299fSAmritha Nambiar ret = fl_set_key_port_range(tb, key, mask); 13935c72299fSAmritha Nambiar if (ret) 13945c72299fSAmritha Nambiar return ret; 13955c72299fSAmritha Nambiar } 13965c72299fSAmritha Nambiar 1397bc3103f1SAmir Vadai if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 1398bc3103f1SAmir Vadai tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 1399bc3103f1SAmir Vadai key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1400970bfcd0SPaul Blakey mask->enc_control.addr_type = ~0; 1401bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv4.src, 1402bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC, 1403bc3103f1SAmir Vadai &mask->enc_ipv4.src, 1404bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 1405bc3103f1SAmir Vadai sizeof(key->enc_ipv4.src)); 1406bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv4.dst, 1407bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST, 1408bc3103f1SAmir Vadai &mask->enc_ipv4.dst, 1409bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 1410bc3103f1SAmir Vadai sizeof(key->enc_ipv4.dst)); 1411bc3103f1SAmir Vadai } 1412bc3103f1SAmir Vadai 1413bc3103f1SAmir Vadai if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 1414bc3103f1SAmir Vadai tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 1415bc3103f1SAmir Vadai key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1416970bfcd0SPaul Blakey mask->enc_control.addr_type = ~0; 1417bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv6.src, 1418bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC, 1419bc3103f1SAmir Vadai &mask->enc_ipv6.src, 1420bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 1421bc3103f1SAmir Vadai sizeof(key->enc_ipv6.src)); 1422bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv6.dst, 1423bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST, 1424bc3103f1SAmir Vadai &mask->enc_ipv6.dst, 1425bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 1426bc3103f1SAmir Vadai sizeof(key->enc_ipv6.dst)); 1427bc3103f1SAmir Vadai } 1428bc3103f1SAmir Vadai 1429bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 1430eb523f42SHadar Hen Zion &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC, 1431bc3103f1SAmir Vadai sizeof(key->enc_key_id.keyid)); 1432bc3103f1SAmir Vadai 1433f4d997fdSHadar Hen Zion fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 1434f4d997fdSHadar Hen Zion &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 1435f4d997fdSHadar Hen Zion sizeof(key->enc_tp.src)); 1436f4d997fdSHadar Hen Zion 1437f4d997fdSHadar Hen Zion fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 1438f4d997fdSHadar Hen Zion &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 1439f4d997fdSHadar Hen Zion sizeof(key->enc_tp.dst)); 1440f4d997fdSHadar Hen Zion 14410e2c17b6SOr Gerlitz fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip); 14420e2c17b6SOr Gerlitz 14430a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPTS]) { 14440a6e7778SPieter Jansen van Vuuren ret = fl_set_enc_opt(tb, key, mask, extack); 14450a6e7778SPieter Jansen van Vuuren if (ret) 14460a6e7778SPieter Jansen van Vuuren return ret; 14470a6e7778SPieter Jansen van Vuuren } 14480a6e7778SPieter Jansen van Vuuren 1449e0ace68aSPaul Blakey ret = fl_set_key_ct(tb, &key->ct, &mask->ct, extack); 1450e0ace68aSPaul Blakey if (ret) 1451e0ace68aSPaul Blakey return ret; 1452e0ace68aSPaul Blakey 1453d9724772SOr Gerlitz if (tb[TCA_FLOWER_KEY_FLAGS]) 1454d9724772SOr Gerlitz ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags); 1455faa3ffceSOr Gerlitz 1456d9724772SOr Gerlitz return ret; 145777b9900eSJiri Pirko } 145877b9900eSJiri Pirko 145905cd271fSPaul Blakey static void fl_mask_copy(struct fl_flow_mask *dst, 146005cd271fSPaul Blakey struct fl_flow_mask *src) 146177b9900eSJiri Pirko { 146205cd271fSPaul Blakey const void *psrc = fl_key_get_start(&src->key, src); 146305cd271fSPaul Blakey void *pdst = fl_key_get_start(&dst->key, src); 146477b9900eSJiri Pirko 146505cd271fSPaul Blakey memcpy(pdst, psrc, fl_mask_range(src)); 146605cd271fSPaul Blakey dst->range = src->range; 146777b9900eSJiri Pirko } 146877b9900eSJiri Pirko 146977b9900eSJiri Pirko static const struct rhashtable_params fl_ht_params = { 147077b9900eSJiri Pirko .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 147177b9900eSJiri Pirko .head_offset = offsetof(struct cls_fl_filter, ht_node), 147277b9900eSJiri Pirko .automatic_shrinking = true, 147377b9900eSJiri Pirko }; 147477b9900eSJiri Pirko 147505cd271fSPaul Blakey static int fl_init_mask_hashtable(struct fl_flow_mask *mask) 147677b9900eSJiri Pirko { 147705cd271fSPaul Blakey mask->filter_ht_params = fl_ht_params; 147805cd271fSPaul Blakey mask->filter_ht_params.key_len = fl_mask_range(mask); 147905cd271fSPaul Blakey mask->filter_ht_params.key_offset += mask->range.start; 148077b9900eSJiri Pirko 148105cd271fSPaul Blakey return rhashtable_init(&mask->ht, &mask->filter_ht_params); 148277b9900eSJiri Pirko } 148377b9900eSJiri Pirko 148477b9900eSJiri Pirko #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 1485c593642cSPankaj Bharadiya #define FL_KEY_MEMBER_SIZE(member) sizeof_field(struct fl_flow_key, member) 148677b9900eSJiri Pirko 1487339ba878SHadar Hen Zion #define FL_KEY_IS_MASKED(mask, member) \ 1488339ba878SHadar Hen Zion memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 1489339ba878SHadar Hen Zion 0, FL_KEY_MEMBER_SIZE(member)) \ 149077b9900eSJiri Pirko 149177b9900eSJiri Pirko #define FL_KEY_SET(keys, cnt, id, member) \ 149277b9900eSJiri Pirko do { \ 149377b9900eSJiri Pirko keys[cnt].key_id = id; \ 149477b9900eSJiri Pirko keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 149577b9900eSJiri Pirko cnt++; \ 149677b9900eSJiri Pirko } while(0); 149777b9900eSJiri Pirko 1498339ba878SHadar Hen Zion #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 149977b9900eSJiri Pirko do { \ 1500339ba878SHadar Hen Zion if (FL_KEY_IS_MASKED(mask, member)) \ 150177b9900eSJiri Pirko FL_KEY_SET(keys, cnt, id, member); \ 150277b9900eSJiri Pirko } while(0); 150377b9900eSJiri Pirko 150433fb5cbaSJiri Pirko static void fl_init_dissector(struct flow_dissector *dissector, 150533fb5cbaSJiri Pirko struct fl_flow_key *mask) 150677b9900eSJiri Pirko { 150777b9900eSJiri Pirko struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 150877b9900eSJiri Pirko size_t cnt = 0; 150977b9900eSJiri Pirko 15108212ed77SJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15118212ed77SJiri Pirko FLOW_DISSECTOR_KEY_META, meta); 151242aecaa9STom Herbert FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 151377b9900eSJiri Pirko FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 151433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 151577b9900eSJiri Pirko FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 151633fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 151777b9900eSJiri Pirko FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 151833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 151977b9900eSJiri Pirko FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 15208ffb055bSYoshiki Komachi FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15218ffb055bSYoshiki Komachi FLOW_DISSECTOR_KEY_PORTS, tp); 15228ffb055bSYoshiki Komachi FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15238ffb055bSYoshiki Komachi FLOW_DISSECTOR_KEY_PORTS_RANGE, tp_range); 152433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15254d80cc0aSOr Gerlitz FLOW_DISSECTOR_KEY_IP, ip); 152633fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1527fdfc7dd6SJiri Pirko FLOW_DISSECTOR_KEY_TCP, tcp); 152833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15297b684884SSimon Horman FLOW_DISSECTOR_KEY_ICMP, icmp); 153033fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 153199d31326SSimon Horman FLOW_DISSECTOR_KEY_ARP, arp); 153233fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1533a577d8f7SBenjamin LaHaise FLOW_DISSECTOR_KEY_MPLS, mpls); 153433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15359399ae9aSHadar Hen Zion FLOW_DISSECTOR_KEY_VLAN, vlan); 153633fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1537d64efd09SJianbo Liu FLOW_DISSECTOR_KEY_CVLAN, cvlan); 153833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1539519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 154033fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1541519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4); 154233fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1543519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6); 154433fb5cbaSJiri Pirko if (FL_KEY_IS_MASKED(mask, enc_ipv4) || 154533fb5cbaSJiri Pirko FL_KEY_IS_MASKED(mask, enc_ipv6)) 1546519d1052SHadar Hen Zion FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL, 1547519d1052SHadar Hen Zion enc_control); 154833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1549f4d997fdSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp); 155033fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15510e2c17b6SOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IP, enc_ip); 15520a6e7778SPieter Jansen van Vuuren FL_KEY_SET_IF_MASKED(mask, keys, cnt, 15530a6e7778SPieter Jansen van Vuuren FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts); 1554e0ace68aSPaul Blakey FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1555e0ace68aSPaul Blakey FLOW_DISSECTOR_KEY_CT, ct); 155677b9900eSJiri Pirko 155733fb5cbaSJiri Pirko skb_flow_dissector_init(dissector, keys, cnt); 155805cd271fSPaul Blakey } 155905cd271fSPaul Blakey 156005cd271fSPaul Blakey static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, 156105cd271fSPaul Blakey struct fl_flow_mask *mask) 156205cd271fSPaul Blakey { 156305cd271fSPaul Blakey struct fl_flow_mask *newmask; 156405cd271fSPaul Blakey int err; 156505cd271fSPaul Blakey 156605cd271fSPaul Blakey newmask = kzalloc(sizeof(*newmask), GFP_KERNEL); 156705cd271fSPaul Blakey if (!newmask) 156805cd271fSPaul Blakey return ERR_PTR(-ENOMEM); 156905cd271fSPaul Blakey 157005cd271fSPaul Blakey fl_mask_copy(newmask, mask); 157105cd271fSPaul Blakey 15728ffb055bSYoshiki Komachi if ((newmask->key.tp_range.tp_min.dst && 15738ffb055bSYoshiki Komachi newmask->key.tp_range.tp_max.dst) || 15748ffb055bSYoshiki Komachi (newmask->key.tp_range.tp_min.src && 15758ffb055bSYoshiki Komachi newmask->key.tp_range.tp_max.src)) 15765c72299fSAmritha Nambiar newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE; 15775c72299fSAmritha Nambiar 157805cd271fSPaul Blakey err = fl_init_mask_hashtable(newmask); 157905cd271fSPaul Blakey if (err) 158005cd271fSPaul Blakey goto errout_free; 158105cd271fSPaul Blakey 158233fb5cbaSJiri Pirko fl_init_dissector(&newmask->dissector, &newmask->key); 158305cd271fSPaul Blakey 158405cd271fSPaul Blakey INIT_LIST_HEAD_RCU(&newmask->filters); 158505cd271fSPaul Blakey 1586f48ef4d5SVlad Buslov refcount_set(&newmask->refcnt, 1); 1587195c234dSVlad Buslov err = rhashtable_replace_fast(&head->ht, &mask->ht_node, 1588195c234dSVlad Buslov &newmask->ht_node, mask_ht_params); 158905cd271fSPaul Blakey if (err) 159005cd271fSPaul Blakey goto errout_destroy; 159105cd271fSPaul Blakey 1592259e60f9SVlad Buslov spin_lock(&head->masks_lock); 159305cd271fSPaul Blakey list_add_tail_rcu(&newmask->list, &head->masks); 1594259e60f9SVlad Buslov spin_unlock(&head->masks_lock); 159505cd271fSPaul Blakey 159605cd271fSPaul Blakey return newmask; 159705cd271fSPaul Blakey 159805cd271fSPaul Blakey errout_destroy: 159905cd271fSPaul Blakey rhashtable_destroy(&newmask->ht); 160005cd271fSPaul Blakey errout_free: 160105cd271fSPaul Blakey kfree(newmask); 160205cd271fSPaul Blakey 160305cd271fSPaul Blakey return ERR_PTR(err); 160477b9900eSJiri Pirko } 160577b9900eSJiri Pirko 160677b9900eSJiri Pirko static int fl_check_assign_mask(struct cls_fl_head *head, 160705cd271fSPaul Blakey struct cls_fl_filter *fnew, 160805cd271fSPaul Blakey struct cls_fl_filter *fold, 160977b9900eSJiri Pirko struct fl_flow_mask *mask) 161077b9900eSJiri Pirko { 161105cd271fSPaul Blakey struct fl_flow_mask *newmask; 1612f48ef4d5SVlad Buslov int ret = 0; 161377b9900eSJiri Pirko 1614f48ef4d5SVlad Buslov rcu_read_lock(); 1615195c234dSVlad Buslov 1616195c234dSVlad Buslov /* Insert mask as temporary node to prevent concurrent creation of mask 1617195c234dSVlad Buslov * with same key. Any concurrent lookups with same key will return 161899815f50SVlad Buslov * -EAGAIN because mask's refcnt is zero. 1619195c234dSVlad Buslov */ 1620195c234dSVlad Buslov fnew->mask = rhashtable_lookup_get_insert_fast(&head->ht, 1621195c234dSVlad Buslov &mask->ht_node, 1622195c234dSVlad Buslov mask_ht_params); 162305cd271fSPaul Blakey if (!fnew->mask) { 1624f48ef4d5SVlad Buslov rcu_read_unlock(); 1625f48ef4d5SVlad Buslov 1626195c234dSVlad Buslov if (fold) { 1627195c234dSVlad Buslov ret = -EINVAL; 1628195c234dSVlad Buslov goto errout_cleanup; 1629195c234dSVlad Buslov } 163005cd271fSPaul Blakey 163105cd271fSPaul Blakey newmask = fl_create_new_mask(head, mask); 1632195c234dSVlad Buslov if (IS_ERR(newmask)) { 1633195c234dSVlad Buslov ret = PTR_ERR(newmask); 1634195c234dSVlad Buslov goto errout_cleanup; 1635195c234dSVlad Buslov } 163605cd271fSPaul Blakey 163705cd271fSPaul Blakey fnew->mask = newmask; 163877b9900eSJiri Pirko return 0; 1639195c234dSVlad Buslov } else if (IS_ERR(fnew->mask)) { 1640195c234dSVlad Buslov ret = PTR_ERR(fnew->mask); 1641f48ef4d5SVlad Buslov } else if (fold && fold->mask != fnew->mask) { 1642f48ef4d5SVlad Buslov ret = -EINVAL; 1643f48ef4d5SVlad Buslov } else if (!refcount_inc_not_zero(&fnew->mask->refcnt)) { 1644f48ef4d5SVlad Buslov /* Mask was deleted concurrently, try again */ 1645f48ef4d5SVlad Buslov ret = -EAGAIN; 1646f48ef4d5SVlad Buslov } 1647f48ef4d5SVlad Buslov rcu_read_unlock(); 1648f48ef4d5SVlad Buslov return ret; 1649195c234dSVlad Buslov 1650195c234dSVlad Buslov errout_cleanup: 1651195c234dSVlad Buslov rhashtable_remove_fast(&head->ht, &mask->ht_node, 1652195c234dSVlad Buslov mask_ht_params); 1653195c234dSVlad Buslov return ret; 165477b9900eSJiri Pirko } 165577b9900eSJiri Pirko 165677b9900eSJiri Pirko static int fl_set_parms(struct net *net, struct tcf_proto *tp, 165777b9900eSJiri Pirko struct cls_fl_filter *f, struct fl_flow_mask *mask, 165877b9900eSJiri Pirko unsigned long base, struct nlattr **tb, 165950a56190SAlexander Aring struct nlattr *est, bool ovr, 1660c24e43d8SVlad Buslov struct fl_flow_tmplt *tmplt, bool rtnl_held, 166150a56190SAlexander Aring struct netlink_ext_ack *extack) 166277b9900eSJiri Pirko { 166377b9900eSJiri Pirko int err; 166477b9900eSJiri Pirko 1665c24e43d8SVlad Buslov err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, rtnl_held, 1666ec6743a1SVlad Buslov extack); 166777b9900eSJiri Pirko if (err < 0) 166877b9900eSJiri Pirko return err; 166977b9900eSJiri Pirko 167077b9900eSJiri Pirko if (tb[TCA_FLOWER_CLASSID]) { 167177b9900eSJiri Pirko f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 1672c24e43d8SVlad Buslov if (!rtnl_held) 1673c24e43d8SVlad Buslov rtnl_lock(); 167477b9900eSJiri Pirko tcf_bind_filter(tp, &f->res, base); 1675c24e43d8SVlad Buslov if (!rtnl_held) 1676c24e43d8SVlad Buslov rtnl_unlock(); 167777b9900eSJiri Pirko } 167877b9900eSJiri Pirko 16791057c55fSAlexander Aring err = fl_set_key(net, tb, &f->key, &mask->key, extack); 168077b9900eSJiri Pirko if (err) 168145507529SJiri Pirko return err; 168277b9900eSJiri Pirko 168377b9900eSJiri Pirko fl_mask_update_range(mask); 168477b9900eSJiri Pirko fl_set_masked_key(&f->mkey, &f->key, mask); 168577b9900eSJiri Pirko 1686b95ec7ebSJiri Pirko if (!fl_mask_fits_tmplt(tmplt, mask)) { 1687b95ec7ebSJiri Pirko NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template"); 1688b95ec7ebSJiri Pirko return -EINVAL; 1689b95ec7ebSJiri Pirko } 1690b95ec7ebSJiri Pirko 169177b9900eSJiri Pirko return 0; 169277b9900eSJiri Pirko } 169377b9900eSJiri Pirko 16941f17f774SVlad Buslov static int fl_ht_insert_unique(struct cls_fl_filter *fnew, 16951f17f774SVlad Buslov struct cls_fl_filter *fold, 16961f17f774SVlad Buslov bool *in_ht) 16971f17f774SVlad Buslov { 16981f17f774SVlad Buslov struct fl_flow_mask *mask = fnew->mask; 16991f17f774SVlad Buslov int err; 17001f17f774SVlad Buslov 17019e35552aSVlad Buslov err = rhashtable_lookup_insert_fast(&mask->ht, 17021f17f774SVlad Buslov &fnew->ht_node, 17031f17f774SVlad Buslov mask->filter_ht_params); 17041f17f774SVlad Buslov if (err) { 17051f17f774SVlad Buslov *in_ht = false; 17061f17f774SVlad Buslov /* It is okay if filter with same key exists when 17071f17f774SVlad Buslov * overwriting. 17081f17f774SVlad Buslov */ 17091f17f774SVlad Buslov return fold && err == -EEXIST ? 0 : err; 17101f17f774SVlad Buslov } 17111f17f774SVlad Buslov 17121f17f774SVlad Buslov *in_ht = true; 17131f17f774SVlad Buslov return 0; 17141f17f774SVlad Buslov } 17151f17f774SVlad Buslov 171677b9900eSJiri Pirko static int fl_change(struct net *net, struct sk_buff *in_skb, 171777b9900eSJiri Pirko struct tcf_proto *tp, unsigned long base, 171877b9900eSJiri Pirko u32 handle, struct nlattr **tca, 171912db03b6SVlad Buslov void **arg, bool ovr, bool rtnl_held, 172012db03b6SVlad Buslov struct netlink_ext_ack *extack) 172177b9900eSJiri Pirko { 1722e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 17238113c095SWANG Cong struct cls_fl_filter *fold = *arg; 172477b9900eSJiri Pirko struct cls_fl_filter *fnew; 17252cddd201SIvan Vecera struct fl_flow_mask *mask; 172639b7b6a6SArnd Bergmann struct nlattr **tb; 17271f17f774SVlad Buslov bool in_ht; 172877b9900eSJiri Pirko int err; 172977b9900eSJiri Pirko 173006177558SVlad Buslov if (!tca[TCA_OPTIONS]) { 173106177558SVlad Buslov err = -EINVAL; 173206177558SVlad Buslov goto errout_fold; 173306177558SVlad Buslov } 173477b9900eSJiri Pirko 17352cddd201SIvan Vecera mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL); 173606177558SVlad Buslov if (!mask) { 173706177558SVlad Buslov err = -ENOBUFS; 173806177558SVlad Buslov goto errout_fold; 173906177558SVlad Buslov } 174039b7b6a6SArnd Bergmann 17412cddd201SIvan Vecera tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 17422cddd201SIvan Vecera if (!tb) { 17432cddd201SIvan Vecera err = -ENOBUFS; 17442cddd201SIvan Vecera goto errout_mask_alloc; 17452cddd201SIvan Vecera } 17462cddd201SIvan Vecera 17478cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 17488cb08174SJohannes Berg tca[TCA_OPTIONS], fl_policy, NULL); 174977b9900eSJiri Pirko if (err < 0) 175039b7b6a6SArnd Bergmann goto errout_tb; 175177b9900eSJiri Pirko 175239b7b6a6SArnd Bergmann if (fold && handle && fold->handle != handle) { 175339b7b6a6SArnd Bergmann err = -EINVAL; 175439b7b6a6SArnd Bergmann goto errout_tb; 175539b7b6a6SArnd Bergmann } 175677b9900eSJiri Pirko 175777b9900eSJiri Pirko fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 175839b7b6a6SArnd Bergmann if (!fnew) { 175939b7b6a6SArnd Bergmann err = -ENOBUFS; 176039b7b6a6SArnd Bergmann goto errout_tb; 176139b7b6a6SArnd Bergmann } 1762c049d56eSVlad Buslov INIT_LIST_HEAD(&fnew->hw_list); 176306177558SVlad Buslov refcount_set(&fnew->refcnt, 1); 176477b9900eSJiri Pirko 176514215108SCong Wang err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0); 1766b9a24bb7SWANG Cong if (err < 0) 1767b9a24bb7SWANG Cong goto errout; 176877b9900eSJiri Pirko 1769ecb3dea4SVlad Buslov if (tb[TCA_FLOWER_FLAGS]) { 1770ecb3dea4SVlad Buslov fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 1771ecb3dea4SVlad Buslov 1772ecb3dea4SVlad Buslov if (!tc_flags_valid(fnew->flags)) { 1773ecb3dea4SVlad Buslov err = -EINVAL; 1774ecb3dea4SVlad Buslov goto errout; 1775ecb3dea4SVlad Buslov } 1776ecb3dea4SVlad Buslov } 1777ecb3dea4SVlad Buslov 1778ecb3dea4SVlad Buslov err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr, 1779c24e43d8SVlad Buslov tp->chain->tmplt_priv, rtnl_held, extack); 1780ecb3dea4SVlad Buslov if (err) 1781ecb3dea4SVlad Buslov goto errout; 1782ecb3dea4SVlad Buslov 1783ecb3dea4SVlad Buslov err = fl_check_assign_mask(head, fnew, fold, mask); 1784ecb3dea4SVlad Buslov if (err) 1785ecb3dea4SVlad Buslov goto errout; 1786ecb3dea4SVlad Buslov 17871f17f774SVlad Buslov err = fl_ht_insert_unique(fnew, fold, &in_ht); 17881f17f774SVlad Buslov if (err) 17891f17f774SVlad Buslov goto errout_mask; 17901f17f774SVlad Buslov 179179685219SHadar Hen Zion if (!tc_skip_hw(fnew->flags)) { 1792c24e43d8SVlad Buslov err = fl_hw_replace_filter(tp, fnew, rtnl_held, extack); 1793e8eb36cdSAmir Vadai if (err) 17941f17f774SVlad Buslov goto errout_ht; 179579685219SHadar Hen Zion } 17965b33f488SAmir Vadai 179755593960SOr Gerlitz if (!tc_in_hw(fnew->flags)) 179855593960SOr Gerlitz fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW; 179955593960SOr Gerlitz 18003d81e711SVlad Buslov spin_lock(&tp->lock); 18013d81e711SVlad Buslov 1802272ffaadSVlad Buslov /* tp was deleted concurrently. -EAGAIN will cause caller to lookup 1803272ffaadSVlad Buslov * proto again or create new one, if necessary. 1804272ffaadSVlad Buslov */ 1805272ffaadSVlad Buslov if (tp->deleting) { 1806272ffaadSVlad Buslov err = -EAGAIN; 1807272ffaadSVlad Buslov goto errout_hw; 1808272ffaadSVlad Buslov } 1809272ffaadSVlad Buslov 18105b33f488SAmir Vadai if (fold) { 1811b2552b8cSVlad Buslov /* Fold filter was deleted concurrently. Retry lookup. */ 1812b2552b8cSVlad Buslov if (fold->deleted) { 1813b2552b8cSVlad Buslov err = -EAGAIN; 1814b2552b8cSVlad Buslov goto errout_hw; 1815b2552b8cSVlad Buslov } 1816b2552b8cSVlad Buslov 1817620da486SVlad Buslov fnew->handle = handle; 1818620da486SVlad Buslov 18191f17f774SVlad Buslov if (!in_ht) { 18201f17f774SVlad Buslov struct rhashtable_params params = 18211f17f774SVlad Buslov fnew->mask->filter_ht_params; 18221f17f774SVlad Buslov 18231f17f774SVlad Buslov err = rhashtable_insert_fast(&fnew->mask->ht, 18241f17f774SVlad Buslov &fnew->ht_node, 18251f17f774SVlad Buslov params); 1826620da486SVlad Buslov if (err) 1827620da486SVlad Buslov goto errout_hw; 18281f17f774SVlad Buslov in_ht = true; 18291f17f774SVlad Buslov } 1830620da486SVlad Buslov 1831c049d56eSVlad Buslov refcount_inc(&fnew->refcnt); 183205cd271fSPaul Blakey rhashtable_remove_fast(&fold->mask->ht, 183305cd271fSPaul Blakey &fold->ht_node, 183405cd271fSPaul Blakey fold->mask->filter_ht_params); 1835234a4624SMatthew Wilcox idr_replace(&head->handle_idr, fnew, fnew->handle); 1836ff3532f2SDaniel Borkmann list_replace_rcu(&fold->list, &fnew->list); 1837b2552b8cSVlad Buslov fold->deleted = true; 1838620da486SVlad Buslov 18393d81e711SVlad Buslov spin_unlock(&tp->lock); 18403d81e711SVlad Buslov 18419994677cSVlad Buslov fl_mask_put(head, fold->mask); 1842620da486SVlad Buslov if (!tc_skip_hw(fold->flags)) 1843c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, fold, rtnl_held, NULL); 184477b9900eSJiri Pirko tcf_unbind_filter(tp, &fold->res); 184506177558SVlad Buslov /* Caller holds reference to fold, so refcnt is always > 0 184606177558SVlad Buslov * after this. 184706177558SVlad Buslov */ 184806177558SVlad Buslov refcount_dec(&fold->refcnt); 184906177558SVlad Buslov __fl_put(fold); 185077b9900eSJiri Pirko } else { 1851620da486SVlad Buslov if (handle) { 1852620da486SVlad Buslov /* user specifies a handle and it doesn't exist */ 1853620da486SVlad Buslov err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1854620da486SVlad Buslov handle, GFP_ATOMIC); 18559a2d9389SVlad Buslov 18569a2d9389SVlad Buslov /* Filter with specified handle was concurrently 18579a2d9389SVlad Buslov * inserted after initial check in cls_api. This is not 18589a2d9389SVlad Buslov * necessarily an error if NLM_F_EXCL is not set in 18599a2d9389SVlad Buslov * message flags. Returning EAGAIN will cause cls_api to 18609a2d9389SVlad Buslov * try to update concurrently inserted rule. 18619a2d9389SVlad Buslov */ 18629a2d9389SVlad Buslov if (err == -ENOSPC) 18639a2d9389SVlad Buslov err = -EAGAIN; 1864620da486SVlad Buslov } else { 1865620da486SVlad Buslov handle = 1; 1866620da486SVlad Buslov err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1867620da486SVlad Buslov INT_MAX, GFP_ATOMIC); 1868620da486SVlad Buslov } 1869620da486SVlad Buslov if (err) 1870620da486SVlad Buslov goto errout_hw; 1871620da486SVlad Buslov 1872c049d56eSVlad Buslov refcount_inc(&fnew->refcnt); 1873620da486SVlad Buslov fnew->handle = handle; 187405cd271fSPaul Blakey list_add_tail_rcu(&fnew->list, &fnew->mask->filters); 18753d81e711SVlad Buslov spin_unlock(&tp->lock); 187677b9900eSJiri Pirko } 187777b9900eSJiri Pirko 1878620da486SVlad Buslov *arg = fnew; 1879620da486SVlad Buslov 188039b7b6a6SArnd Bergmann kfree(tb); 188199815f50SVlad Buslov tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 188277b9900eSJiri Pirko return 0; 188377b9900eSJiri Pirko 1884c049d56eSVlad Buslov errout_ht: 1885c049d56eSVlad Buslov spin_lock(&tp->lock); 1886620da486SVlad Buslov errout_hw: 1887c049d56eSVlad Buslov fnew->deleted = true; 18883d81e711SVlad Buslov spin_unlock(&tp->lock); 1889620da486SVlad Buslov if (!tc_skip_hw(fnew->flags)) 1890c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, fnew, rtnl_held, NULL); 18911f17f774SVlad Buslov if (in_ht) 18921f17f774SVlad Buslov rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node, 18931f17f774SVlad Buslov fnew->mask->filter_ht_params); 1894ecb3dea4SVlad Buslov errout_mask: 18959994677cSVlad Buslov fl_mask_put(head, fnew->mask); 189677b9900eSJiri Pirko errout: 1897c049d56eSVlad Buslov __fl_put(fnew); 189839b7b6a6SArnd Bergmann errout_tb: 189939b7b6a6SArnd Bergmann kfree(tb); 19002cddd201SIvan Vecera errout_mask_alloc: 190199815f50SVlad Buslov tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 190206177558SVlad Buslov errout_fold: 190306177558SVlad Buslov if (fold) 190406177558SVlad Buslov __fl_put(fold); 190577b9900eSJiri Pirko return err; 190677b9900eSJiri Pirko } 190777b9900eSJiri Pirko 1908571acf21SAlexander Aring static int fl_delete(struct tcf_proto *tp, void *arg, bool *last, 190912db03b6SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack) 191077b9900eSJiri Pirko { 1911e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 19128113c095SWANG Cong struct cls_fl_filter *f = arg; 1913b2552b8cSVlad Buslov bool last_on_mask; 1914b2552b8cSVlad Buslov int err = 0; 191577b9900eSJiri Pirko 1916c24e43d8SVlad Buslov err = __fl_delete(tp, f, &last_on_mask, rtnl_held, extack); 191705cd271fSPaul Blakey *last = list_empty(&head->masks); 191806177558SVlad Buslov __fl_put(f); 191906177558SVlad Buslov 1920b2552b8cSVlad Buslov return err; 192177b9900eSJiri Pirko } 192277b9900eSJiri Pirko 192312db03b6SVlad Buslov static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg, 192412db03b6SVlad Buslov bool rtnl_held) 192577b9900eSJiri Pirko { 1926d39d7149SCong Wang struct cls_fl_head *head = fl_head_dereference(tp); 1927d39d7149SCong Wang unsigned long id = arg->cookie, tmp; 192877b9900eSJiri Pirko struct cls_fl_filter *f; 192977b9900eSJiri Pirko 193001683a14SVlad Buslov arg->count = arg->skip; 193101683a14SVlad Buslov 1932d39d7149SCong Wang idr_for_each_entry_continue_ul(&head->handle_idr, f, tmp, id) { 1933d39d7149SCong Wang /* don't return filters that are being deleted */ 1934d39d7149SCong Wang if (!refcount_inc_not_zero(&f->refcnt)) 1935d39d7149SCong Wang continue; 19368113c095SWANG Cong if (arg->fn(tp, f, arg) < 0) { 193706177558SVlad Buslov __fl_put(f); 193877b9900eSJiri Pirko arg->stop = 1; 193977b9900eSJiri Pirko break; 194077b9900eSJiri Pirko } 194106177558SVlad Buslov __fl_put(f); 194277b9900eSJiri Pirko arg->count++; 194377b9900eSJiri Pirko } 1944d39d7149SCong Wang arg->cookie = id; 194577b9900eSJiri Pirko } 194677b9900eSJiri Pirko 1947c049d56eSVlad Buslov static struct cls_fl_filter * 1948c049d56eSVlad Buslov fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add) 1949c049d56eSVlad Buslov { 1950c049d56eSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 1951c049d56eSVlad Buslov 1952c049d56eSVlad Buslov spin_lock(&tp->lock); 1953c049d56eSVlad Buslov if (list_empty(&head->hw_filters)) { 1954c049d56eSVlad Buslov spin_unlock(&tp->lock); 1955c049d56eSVlad Buslov return NULL; 1956c049d56eSVlad Buslov } 1957c049d56eSVlad Buslov 1958c049d56eSVlad Buslov if (!f) 1959c049d56eSVlad Buslov f = list_entry(&head->hw_filters, struct cls_fl_filter, 1960c049d56eSVlad Buslov hw_list); 1961c049d56eSVlad Buslov list_for_each_entry_continue(f, &head->hw_filters, hw_list) { 1962c049d56eSVlad Buslov if (!(add && f->deleted) && refcount_inc_not_zero(&f->refcnt)) { 1963c049d56eSVlad Buslov spin_unlock(&tp->lock); 1964c049d56eSVlad Buslov return f; 1965c049d56eSVlad Buslov } 1966c049d56eSVlad Buslov } 1967c049d56eSVlad Buslov 1968c049d56eSVlad Buslov spin_unlock(&tp->lock); 1969c049d56eSVlad Buslov return NULL; 1970c049d56eSVlad Buslov } 1971c049d56eSVlad Buslov 1972a7323311SPablo Neira Ayuso static int fl_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, 197331533cbaSJohn Hurley void *cb_priv, struct netlink_ext_ack *extack) 197431533cbaSJohn Hurley { 197531533cbaSJohn Hurley struct tcf_block *block = tp->chain->block; 1976f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 1977c049d56eSVlad Buslov struct cls_fl_filter *f = NULL; 197831533cbaSJohn Hurley int err; 197931533cbaSJohn Hurley 1980c049d56eSVlad Buslov /* hw_filters list can only be changed by hw offload functions after 1981c049d56eSVlad Buslov * obtaining rtnl lock. Make sure it is not changed while reoffload is 1982c049d56eSVlad Buslov * iterating it. 1983c049d56eSVlad Buslov */ 1984c049d56eSVlad Buslov ASSERT_RTNL(); 198531533cbaSJohn Hurley 1986c049d56eSVlad Buslov while ((f = fl_get_next_hw_filter(tp, f, add))) { 1987e3ab786bSPablo Neira Ayuso cls_flower.rule = 1988e3ab786bSPablo Neira Ayuso flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 198995e27a4dSJohn Hurley if (!cls_flower.rule) { 199095e27a4dSJohn Hurley __fl_put(f); 19918f256622SPablo Neira Ayuso return -ENOMEM; 199295e27a4dSJohn Hurley } 19938f256622SPablo Neira Ayuso 199495e27a4dSJohn Hurley tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, 1995d6787147SPieter Jansen van Vuuren extack); 199631533cbaSJohn Hurley cls_flower.command = add ? 1997f9e30088SPablo Neira Ayuso FLOW_CLS_REPLACE : FLOW_CLS_DESTROY; 199831533cbaSJohn Hurley cls_flower.cookie = (unsigned long)f; 199995e27a4dSJohn Hurley cls_flower.rule->match.dissector = &f->mask->dissector; 200095e27a4dSJohn Hurley cls_flower.rule->match.mask = &f->mask->key; 20018f256622SPablo Neira Ayuso cls_flower.rule->match.key = &f->mkey; 20023a7b6861SPablo Neira Ayuso 20039838b20aSVlad Buslov err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts, 20049838b20aSVlad Buslov true); 20053a7b6861SPablo Neira Ayuso if (err) { 20063a7b6861SPablo Neira Ayuso kfree(cls_flower.rule); 20071f15bb4fSVlad Buslov if (tc_skip_sw(f->flags)) { 20081f15bb4fSVlad Buslov NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 200995e27a4dSJohn Hurley __fl_put(f); 20103a7b6861SPablo Neira Ayuso return err; 20113a7b6861SPablo Neira Ayuso } 201295e27a4dSJohn Hurley goto next_flow; 20131f15bb4fSVlad Buslov } 20143a7b6861SPablo Neira Ayuso 201531533cbaSJohn Hurley cls_flower.classid = f->res.classid; 201631533cbaSJohn Hurley 201740119211SVlad Buslov err = tc_setup_cb_reoffload(block, tp, add, cb, 201840119211SVlad Buslov TC_SETUP_CLSFLOWER, &cls_flower, 201940119211SVlad Buslov cb_priv, &f->flags, 202040119211SVlad Buslov &f->in_hw_count); 20215a6ff4b1SVlad Buslov tc_cleanup_flow_action(&cls_flower.rule->action); 20228f256622SPablo Neira Ayuso kfree(cls_flower.rule); 20238f256622SPablo Neira Ayuso 202431533cbaSJohn Hurley if (err) { 202595e27a4dSJohn Hurley __fl_put(f); 202631533cbaSJohn Hurley return err; 202795e27a4dSJohn Hurley } 202895e27a4dSJohn Hurley next_flow: 202995e27a4dSJohn Hurley __fl_put(f); 203031533cbaSJohn Hurley } 203131533cbaSJohn Hurley 203231533cbaSJohn Hurley return 0; 203331533cbaSJohn Hurley } 203431533cbaSJohn Hurley 2035a449a3e7SVlad Buslov static void fl_hw_add(struct tcf_proto *tp, void *type_data) 2036a449a3e7SVlad Buslov { 2037a449a3e7SVlad Buslov struct flow_cls_offload *cls_flower = type_data; 2038a449a3e7SVlad Buslov struct cls_fl_filter *f = 2039a449a3e7SVlad Buslov (struct cls_fl_filter *) cls_flower->cookie; 2040a449a3e7SVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 2041a449a3e7SVlad Buslov 2042a449a3e7SVlad Buslov spin_lock(&tp->lock); 2043a449a3e7SVlad Buslov list_add(&f->hw_list, &head->hw_filters); 2044a449a3e7SVlad Buslov spin_unlock(&tp->lock); 2045a449a3e7SVlad Buslov } 2046a449a3e7SVlad Buslov 2047a449a3e7SVlad Buslov static void fl_hw_del(struct tcf_proto *tp, void *type_data) 2048a449a3e7SVlad Buslov { 2049a449a3e7SVlad Buslov struct flow_cls_offload *cls_flower = type_data; 2050a449a3e7SVlad Buslov struct cls_fl_filter *f = 2051a449a3e7SVlad Buslov (struct cls_fl_filter *) cls_flower->cookie; 2052a449a3e7SVlad Buslov 2053a449a3e7SVlad Buslov spin_lock(&tp->lock); 2054a449a3e7SVlad Buslov if (!list_empty(&f->hw_list)) 2055a449a3e7SVlad Buslov list_del_init(&f->hw_list); 2056a449a3e7SVlad Buslov spin_unlock(&tp->lock); 2057a449a3e7SVlad Buslov } 2058a449a3e7SVlad Buslov 20598f256622SPablo Neira Ayuso static int fl_hw_create_tmplt(struct tcf_chain *chain, 206034738452SJiri Pirko struct fl_flow_tmplt *tmplt) 206134738452SJiri Pirko { 2062f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 206334738452SJiri Pirko struct tcf_block *block = chain->block; 206434738452SJiri Pirko 2065e3ab786bSPablo Neira Ayuso cls_flower.rule = flow_rule_alloc(0); 20668f256622SPablo Neira Ayuso if (!cls_flower.rule) 20678f256622SPablo Neira Ayuso return -ENOMEM; 20688f256622SPablo Neira Ayuso 206934738452SJiri Pirko cls_flower.common.chain_index = chain->index; 2070f9e30088SPablo Neira Ayuso cls_flower.command = FLOW_CLS_TMPLT_CREATE; 207134738452SJiri Pirko cls_flower.cookie = (unsigned long) tmplt; 20728f256622SPablo Neira Ayuso cls_flower.rule->match.dissector = &tmplt->dissector; 20738f256622SPablo Neira Ayuso cls_flower.rule->match.mask = &tmplt->mask; 20748f256622SPablo Neira Ayuso cls_flower.rule->match.key = &tmplt->dummy_key; 207534738452SJiri Pirko 207634738452SJiri Pirko /* We don't care if driver (any of them) fails to handle this 207734738452SJiri Pirko * call. It serves just as a hint for it. 207834738452SJiri Pirko */ 207940119211SVlad Buslov tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true); 20808f256622SPablo Neira Ayuso kfree(cls_flower.rule); 20818f256622SPablo Neira Ayuso 20828f256622SPablo Neira Ayuso return 0; 208334738452SJiri Pirko } 208434738452SJiri Pirko 208534738452SJiri Pirko static void fl_hw_destroy_tmplt(struct tcf_chain *chain, 208634738452SJiri Pirko struct fl_flow_tmplt *tmplt) 208734738452SJiri Pirko { 2088f9e30088SPablo Neira Ayuso struct flow_cls_offload cls_flower = {}; 208934738452SJiri Pirko struct tcf_block *block = chain->block; 209034738452SJiri Pirko 209134738452SJiri Pirko cls_flower.common.chain_index = chain->index; 2092f9e30088SPablo Neira Ayuso cls_flower.command = FLOW_CLS_TMPLT_DESTROY; 209334738452SJiri Pirko cls_flower.cookie = (unsigned long) tmplt; 209434738452SJiri Pirko 209540119211SVlad Buslov tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true); 209634738452SJiri Pirko } 209734738452SJiri Pirko 2098b95ec7ebSJiri Pirko static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain, 2099b95ec7ebSJiri Pirko struct nlattr **tca, 2100b95ec7ebSJiri Pirko struct netlink_ext_ack *extack) 2101b95ec7ebSJiri Pirko { 2102b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt; 2103b95ec7ebSJiri Pirko struct nlattr **tb; 2104b95ec7ebSJiri Pirko int err; 2105b95ec7ebSJiri Pirko 2106b95ec7ebSJiri Pirko if (!tca[TCA_OPTIONS]) 2107b95ec7ebSJiri Pirko return ERR_PTR(-EINVAL); 2108b95ec7ebSJiri Pirko 2109b95ec7ebSJiri Pirko tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 2110b95ec7ebSJiri Pirko if (!tb) 2111b95ec7ebSJiri Pirko return ERR_PTR(-ENOBUFS); 21128cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 21138cb08174SJohannes Berg tca[TCA_OPTIONS], fl_policy, NULL); 2114b95ec7ebSJiri Pirko if (err) 2115b95ec7ebSJiri Pirko goto errout_tb; 2116b95ec7ebSJiri Pirko 2117b95ec7ebSJiri Pirko tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL); 21181cbc36a5SDan Carpenter if (!tmplt) { 21191cbc36a5SDan Carpenter err = -ENOMEM; 2120b95ec7ebSJiri Pirko goto errout_tb; 21211cbc36a5SDan Carpenter } 2122b95ec7ebSJiri Pirko tmplt->chain = chain; 2123b95ec7ebSJiri Pirko err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack); 2124b95ec7ebSJiri Pirko if (err) 2125b95ec7ebSJiri Pirko goto errout_tmplt; 2126b95ec7ebSJiri Pirko 2127b95ec7ebSJiri Pirko fl_init_dissector(&tmplt->dissector, &tmplt->mask); 2128b95ec7ebSJiri Pirko 21298f256622SPablo Neira Ayuso err = fl_hw_create_tmplt(chain, tmplt); 21308f256622SPablo Neira Ayuso if (err) 21318f256622SPablo Neira Ayuso goto errout_tmplt; 213234738452SJiri Pirko 21338f256622SPablo Neira Ayuso kfree(tb); 2134b95ec7ebSJiri Pirko return tmplt; 2135b95ec7ebSJiri Pirko 2136b95ec7ebSJiri Pirko errout_tmplt: 2137b95ec7ebSJiri Pirko kfree(tmplt); 2138b95ec7ebSJiri Pirko errout_tb: 2139b95ec7ebSJiri Pirko kfree(tb); 2140b95ec7ebSJiri Pirko return ERR_PTR(err); 2141b95ec7ebSJiri Pirko } 2142b95ec7ebSJiri Pirko 2143b95ec7ebSJiri Pirko static void fl_tmplt_destroy(void *tmplt_priv) 2144b95ec7ebSJiri Pirko { 2145b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt = tmplt_priv; 2146b95ec7ebSJiri Pirko 214795278ddaSCong Wang fl_hw_destroy_tmplt(tmplt->chain, tmplt); 214895278ddaSCong Wang kfree(tmplt); 2149b95ec7ebSJiri Pirko } 2150b95ec7ebSJiri Pirko 215177b9900eSJiri Pirko static int fl_dump_key_val(struct sk_buff *skb, 215277b9900eSJiri Pirko void *val, int val_type, 215377b9900eSJiri Pirko void *mask, int mask_type, int len) 215477b9900eSJiri Pirko { 215577b9900eSJiri Pirko int err; 215677b9900eSJiri Pirko 215777b9900eSJiri Pirko if (!memchr_inv(mask, 0, len)) 215877b9900eSJiri Pirko return 0; 215977b9900eSJiri Pirko err = nla_put(skb, val_type, len, val); 216077b9900eSJiri Pirko if (err) 216177b9900eSJiri Pirko return err; 216277b9900eSJiri Pirko if (mask_type != TCA_FLOWER_UNSPEC) { 216377b9900eSJiri Pirko err = nla_put(skb, mask_type, len, mask); 216477b9900eSJiri Pirko if (err) 216577b9900eSJiri Pirko return err; 216677b9900eSJiri Pirko } 216777b9900eSJiri Pirko return 0; 216877b9900eSJiri Pirko } 216977b9900eSJiri Pirko 21705c72299fSAmritha Nambiar static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key, 21715c72299fSAmritha Nambiar struct fl_flow_key *mask) 21725c72299fSAmritha Nambiar { 21738ffb055bSYoshiki Komachi if (fl_dump_key_val(skb, &key->tp_range.tp_min.dst, 21748ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_DST_MIN, 21758ffb055bSYoshiki Komachi &mask->tp_range.tp_min.dst, TCA_FLOWER_UNSPEC, 21768ffb055bSYoshiki Komachi sizeof(key->tp_range.tp_min.dst)) || 21778ffb055bSYoshiki Komachi fl_dump_key_val(skb, &key->tp_range.tp_max.dst, 21788ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_DST_MAX, 21798ffb055bSYoshiki Komachi &mask->tp_range.tp_max.dst, TCA_FLOWER_UNSPEC, 21808ffb055bSYoshiki Komachi sizeof(key->tp_range.tp_max.dst)) || 21818ffb055bSYoshiki Komachi fl_dump_key_val(skb, &key->tp_range.tp_min.src, 21828ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_SRC_MIN, 21838ffb055bSYoshiki Komachi &mask->tp_range.tp_min.src, TCA_FLOWER_UNSPEC, 21848ffb055bSYoshiki Komachi sizeof(key->tp_range.tp_min.src)) || 21858ffb055bSYoshiki Komachi fl_dump_key_val(skb, &key->tp_range.tp_max.src, 21868ffb055bSYoshiki Komachi TCA_FLOWER_KEY_PORT_SRC_MAX, 21878ffb055bSYoshiki Komachi &mask->tp_range.tp_max.src, TCA_FLOWER_UNSPEC, 21888ffb055bSYoshiki Komachi sizeof(key->tp_range.tp_max.src))) 21895c72299fSAmritha Nambiar return -1; 21905c72299fSAmritha Nambiar 21915c72299fSAmritha Nambiar return 0; 21925c72299fSAmritha Nambiar } 21935c72299fSAmritha Nambiar 2194a577d8f7SBenjamin LaHaise static int fl_dump_key_mpls(struct sk_buff *skb, 2195a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *mpls_key, 2196a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *mpls_mask) 2197a577d8f7SBenjamin LaHaise { 2198a577d8f7SBenjamin LaHaise int err; 2199a577d8f7SBenjamin LaHaise 2200a577d8f7SBenjamin LaHaise if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask))) 2201a577d8f7SBenjamin LaHaise return 0; 2202a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_ttl) { 2203a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL, 2204a577d8f7SBenjamin LaHaise mpls_key->mpls_ttl); 2205a577d8f7SBenjamin LaHaise if (err) 2206a577d8f7SBenjamin LaHaise return err; 2207a577d8f7SBenjamin LaHaise } 2208a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_tc) { 2209a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC, 2210a577d8f7SBenjamin LaHaise mpls_key->mpls_tc); 2211a577d8f7SBenjamin LaHaise if (err) 2212a577d8f7SBenjamin LaHaise return err; 2213a577d8f7SBenjamin LaHaise } 2214a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_label) { 2215a577d8f7SBenjamin LaHaise err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL, 2216a577d8f7SBenjamin LaHaise mpls_key->mpls_label); 2217a577d8f7SBenjamin LaHaise if (err) 2218a577d8f7SBenjamin LaHaise return err; 2219a577d8f7SBenjamin LaHaise } 2220a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_bos) { 2221a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS, 2222a577d8f7SBenjamin LaHaise mpls_key->mpls_bos); 2223a577d8f7SBenjamin LaHaise if (err) 2224a577d8f7SBenjamin LaHaise return err; 2225a577d8f7SBenjamin LaHaise } 2226a577d8f7SBenjamin LaHaise return 0; 2227a577d8f7SBenjamin LaHaise } 2228a577d8f7SBenjamin LaHaise 22290e2c17b6SOr Gerlitz static int fl_dump_key_ip(struct sk_buff *skb, bool encap, 22304d80cc0aSOr Gerlitz struct flow_dissector_key_ip *key, 22314d80cc0aSOr Gerlitz struct flow_dissector_key_ip *mask) 22324d80cc0aSOr Gerlitz { 22330e2c17b6SOr Gerlitz int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 22340e2c17b6SOr Gerlitz int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 22350e2c17b6SOr Gerlitz int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 22360e2c17b6SOr Gerlitz int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 22370e2c17b6SOr Gerlitz 22380e2c17b6SOr Gerlitz if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) || 22390e2c17b6SOr Gerlitz fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl))) 22404d80cc0aSOr Gerlitz return -1; 22414d80cc0aSOr Gerlitz 22424d80cc0aSOr Gerlitz return 0; 22434d80cc0aSOr Gerlitz } 22444d80cc0aSOr Gerlitz 22459399ae9aSHadar Hen Zion static int fl_dump_key_vlan(struct sk_buff *skb, 2246d64efd09SJianbo Liu int vlan_id_key, int vlan_prio_key, 22479399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *vlan_key, 22489399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *vlan_mask) 22499399ae9aSHadar Hen Zion { 22509399ae9aSHadar Hen Zion int err; 22519399ae9aSHadar Hen Zion 22529399ae9aSHadar Hen Zion if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 22539399ae9aSHadar Hen Zion return 0; 22549399ae9aSHadar Hen Zion if (vlan_mask->vlan_id) { 2255d64efd09SJianbo Liu err = nla_put_u16(skb, vlan_id_key, 22569399ae9aSHadar Hen Zion vlan_key->vlan_id); 22579399ae9aSHadar Hen Zion if (err) 22589399ae9aSHadar Hen Zion return err; 22599399ae9aSHadar Hen Zion } 22609399ae9aSHadar Hen Zion if (vlan_mask->vlan_priority) { 2261d64efd09SJianbo Liu err = nla_put_u8(skb, vlan_prio_key, 22629399ae9aSHadar Hen Zion vlan_key->vlan_priority); 22639399ae9aSHadar Hen Zion if (err) 22649399ae9aSHadar Hen Zion return err; 22659399ae9aSHadar Hen Zion } 22669399ae9aSHadar Hen Zion return 0; 22679399ae9aSHadar Hen Zion } 22689399ae9aSHadar Hen Zion 2269faa3ffceSOr Gerlitz static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask, 2270faa3ffceSOr Gerlitz u32 *flower_key, u32 *flower_mask, 2271faa3ffceSOr Gerlitz u32 flower_flag_bit, u32 dissector_flag_bit) 2272faa3ffceSOr Gerlitz { 2273faa3ffceSOr Gerlitz if (dissector_mask & dissector_flag_bit) { 2274faa3ffceSOr Gerlitz *flower_mask |= flower_flag_bit; 2275faa3ffceSOr Gerlitz if (dissector_key & dissector_flag_bit) 2276faa3ffceSOr Gerlitz *flower_key |= flower_flag_bit; 2277faa3ffceSOr Gerlitz } 2278faa3ffceSOr Gerlitz } 2279faa3ffceSOr Gerlitz 2280faa3ffceSOr Gerlitz static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask) 2281faa3ffceSOr Gerlitz { 2282faa3ffceSOr Gerlitz u32 key, mask; 2283faa3ffceSOr Gerlitz __be32 _key, _mask; 2284faa3ffceSOr Gerlitz int err; 2285faa3ffceSOr Gerlitz 2286faa3ffceSOr Gerlitz if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask))) 2287faa3ffceSOr Gerlitz return 0; 2288faa3ffceSOr Gerlitz 2289faa3ffceSOr Gerlitz key = 0; 2290faa3ffceSOr Gerlitz mask = 0; 2291faa3ffceSOr Gerlitz 2292faa3ffceSOr Gerlitz fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2293faa3ffceSOr Gerlitz TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 2294459d153dSPieter Jansen van Vuuren fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2295459d153dSPieter Jansen van Vuuren TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 2296459d153dSPieter Jansen van Vuuren FLOW_DIS_FIRST_FRAG); 2297faa3ffceSOr Gerlitz 2298faa3ffceSOr Gerlitz _key = cpu_to_be32(key); 2299faa3ffceSOr Gerlitz _mask = cpu_to_be32(mask); 2300faa3ffceSOr Gerlitz 2301faa3ffceSOr Gerlitz err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key); 2302faa3ffceSOr Gerlitz if (err) 2303faa3ffceSOr Gerlitz return err; 2304faa3ffceSOr Gerlitz 2305faa3ffceSOr Gerlitz return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask); 2306faa3ffceSOr Gerlitz } 2307faa3ffceSOr Gerlitz 23080a6e7778SPieter Jansen van Vuuren static int fl_dump_key_geneve_opt(struct sk_buff *skb, 23090a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *enc_opts) 23100a6e7778SPieter Jansen van Vuuren { 23110a6e7778SPieter Jansen van Vuuren struct geneve_opt *opt; 23120a6e7778SPieter Jansen van Vuuren struct nlattr *nest; 23130a6e7778SPieter Jansen van Vuuren int opt_off = 0; 23140a6e7778SPieter Jansen van Vuuren 2315ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE); 23160a6e7778SPieter Jansen van Vuuren if (!nest) 23170a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 23180a6e7778SPieter Jansen van Vuuren 23190a6e7778SPieter Jansen van Vuuren while (enc_opts->len > opt_off) { 23200a6e7778SPieter Jansen van Vuuren opt = (struct geneve_opt *)&enc_opts->data[opt_off]; 23210a6e7778SPieter Jansen van Vuuren 23220a6e7778SPieter Jansen van Vuuren if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS, 23230a6e7778SPieter Jansen van Vuuren opt->opt_class)) 23240a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 23250a6e7778SPieter Jansen van Vuuren if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE, 23260a6e7778SPieter Jansen van Vuuren opt->type)) 23270a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 23280a6e7778SPieter Jansen van Vuuren if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA, 23290a6e7778SPieter Jansen van Vuuren opt->length * 4, opt->opt_data)) 23300a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 23310a6e7778SPieter Jansen van Vuuren 23320a6e7778SPieter Jansen van Vuuren opt_off += sizeof(struct geneve_opt) + opt->length * 4; 23330a6e7778SPieter Jansen van Vuuren } 23340a6e7778SPieter Jansen van Vuuren nla_nest_end(skb, nest); 23350a6e7778SPieter Jansen van Vuuren return 0; 23360a6e7778SPieter Jansen van Vuuren 23370a6e7778SPieter Jansen van Vuuren nla_put_failure: 23380a6e7778SPieter Jansen van Vuuren nla_nest_cancel(skb, nest); 23390a6e7778SPieter Jansen van Vuuren return -EMSGSIZE; 23400a6e7778SPieter Jansen van Vuuren } 23410a6e7778SPieter Jansen van Vuuren 2342d8f9dfaeSXin Long static int fl_dump_key_vxlan_opt(struct sk_buff *skb, 2343d8f9dfaeSXin Long struct flow_dissector_key_enc_opts *enc_opts) 2344d8f9dfaeSXin Long { 2345d8f9dfaeSXin Long struct vxlan_metadata *md; 2346d8f9dfaeSXin Long struct nlattr *nest; 2347d8f9dfaeSXin Long 2348d8f9dfaeSXin Long nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_VXLAN); 2349d8f9dfaeSXin Long if (!nest) 2350d8f9dfaeSXin Long goto nla_put_failure; 2351d8f9dfaeSXin Long 2352d8f9dfaeSXin Long md = (struct vxlan_metadata *)&enc_opts->data[0]; 2353d8f9dfaeSXin Long if (nla_put_u32(skb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP, md->gbp)) 2354d8f9dfaeSXin Long goto nla_put_failure; 2355d8f9dfaeSXin Long 2356d8f9dfaeSXin Long nla_nest_end(skb, nest); 2357d8f9dfaeSXin Long return 0; 2358d8f9dfaeSXin Long 2359d8f9dfaeSXin Long nla_put_failure: 2360d8f9dfaeSXin Long nla_nest_cancel(skb, nest); 2361d8f9dfaeSXin Long return -EMSGSIZE; 2362d8f9dfaeSXin Long } 2363d8f9dfaeSXin Long 236479b1011cSXin Long static int fl_dump_key_erspan_opt(struct sk_buff *skb, 236579b1011cSXin Long struct flow_dissector_key_enc_opts *enc_opts) 236679b1011cSXin Long { 236779b1011cSXin Long struct erspan_metadata *md; 236879b1011cSXin Long struct nlattr *nest; 236979b1011cSXin Long 237079b1011cSXin Long nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_ERSPAN); 237179b1011cSXin Long if (!nest) 237279b1011cSXin Long goto nla_put_failure; 237379b1011cSXin Long 237479b1011cSXin Long md = (struct erspan_metadata *)&enc_opts->data[0]; 237579b1011cSXin Long if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER, md->version)) 237679b1011cSXin Long goto nla_put_failure; 237779b1011cSXin Long 237879b1011cSXin Long if (md->version == 1 && 237979b1011cSXin Long nla_put_be32(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX, md->u.index)) 238079b1011cSXin Long goto nla_put_failure; 238179b1011cSXin Long 238279b1011cSXin Long if (md->version == 2 && 238379b1011cSXin Long (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR, 238479b1011cSXin Long md->u.md2.dir) || 238579b1011cSXin Long nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID, 238679b1011cSXin Long get_hwid(&md->u.md2)))) 238779b1011cSXin Long goto nla_put_failure; 238879b1011cSXin Long 238979b1011cSXin Long nla_nest_end(skb, nest); 239079b1011cSXin Long return 0; 239179b1011cSXin Long 239279b1011cSXin Long nla_put_failure: 239379b1011cSXin Long nla_nest_cancel(skb, nest); 239479b1011cSXin Long return -EMSGSIZE; 239579b1011cSXin Long } 239679b1011cSXin Long 2397e0ace68aSPaul Blakey static int fl_dump_key_ct(struct sk_buff *skb, 2398e0ace68aSPaul Blakey struct flow_dissector_key_ct *key, 2399e0ace68aSPaul Blakey struct flow_dissector_key_ct *mask) 2400e0ace68aSPaul Blakey { 2401e0ace68aSPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK) && 2402e0ace68aSPaul Blakey fl_dump_key_val(skb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE, 2403e0ace68aSPaul Blakey &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK, 2404e0ace68aSPaul Blakey sizeof(key->ct_state))) 2405e0ace68aSPaul Blakey goto nla_put_failure; 2406e0ace68aSPaul Blakey 2407e0ace68aSPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) && 2408e0ace68aSPaul Blakey fl_dump_key_val(skb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE, 2409e0ace68aSPaul Blakey &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK, 2410e0ace68aSPaul Blakey sizeof(key->ct_zone))) 2411e0ace68aSPaul Blakey goto nla_put_failure; 2412e0ace68aSPaul Blakey 2413e0ace68aSPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) && 2414e0ace68aSPaul Blakey fl_dump_key_val(skb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK, 2415e0ace68aSPaul Blakey &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK, 2416e0ace68aSPaul Blakey sizeof(key->ct_mark))) 2417e0ace68aSPaul Blakey goto nla_put_failure; 2418e0ace68aSPaul Blakey 2419e0ace68aSPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) && 2420e0ace68aSPaul Blakey fl_dump_key_val(skb, &key->ct_labels, TCA_FLOWER_KEY_CT_LABELS, 2421e0ace68aSPaul Blakey &mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK, 2422e0ace68aSPaul Blakey sizeof(key->ct_labels))) 2423e0ace68aSPaul Blakey goto nla_put_failure; 2424e0ace68aSPaul Blakey 2425e0ace68aSPaul Blakey return 0; 2426e0ace68aSPaul Blakey 2427e0ace68aSPaul Blakey nla_put_failure: 2428e0ace68aSPaul Blakey return -EMSGSIZE; 2429e0ace68aSPaul Blakey } 2430e0ace68aSPaul Blakey 24310a6e7778SPieter Jansen van Vuuren static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type, 24320a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *enc_opts) 24330a6e7778SPieter Jansen van Vuuren { 24340a6e7778SPieter Jansen van Vuuren struct nlattr *nest; 24350a6e7778SPieter Jansen van Vuuren int err; 24360a6e7778SPieter Jansen van Vuuren 24370a6e7778SPieter Jansen van Vuuren if (!enc_opts->len) 24380a6e7778SPieter Jansen van Vuuren return 0; 24390a6e7778SPieter Jansen van Vuuren 2440ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, enc_opt_type); 24410a6e7778SPieter Jansen van Vuuren if (!nest) 24420a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 24430a6e7778SPieter Jansen van Vuuren 24440a6e7778SPieter Jansen van Vuuren switch (enc_opts->dst_opt_type) { 24450a6e7778SPieter Jansen van Vuuren case TUNNEL_GENEVE_OPT: 24460a6e7778SPieter Jansen van Vuuren err = fl_dump_key_geneve_opt(skb, enc_opts); 24470a6e7778SPieter Jansen van Vuuren if (err) 24480a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 24490a6e7778SPieter Jansen van Vuuren break; 2450d8f9dfaeSXin Long case TUNNEL_VXLAN_OPT: 2451d8f9dfaeSXin Long err = fl_dump_key_vxlan_opt(skb, enc_opts); 2452d8f9dfaeSXin Long if (err) 2453d8f9dfaeSXin Long goto nla_put_failure; 2454d8f9dfaeSXin Long break; 245579b1011cSXin Long case TUNNEL_ERSPAN_OPT: 245679b1011cSXin Long err = fl_dump_key_erspan_opt(skb, enc_opts); 245779b1011cSXin Long if (err) 245879b1011cSXin Long goto nla_put_failure; 245979b1011cSXin Long break; 24600a6e7778SPieter Jansen van Vuuren default: 24610a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 24620a6e7778SPieter Jansen van Vuuren } 24630a6e7778SPieter Jansen van Vuuren nla_nest_end(skb, nest); 24640a6e7778SPieter Jansen van Vuuren return 0; 24650a6e7778SPieter Jansen van Vuuren 24660a6e7778SPieter Jansen van Vuuren nla_put_failure: 24670a6e7778SPieter Jansen van Vuuren nla_nest_cancel(skb, nest); 24680a6e7778SPieter Jansen van Vuuren return -EMSGSIZE; 24690a6e7778SPieter Jansen van Vuuren } 24700a6e7778SPieter Jansen van Vuuren 24710a6e7778SPieter Jansen van Vuuren static int fl_dump_key_enc_opt(struct sk_buff *skb, 24720a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *key_opts, 24730a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *msk_opts) 24740a6e7778SPieter Jansen van Vuuren { 24750a6e7778SPieter Jansen van Vuuren int err; 24760a6e7778SPieter Jansen van Vuuren 24770a6e7778SPieter Jansen van Vuuren err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts); 24780a6e7778SPieter Jansen van Vuuren if (err) 24790a6e7778SPieter Jansen van Vuuren return err; 24800a6e7778SPieter Jansen van Vuuren 24810a6e7778SPieter Jansen van Vuuren return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts); 24820a6e7778SPieter Jansen van Vuuren } 24830a6e7778SPieter Jansen van Vuuren 2484f5749081SJiri Pirko static int fl_dump_key(struct sk_buff *skb, struct net *net, 2485f5749081SJiri Pirko struct fl_flow_key *key, struct fl_flow_key *mask) 248677b9900eSJiri Pirko { 24878212ed77SJiri Pirko if (mask->meta.ingress_ifindex) { 248877b9900eSJiri Pirko struct net_device *dev; 248977b9900eSJiri Pirko 24908212ed77SJiri Pirko dev = __dev_get_by_index(net, key->meta.ingress_ifindex); 249177b9900eSJiri Pirko if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 249277b9900eSJiri Pirko goto nla_put_failure; 249377b9900eSJiri Pirko } 249477b9900eSJiri Pirko 249577b9900eSJiri Pirko if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 249677b9900eSJiri Pirko mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 249777b9900eSJiri Pirko sizeof(key->eth.dst)) || 249877b9900eSJiri Pirko fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 249977b9900eSJiri Pirko mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 250077b9900eSJiri Pirko sizeof(key->eth.src)) || 250177b9900eSJiri Pirko fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 250277b9900eSJiri Pirko &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 250377b9900eSJiri Pirko sizeof(key->basic.n_proto))) 250477b9900eSJiri Pirko goto nla_put_failure; 25059399ae9aSHadar Hen Zion 2506a577d8f7SBenjamin LaHaise if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls)) 2507a577d8f7SBenjamin LaHaise goto nla_put_failure; 2508a577d8f7SBenjamin LaHaise 2509d64efd09SJianbo Liu if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID, 2510d64efd09SJianbo Liu TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan)) 25119399ae9aSHadar Hen Zion goto nla_put_failure; 25129399ae9aSHadar Hen Zion 2513d64efd09SJianbo Liu if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID, 2514d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_PRIO, 2515d64efd09SJianbo Liu &key->cvlan, &mask->cvlan) || 2516d64efd09SJianbo Liu (mask->cvlan.vlan_tpid && 2517158abbf1SJianbo Liu nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2518d64efd09SJianbo Liu key->cvlan.vlan_tpid))) 2519d3069512SJianbo Liu goto nla_put_failure; 2520d3069512SJianbo Liu 25215e9a0fe4SJianbo Liu if (mask->basic.n_proto) { 2522d64efd09SJianbo Liu if (mask->cvlan.vlan_tpid) { 2523d64efd09SJianbo Liu if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 2524d64efd09SJianbo Liu key->basic.n_proto)) 2525d64efd09SJianbo Liu goto nla_put_failure; 2526d64efd09SJianbo Liu } else if (mask->vlan.vlan_tpid) { 2527d64efd09SJianbo Liu if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2528d64efd09SJianbo Liu key->basic.n_proto)) 2529d64efd09SJianbo Liu goto nla_put_failure; 2530d64efd09SJianbo Liu } 25315e9a0fe4SJianbo Liu } 2532d64efd09SJianbo Liu 253377b9900eSJiri Pirko if ((key->basic.n_proto == htons(ETH_P_IP) || 253477b9900eSJiri Pirko key->basic.n_proto == htons(ETH_P_IPV6)) && 25354d80cc0aSOr Gerlitz (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 253677b9900eSJiri Pirko &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 25374d80cc0aSOr Gerlitz sizeof(key->basic.ip_proto)) || 25380e2c17b6SOr Gerlitz fl_dump_key_ip(skb, false, &key->ip, &mask->ip))) 253977b9900eSJiri Pirko goto nla_put_failure; 254077b9900eSJiri Pirko 2541c3f83241STom Herbert if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 254277b9900eSJiri Pirko (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 254377b9900eSJiri Pirko &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 254477b9900eSJiri Pirko sizeof(key->ipv4.src)) || 254577b9900eSJiri Pirko fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 254677b9900eSJiri Pirko &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 254777b9900eSJiri Pirko sizeof(key->ipv4.dst)))) 254877b9900eSJiri Pirko goto nla_put_failure; 2549c3f83241STom Herbert else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 255077b9900eSJiri Pirko (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 255177b9900eSJiri Pirko &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 255277b9900eSJiri Pirko sizeof(key->ipv6.src)) || 255377b9900eSJiri Pirko fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 255477b9900eSJiri Pirko &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 255577b9900eSJiri Pirko sizeof(key->ipv6.dst)))) 255677b9900eSJiri Pirko goto nla_put_failure; 255777b9900eSJiri Pirko 255877b9900eSJiri Pirko if (key->basic.ip_proto == IPPROTO_TCP && 255977b9900eSJiri Pirko (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 2560aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 256177b9900eSJiri Pirko sizeof(key->tp.src)) || 256277b9900eSJiri Pirko fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 2563aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 2564fdfc7dd6SJiri Pirko sizeof(key->tp.dst)) || 2565fdfc7dd6SJiri Pirko fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 2566fdfc7dd6SJiri Pirko &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 2567fdfc7dd6SJiri Pirko sizeof(key->tcp.flags)))) 256877b9900eSJiri Pirko goto nla_put_failure; 256977b9900eSJiri Pirko else if (key->basic.ip_proto == IPPROTO_UDP && 257077b9900eSJiri Pirko (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 2571aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 257277b9900eSJiri Pirko sizeof(key->tp.src)) || 257377b9900eSJiri Pirko fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 2574aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 257577b9900eSJiri Pirko sizeof(key->tp.dst)))) 257677b9900eSJiri Pirko goto nla_put_failure; 25775976c5f4SSimon Horman else if (key->basic.ip_proto == IPPROTO_SCTP && 25785976c5f4SSimon Horman (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 25795976c5f4SSimon Horman &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 25805976c5f4SSimon Horman sizeof(key->tp.src)) || 25815976c5f4SSimon Horman fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 25825976c5f4SSimon Horman &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 25835976c5f4SSimon Horman sizeof(key->tp.dst)))) 25845976c5f4SSimon Horman goto nla_put_failure; 25857b684884SSimon Horman else if (key->basic.n_proto == htons(ETH_P_IP) && 25867b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMP && 25877b684884SSimon Horman (fl_dump_key_val(skb, &key->icmp.type, 25887b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type, 25897b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 25907b684884SSimon Horman sizeof(key->icmp.type)) || 25917b684884SSimon Horman fl_dump_key_val(skb, &key->icmp.code, 25927b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code, 25937b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 25947b684884SSimon Horman sizeof(key->icmp.code)))) 25957b684884SSimon Horman goto nla_put_failure; 25967b684884SSimon Horman else if (key->basic.n_proto == htons(ETH_P_IPV6) && 25977b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMPV6 && 25987b684884SSimon Horman (fl_dump_key_val(skb, &key->icmp.type, 25997b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type, 26007b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 26017b684884SSimon Horman sizeof(key->icmp.type)) || 26027b684884SSimon Horman fl_dump_key_val(skb, &key->icmp.code, 26037b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code, 26047b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 26057b684884SSimon Horman sizeof(key->icmp.code)))) 26067b684884SSimon Horman goto nla_put_failure; 260799d31326SSimon Horman else if ((key->basic.n_proto == htons(ETH_P_ARP) || 260899d31326SSimon Horman key->basic.n_proto == htons(ETH_P_RARP)) && 260999d31326SSimon Horman (fl_dump_key_val(skb, &key->arp.sip, 261099d31326SSimon Horman TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip, 261199d31326SSimon Horman TCA_FLOWER_KEY_ARP_SIP_MASK, 261299d31326SSimon Horman sizeof(key->arp.sip)) || 261399d31326SSimon Horman fl_dump_key_val(skb, &key->arp.tip, 261499d31326SSimon Horman TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip, 261599d31326SSimon Horman TCA_FLOWER_KEY_ARP_TIP_MASK, 261699d31326SSimon Horman sizeof(key->arp.tip)) || 261799d31326SSimon Horman fl_dump_key_val(skb, &key->arp.op, 261899d31326SSimon Horman TCA_FLOWER_KEY_ARP_OP, &mask->arp.op, 261999d31326SSimon Horman TCA_FLOWER_KEY_ARP_OP_MASK, 262099d31326SSimon Horman sizeof(key->arp.op)) || 262199d31326SSimon Horman fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 262299d31326SSimon Horman mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 262399d31326SSimon Horman sizeof(key->arp.sha)) || 262499d31326SSimon Horman fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 262599d31326SSimon Horman mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 262699d31326SSimon Horman sizeof(key->arp.tha)))) 262799d31326SSimon Horman goto nla_put_failure; 262877b9900eSJiri Pirko 26295c72299fSAmritha Nambiar if ((key->basic.ip_proto == IPPROTO_TCP || 26305c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_UDP || 26315c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_SCTP) && 26325c72299fSAmritha Nambiar fl_dump_key_port_range(skb, key, mask)) 26335c72299fSAmritha Nambiar goto nla_put_failure; 26345c72299fSAmritha Nambiar 2635bc3103f1SAmir Vadai if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2636bc3103f1SAmir Vadai (fl_dump_key_val(skb, &key->enc_ipv4.src, 2637bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 2638bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 2639bc3103f1SAmir Vadai sizeof(key->enc_ipv4.src)) || 2640bc3103f1SAmir Vadai fl_dump_key_val(skb, &key->enc_ipv4.dst, 2641bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 2642bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 2643bc3103f1SAmir Vadai sizeof(key->enc_ipv4.dst)))) 2644bc3103f1SAmir Vadai goto nla_put_failure; 2645bc3103f1SAmir Vadai else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2646bc3103f1SAmir Vadai (fl_dump_key_val(skb, &key->enc_ipv6.src, 2647bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 2648bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 2649bc3103f1SAmir Vadai sizeof(key->enc_ipv6.src)) || 2650bc3103f1SAmir Vadai fl_dump_key_val(skb, &key->enc_ipv6.dst, 2651bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST, 2652bc3103f1SAmir Vadai &mask->enc_ipv6.dst, 2653bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 2654bc3103f1SAmir Vadai sizeof(key->enc_ipv6.dst)))) 2655bc3103f1SAmir Vadai goto nla_put_failure; 2656bc3103f1SAmir Vadai 2657bc3103f1SAmir Vadai if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 2658eb523f42SHadar Hen Zion &mask->enc_key_id, TCA_FLOWER_UNSPEC, 2659f4d997fdSHadar Hen Zion sizeof(key->enc_key_id)) || 2660f4d997fdSHadar Hen Zion fl_dump_key_val(skb, &key->enc_tp.src, 2661f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 2662f4d997fdSHadar Hen Zion &mask->enc_tp.src, 2663f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 2664f4d997fdSHadar Hen Zion sizeof(key->enc_tp.src)) || 2665f4d997fdSHadar Hen Zion fl_dump_key_val(skb, &key->enc_tp.dst, 2666f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 2667f4d997fdSHadar Hen Zion &mask->enc_tp.dst, 2668f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 26690e2c17b6SOr Gerlitz sizeof(key->enc_tp.dst)) || 26700a6e7778SPieter Jansen van Vuuren fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) || 26710a6e7778SPieter Jansen van Vuuren fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts)) 2672bc3103f1SAmir Vadai goto nla_put_failure; 2673bc3103f1SAmir Vadai 2674e0ace68aSPaul Blakey if (fl_dump_key_ct(skb, &key->ct, &mask->ct)) 2675e0ace68aSPaul Blakey goto nla_put_failure; 2676e0ace68aSPaul Blakey 2677faa3ffceSOr Gerlitz if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags)) 2678faa3ffceSOr Gerlitz goto nla_put_failure; 2679faa3ffceSOr Gerlitz 2680f5749081SJiri Pirko return 0; 2681f5749081SJiri Pirko 2682f5749081SJiri Pirko nla_put_failure: 2683f5749081SJiri Pirko return -EMSGSIZE; 2684f5749081SJiri Pirko } 2685f5749081SJiri Pirko 2686f5749081SJiri Pirko static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh, 268712db03b6SVlad Buslov struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 2688f5749081SJiri Pirko { 2689f5749081SJiri Pirko struct cls_fl_filter *f = fh; 2690f5749081SJiri Pirko struct nlattr *nest; 2691f5749081SJiri Pirko struct fl_flow_key *key, *mask; 26923d81e711SVlad Buslov bool skip_hw; 2693f5749081SJiri Pirko 2694f5749081SJiri Pirko if (!f) 2695f5749081SJiri Pirko return skb->len; 2696f5749081SJiri Pirko 2697f5749081SJiri Pirko t->tcm_handle = f->handle; 2698f5749081SJiri Pirko 2699ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 2700f5749081SJiri Pirko if (!nest) 2701f5749081SJiri Pirko goto nla_put_failure; 2702f5749081SJiri Pirko 27033d81e711SVlad Buslov spin_lock(&tp->lock); 27043d81e711SVlad Buslov 2705f5749081SJiri Pirko if (f->res.classid && 2706f5749081SJiri Pirko nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 27073d81e711SVlad Buslov goto nla_put_failure_locked; 2708f5749081SJiri Pirko 2709f5749081SJiri Pirko key = &f->key; 2710f5749081SJiri Pirko mask = &f->mask->key; 27113d81e711SVlad Buslov skip_hw = tc_skip_hw(f->flags); 2712f5749081SJiri Pirko 2713f5749081SJiri Pirko if (fl_dump_key(skb, net, key, mask)) 27143d81e711SVlad Buslov goto nla_put_failure_locked; 2715f5749081SJiri Pirko 2716749e6720SOr Gerlitz if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags)) 27173d81e711SVlad Buslov goto nla_put_failure_locked; 27183d81e711SVlad Buslov 27193d81e711SVlad Buslov spin_unlock(&tp->lock); 27203d81e711SVlad Buslov 27213d81e711SVlad Buslov if (!skip_hw) 2722c24e43d8SVlad Buslov fl_hw_update_stats(tp, f, rtnl_held); 2723e69985c6SAmir Vadai 272486c55361SVlad Buslov if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count)) 272586c55361SVlad Buslov goto nla_put_failure; 272686c55361SVlad Buslov 272777b9900eSJiri Pirko if (tcf_exts_dump(skb, &f->exts)) 272877b9900eSJiri Pirko goto nla_put_failure; 272977b9900eSJiri Pirko 273077b9900eSJiri Pirko nla_nest_end(skb, nest); 273177b9900eSJiri Pirko 273277b9900eSJiri Pirko if (tcf_exts_dump_stats(skb, &f->exts) < 0) 273377b9900eSJiri Pirko goto nla_put_failure; 273477b9900eSJiri Pirko 273577b9900eSJiri Pirko return skb->len; 273677b9900eSJiri Pirko 27373d81e711SVlad Buslov nla_put_failure_locked: 27383d81e711SVlad Buslov spin_unlock(&tp->lock); 273977b9900eSJiri Pirko nla_put_failure: 274077b9900eSJiri Pirko nla_nest_cancel(skb, nest); 274177b9900eSJiri Pirko return -1; 274277b9900eSJiri Pirko } 274377b9900eSJiri Pirko 2744b95ec7ebSJiri Pirko static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv) 2745b95ec7ebSJiri Pirko { 2746b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt = tmplt_priv; 2747b95ec7ebSJiri Pirko struct fl_flow_key *key, *mask; 2748b95ec7ebSJiri Pirko struct nlattr *nest; 2749b95ec7ebSJiri Pirko 2750ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 2751b95ec7ebSJiri Pirko if (!nest) 2752b95ec7ebSJiri Pirko goto nla_put_failure; 2753b95ec7ebSJiri Pirko 2754b95ec7ebSJiri Pirko key = &tmplt->dummy_key; 2755b95ec7ebSJiri Pirko mask = &tmplt->mask; 2756b95ec7ebSJiri Pirko 2757b95ec7ebSJiri Pirko if (fl_dump_key(skb, net, key, mask)) 2758b95ec7ebSJiri Pirko goto nla_put_failure; 2759b95ec7ebSJiri Pirko 2760b95ec7ebSJiri Pirko nla_nest_end(skb, nest); 2761b95ec7ebSJiri Pirko 2762b95ec7ebSJiri Pirko return skb->len; 2763b95ec7ebSJiri Pirko 2764b95ec7ebSJiri Pirko nla_put_failure: 2765b95ec7ebSJiri Pirko nla_nest_cancel(skb, nest); 2766b95ec7ebSJiri Pirko return -EMSGSIZE; 2767b95ec7ebSJiri Pirko } 2768b95ec7ebSJiri Pirko 27692e24cd75SCong Wang static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q, 27702e24cd75SCong Wang unsigned long base) 277107d79fc7SCong Wang { 277207d79fc7SCong Wang struct cls_fl_filter *f = fh; 277307d79fc7SCong Wang 27742e24cd75SCong Wang if (f && f->res.classid == classid) { 27752e24cd75SCong Wang if (cl) 27762e24cd75SCong Wang __tcf_bind_filter(q, &f->res, base); 27772e24cd75SCong Wang else 27782e24cd75SCong Wang __tcf_unbind_filter(q, &f->res); 27792e24cd75SCong Wang } 278007d79fc7SCong Wang } 278107d79fc7SCong Wang 2782a5b72a08SDavide Caratti static bool fl_delete_empty(struct tcf_proto *tp) 2783a5b72a08SDavide Caratti { 2784a5b72a08SDavide Caratti struct cls_fl_head *head = fl_head_dereference(tp); 2785a5b72a08SDavide Caratti 2786a5b72a08SDavide Caratti spin_lock(&tp->lock); 2787a5b72a08SDavide Caratti tp->deleting = idr_is_empty(&head->handle_idr); 2788a5b72a08SDavide Caratti spin_unlock(&tp->lock); 2789a5b72a08SDavide Caratti 2790a5b72a08SDavide Caratti return tp->deleting; 2791a5b72a08SDavide Caratti } 2792a5b72a08SDavide Caratti 279377b9900eSJiri Pirko static struct tcf_proto_ops cls_fl_ops __read_mostly = { 279477b9900eSJiri Pirko .kind = "flower", 279577b9900eSJiri Pirko .classify = fl_classify, 279677b9900eSJiri Pirko .init = fl_init, 279777b9900eSJiri Pirko .destroy = fl_destroy, 279877b9900eSJiri Pirko .get = fl_get, 279906177558SVlad Buslov .put = fl_put, 280077b9900eSJiri Pirko .change = fl_change, 280177b9900eSJiri Pirko .delete = fl_delete, 2802a5b72a08SDavide Caratti .delete_empty = fl_delete_empty, 280377b9900eSJiri Pirko .walk = fl_walk, 280431533cbaSJohn Hurley .reoffload = fl_reoffload, 2805a449a3e7SVlad Buslov .hw_add = fl_hw_add, 2806a449a3e7SVlad Buslov .hw_del = fl_hw_del, 280777b9900eSJiri Pirko .dump = fl_dump, 280807d79fc7SCong Wang .bind_class = fl_bind_class, 2809b95ec7ebSJiri Pirko .tmplt_create = fl_tmplt_create, 2810b95ec7ebSJiri Pirko .tmplt_destroy = fl_tmplt_destroy, 2811b95ec7ebSJiri Pirko .tmplt_dump = fl_tmplt_dump, 281277b9900eSJiri Pirko .owner = THIS_MODULE, 281392149190SVlad Buslov .flags = TCF_PROTO_OPS_DOIT_UNLOCKED, 281477b9900eSJiri Pirko }; 281577b9900eSJiri Pirko 281677b9900eSJiri Pirko static int __init cls_fl_init(void) 281777b9900eSJiri Pirko { 281877b9900eSJiri Pirko return register_tcf_proto_ops(&cls_fl_ops); 281977b9900eSJiri Pirko } 282077b9900eSJiri Pirko 282177b9900eSJiri Pirko static void __exit cls_fl_exit(void) 282277b9900eSJiri Pirko { 282377b9900eSJiri Pirko unregister_tcf_proto_ops(&cls_fl_ops); 282477b9900eSJiri Pirko } 282577b9900eSJiri Pirko 282677b9900eSJiri Pirko module_init(cls_fl_init); 282777b9900eSJiri Pirko module_exit(cls_fl_exit); 282877b9900eSJiri Pirko 282977b9900eSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 283077b9900eSJiri Pirko MODULE_DESCRIPTION("Flower classifier"); 283177b9900eSJiri Pirko MODULE_LICENSE("GPL v2"); 2832