12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 277b9900eSJiri Pirko /* 377b9900eSJiri Pirko * net/sched/cls_flower.c Flower classifier 477b9900eSJiri Pirko * 577b9900eSJiri Pirko * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 677b9900eSJiri Pirko */ 777b9900eSJiri Pirko 877b9900eSJiri Pirko #include <linux/kernel.h> 977b9900eSJiri Pirko #include <linux/init.h> 1077b9900eSJiri Pirko #include <linux/module.h> 1177b9900eSJiri Pirko #include <linux/rhashtable.h> 12d9363774SDaniel Borkmann #include <linux/workqueue.h> 1306177558SVlad Buslov #include <linux/refcount.h> 1477b9900eSJiri Pirko 1577b9900eSJiri Pirko #include <linux/if_ether.h> 1677b9900eSJiri Pirko #include <linux/in6.h> 1777b9900eSJiri Pirko #include <linux/ip.h> 18a577d8f7SBenjamin LaHaise #include <linux/mpls.h> 1977b9900eSJiri Pirko 2077b9900eSJiri Pirko #include <net/sch_generic.h> 2177b9900eSJiri Pirko #include <net/pkt_cls.h> 2277b9900eSJiri Pirko #include <net/ip.h> 2377b9900eSJiri Pirko #include <net/flow_dissector.h> 240a6e7778SPieter Jansen van Vuuren #include <net/geneve.h> 2577b9900eSJiri Pirko 26bc3103f1SAmir Vadai #include <net/dst.h> 27bc3103f1SAmir Vadai #include <net/dst_metadata.h> 28bc3103f1SAmir Vadai 2977b9900eSJiri Pirko struct fl_flow_key { 3077b9900eSJiri Pirko int indev_ifindex; 3142aecaa9STom Herbert struct flow_dissector_key_control control; 32bc3103f1SAmir Vadai struct flow_dissector_key_control enc_control; 3377b9900eSJiri Pirko struct flow_dissector_key_basic basic; 3477b9900eSJiri Pirko struct flow_dissector_key_eth_addrs eth; 359399ae9aSHadar Hen Zion struct flow_dissector_key_vlan vlan; 36d64efd09SJianbo Liu struct flow_dissector_key_vlan cvlan; 3777b9900eSJiri Pirko union { 38c3f83241STom Herbert struct flow_dissector_key_ipv4_addrs ipv4; 3977b9900eSJiri Pirko struct flow_dissector_key_ipv6_addrs ipv6; 4077b9900eSJiri Pirko }; 4177b9900eSJiri Pirko struct flow_dissector_key_ports tp; 427b684884SSimon Horman struct flow_dissector_key_icmp icmp; 4399d31326SSimon Horman struct flow_dissector_key_arp arp; 44bc3103f1SAmir Vadai struct flow_dissector_key_keyid enc_key_id; 45bc3103f1SAmir Vadai union { 46bc3103f1SAmir Vadai struct flow_dissector_key_ipv4_addrs enc_ipv4; 47bc3103f1SAmir Vadai struct flow_dissector_key_ipv6_addrs enc_ipv6; 48bc3103f1SAmir Vadai }; 49f4d997fdSHadar Hen Zion struct flow_dissector_key_ports enc_tp; 50a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls mpls; 51fdfc7dd6SJiri Pirko struct flow_dissector_key_tcp tcp; 524d80cc0aSOr Gerlitz struct flow_dissector_key_ip ip; 530e2c17b6SOr Gerlitz struct flow_dissector_key_ip enc_ip; 540a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts enc_opts; 555c72299fSAmritha Nambiar struct flow_dissector_key_ports tp_min; 565c72299fSAmritha Nambiar struct flow_dissector_key_ports tp_max; 5777b9900eSJiri Pirko } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 5877b9900eSJiri Pirko 5977b9900eSJiri Pirko struct fl_flow_mask_range { 6077b9900eSJiri Pirko unsigned short int start; 6177b9900eSJiri Pirko unsigned short int end; 6277b9900eSJiri Pirko }; 6377b9900eSJiri Pirko 6477b9900eSJiri Pirko struct fl_flow_mask { 6577b9900eSJiri Pirko struct fl_flow_key key; 6677b9900eSJiri Pirko struct fl_flow_mask_range range; 675c72299fSAmritha Nambiar u32 flags; 6805cd271fSPaul Blakey struct rhash_head ht_node; 6905cd271fSPaul Blakey struct rhashtable ht; 7005cd271fSPaul Blakey struct rhashtable_params filter_ht_params; 7105cd271fSPaul Blakey struct flow_dissector dissector; 7205cd271fSPaul Blakey struct list_head filters; 7344a5cd43SPaolo Abeni struct rcu_work rwork; 7405cd271fSPaul Blakey struct list_head list; 75f48ef4d5SVlad Buslov refcount_t refcnt; 7677b9900eSJiri Pirko }; 7777b9900eSJiri Pirko 78b95ec7ebSJiri Pirko struct fl_flow_tmplt { 79b95ec7ebSJiri Pirko struct fl_flow_key dummy_key; 80b95ec7ebSJiri Pirko struct fl_flow_key mask; 81b95ec7ebSJiri Pirko struct flow_dissector dissector; 82b95ec7ebSJiri Pirko struct tcf_chain *chain; 83b95ec7ebSJiri Pirko }; 84b95ec7ebSJiri Pirko 8577b9900eSJiri Pirko struct cls_fl_head { 8677b9900eSJiri Pirko struct rhashtable ht; 87259e60f9SVlad Buslov spinlock_t masks_lock; /* Protect masks list */ 8805cd271fSPaul Blakey struct list_head masks; 89c049d56eSVlad Buslov struct list_head hw_filters; 90aaa908ffSCong Wang struct rcu_work rwork; 91c15ab236SChris Mi struct idr handle_idr; 92d9363774SDaniel Borkmann }; 9377b9900eSJiri Pirko 9477b9900eSJiri Pirko struct cls_fl_filter { 9505cd271fSPaul Blakey struct fl_flow_mask *mask; 9677b9900eSJiri Pirko struct rhash_head ht_node; 9777b9900eSJiri Pirko struct fl_flow_key mkey; 9877b9900eSJiri Pirko struct tcf_exts exts; 9977b9900eSJiri Pirko struct tcf_result res; 10077b9900eSJiri Pirko struct fl_flow_key key; 10177b9900eSJiri Pirko struct list_head list; 102c049d56eSVlad Buslov struct list_head hw_list; 10377b9900eSJiri Pirko u32 handle; 104e69985c6SAmir Vadai u32 flags; 10586c55361SVlad Buslov u32 in_hw_count; 106aaa908ffSCong Wang struct rcu_work rwork; 1077091d8c7SHadar Hen Zion struct net_device *hw_dev; 10806177558SVlad Buslov /* Flower classifier is unlocked, which means that its reference counter 10906177558SVlad Buslov * can be changed concurrently without any kind of external 11006177558SVlad Buslov * synchronization. Use atomic reference counter to be concurrency-safe. 11106177558SVlad Buslov */ 11206177558SVlad Buslov refcount_t refcnt; 113b2552b8cSVlad Buslov bool deleted; 11477b9900eSJiri Pirko }; 11577b9900eSJiri Pirko 11605cd271fSPaul Blakey static const struct rhashtable_params mask_ht_params = { 11705cd271fSPaul Blakey .key_offset = offsetof(struct fl_flow_mask, key), 11805cd271fSPaul Blakey .key_len = sizeof(struct fl_flow_key), 11905cd271fSPaul Blakey .head_offset = offsetof(struct fl_flow_mask, ht_node), 12005cd271fSPaul Blakey .automatic_shrinking = true, 12105cd271fSPaul Blakey }; 12205cd271fSPaul Blakey 12377b9900eSJiri Pirko static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 12477b9900eSJiri Pirko { 12577b9900eSJiri Pirko return mask->range.end - mask->range.start; 12677b9900eSJiri Pirko } 12777b9900eSJiri Pirko 12877b9900eSJiri Pirko static void fl_mask_update_range(struct fl_flow_mask *mask) 12977b9900eSJiri Pirko { 13077b9900eSJiri Pirko const u8 *bytes = (const u8 *) &mask->key; 13177b9900eSJiri Pirko size_t size = sizeof(mask->key); 13205cd271fSPaul Blakey size_t i, first = 0, last; 13377b9900eSJiri Pirko 13405cd271fSPaul Blakey for (i = 0; i < size; i++) { 13577b9900eSJiri Pirko if (bytes[i]) { 13677b9900eSJiri Pirko first = i; 13705cd271fSPaul Blakey break; 13805cd271fSPaul Blakey } 13905cd271fSPaul Blakey } 14005cd271fSPaul Blakey last = first; 14105cd271fSPaul Blakey for (i = size - 1; i != first; i--) { 14205cd271fSPaul Blakey if (bytes[i]) { 14377b9900eSJiri Pirko last = i; 14405cd271fSPaul Blakey break; 14577b9900eSJiri Pirko } 14677b9900eSJiri Pirko } 14777b9900eSJiri Pirko mask->range.start = rounddown(first, sizeof(long)); 14877b9900eSJiri Pirko mask->range.end = roundup(last + 1, sizeof(long)); 14977b9900eSJiri Pirko } 15077b9900eSJiri Pirko 15177b9900eSJiri Pirko static void *fl_key_get_start(struct fl_flow_key *key, 15277b9900eSJiri Pirko const struct fl_flow_mask *mask) 15377b9900eSJiri Pirko { 15477b9900eSJiri Pirko return (u8 *) key + mask->range.start; 15577b9900eSJiri Pirko } 15677b9900eSJiri Pirko 15777b9900eSJiri Pirko static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 15877b9900eSJiri Pirko struct fl_flow_mask *mask) 15977b9900eSJiri Pirko { 16077b9900eSJiri Pirko const long *lkey = fl_key_get_start(key, mask); 16177b9900eSJiri Pirko const long *lmask = fl_key_get_start(&mask->key, mask); 16277b9900eSJiri Pirko long *lmkey = fl_key_get_start(mkey, mask); 16377b9900eSJiri Pirko int i; 16477b9900eSJiri Pirko 16577b9900eSJiri Pirko for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 16677b9900eSJiri Pirko *lmkey++ = *lkey++ & *lmask++; 16777b9900eSJiri Pirko } 16877b9900eSJiri Pirko 169b95ec7ebSJiri Pirko static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt, 170b95ec7ebSJiri Pirko struct fl_flow_mask *mask) 171b95ec7ebSJiri Pirko { 172b95ec7ebSJiri Pirko const long *lmask = fl_key_get_start(&mask->key, mask); 173b95ec7ebSJiri Pirko const long *ltmplt; 174b95ec7ebSJiri Pirko int i; 175b95ec7ebSJiri Pirko 176b95ec7ebSJiri Pirko if (!tmplt) 177b95ec7ebSJiri Pirko return true; 178b95ec7ebSJiri Pirko ltmplt = fl_key_get_start(&tmplt->mask, mask); 179b95ec7ebSJiri Pirko for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) { 180b95ec7ebSJiri Pirko if (~*ltmplt++ & *lmask++) 181b95ec7ebSJiri Pirko return false; 182b95ec7ebSJiri Pirko } 183b95ec7ebSJiri Pirko return true; 184b95ec7ebSJiri Pirko } 185b95ec7ebSJiri Pirko 18677b9900eSJiri Pirko static void fl_clear_masked_range(struct fl_flow_key *key, 18777b9900eSJiri Pirko struct fl_flow_mask *mask) 18877b9900eSJiri Pirko { 18977b9900eSJiri Pirko memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 19077b9900eSJiri Pirko } 19177b9900eSJiri Pirko 1925c72299fSAmritha Nambiar static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter, 1935c72299fSAmritha Nambiar struct fl_flow_key *key, 1945c72299fSAmritha Nambiar struct fl_flow_key *mkey) 1955c72299fSAmritha Nambiar { 1965c72299fSAmritha Nambiar __be16 min_mask, max_mask, min_val, max_val; 1975c72299fSAmritha Nambiar 1985c72299fSAmritha Nambiar min_mask = htons(filter->mask->key.tp_min.dst); 1995c72299fSAmritha Nambiar max_mask = htons(filter->mask->key.tp_max.dst); 2005c72299fSAmritha Nambiar min_val = htons(filter->key.tp_min.dst); 2015c72299fSAmritha Nambiar max_val = htons(filter->key.tp_max.dst); 2025c72299fSAmritha Nambiar 2035c72299fSAmritha Nambiar if (min_mask && max_mask) { 2045c72299fSAmritha Nambiar if (htons(key->tp.dst) < min_val || 2055c72299fSAmritha Nambiar htons(key->tp.dst) > max_val) 2065c72299fSAmritha Nambiar return false; 2075c72299fSAmritha Nambiar 2085c72299fSAmritha Nambiar /* skb does not have min and max values */ 2095c72299fSAmritha Nambiar mkey->tp_min.dst = filter->mkey.tp_min.dst; 2105c72299fSAmritha Nambiar mkey->tp_max.dst = filter->mkey.tp_max.dst; 2115c72299fSAmritha Nambiar } 2125c72299fSAmritha Nambiar return true; 2135c72299fSAmritha Nambiar } 2145c72299fSAmritha Nambiar 2155c72299fSAmritha Nambiar static bool fl_range_port_src_cmp(struct cls_fl_filter *filter, 2165c72299fSAmritha Nambiar struct fl_flow_key *key, 2175c72299fSAmritha Nambiar struct fl_flow_key *mkey) 2185c72299fSAmritha Nambiar { 2195c72299fSAmritha Nambiar __be16 min_mask, max_mask, min_val, max_val; 2205c72299fSAmritha Nambiar 2215c72299fSAmritha Nambiar min_mask = htons(filter->mask->key.tp_min.src); 2225c72299fSAmritha Nambiar max_mask = htons(filter->mask->key.tp_max.src); 2235c72299fSAmritha Nambiar min_val = htons(filter->key.tp_min.src); 2245c72299fSAmritha Nambiar max_val = htons(filter->key.tp_max.src); 2255c72299fSAmritha Nambiar 2265c72299fSAmritha Nambiar if (min_mask && max_mask) { 2275c72299fSAmritha Nambiar if (htons(key->tp.src) < min_val || 2285c72299fSAmritha Nambiar htons(key->tp.src) > max_val) 2295c72299fSAmritha Nambiar return false; 2305c72299fSAmritha Nambiar 2315c72299fSAmritha Nambiar /* skb does not have min and max values */ 2325c72299fSAmritha Nambiar mkey->tp_min.src = filter->mkey.tp_min.src; 2335c72299fSAmritha Nambiar mkey->tp_max.src = filter->mkey.tp_max.src; 2345c72299fSAmritha Nambiar } 2355c72299fSAmritha Nambiar return true; 2365c72299fSAmritha Nambiar } 2375c72299fSAmritha Nambiar 2385c72299fSAmritha Nambiar static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask, 239a3308d8fSPaul Blakey struct fl_flow_key *mkey) 240a3308d8fSPaul Blakey { 24105cd271fSPaul Blakey return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask), 24205cd271fSPaul Blakey mask->filter_ht_params); 243a3308d8fSPaul Blakey } 244a3308d8fSPaul Blakey 2455c72299fSAmritha Nambiar static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask, 2465c72299fSAmritha Nambiar struct fl_flow_key *mkey, 2475c72299fSAmritha Nambiar struct fl_flow_key *key) 2485c72299fSAmritha Nambiar { 2495c72299fSAmritha Nambiar struct cls_fl_filter *filter, *f; 2505c72299fSAmritha Nambiar 2515c72299fSAmritha Nambiar list_for_each_entry_rcu(filter, &mask->filters, list) { 2525c72299fSAmritha Nambiar if (!fl_range_port_dst_cmp(filter, key, mkey)) 2535c72299fSAmritha Nambiar continue; 2545c72299fSAmritha Nambiar 2555c72299fSAmritha Nambiar if (!fl_range_port_src_cmp(filter, key, mkey)) 2565c72299fSAmritha Nambiar continue; 2575c72299fSAmritha Nambiar 2585c72299fSAmritha Nambiar f = __fl_lookup(mask, mkey); 2595c72299fSAmritha Nambiar if (f) 2605c72299fSAmritha Nambiar return f; 2615c72299fSAmritha Nambiar } 2625c72299fSAmritha Nambiar return NULL; 2635c72299fSAmritha Nambiar } 2645c72299fSAmritha Nambiar 2655c72299fSAmritha Nambiar static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask, 2665c72299fSAmritha Nambiar struct fl_flow_key *mkey, 2675c72299fSAmritha Nambiar struct fl_flow_key *key) 2685c72299fSAmritha Nambiar { 2695c72299fSAmritha Nambiar if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE)) 2705c72299fSAmritha Nambiar return fl_lookup_range(mask, mkey, key); 2715c72299fSAmritha Nambiar 2725c72299fSAmritha Nambiar return __fl_lookup(mask, mkey); 2735c72299fSAmritha Nambiar } 2745c72299fSAmritha Nambiar 27577b9900eSJiri Pirko static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 27677b9900eSJiri Pirko struct tcf_result *res) 27777b9900eSJiri Pirko { 27877b9900eSJiri Pirko struct cls_fl_head *head = rcu_dereference_bh(tp->root); 27977b9900eSJiri Pirko struct cls_fl_filter *f; 28005cd271fSPaul Blakey struct fl_flow_mask *mask; 28177b9900eSJiri Pirko struct fl_flow_key skb_key; 28277b9900eSJiri Pirko struct fl_flow_key skb_mkey; 28377b9900eSJiri Pirko 28405cd271fSPaul Blakey list_for_each_entry_rcu(mask, &head->masks, list) { 28505cd271fSPaul Blakey fl_clear_masked_range(&skb_key, mask); 286bc3103f1SAmir Vadai 28777b9900eSJiri Pirko skb_key.indev_ifindex = skb->skb_iif; 28805cd271fSPaul Blakey /* skb_flow_dissect() does not set n_proto in case an unknown 28905cd271fSPaul Blakey * protocol, so do it rather here. 29077b9900eSJiri Pirko */ 29177b9900eSJiri Pirko skb_key.basic.n_proto = skb->protocol; 29205cd271fSPaul Blakey skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key); 29305cd271fSPaul Blakey skb_flow_dissect(skb, &mask->dissector, &skb_key, 0); 29477b9900eSJiri Pirko 29505cd271fSPaul Blakey fl_set_masked_key(&skb_mkey, &skb_key, mask); 29677b9900eSJiri Pirko 2975c72299fSAmritha Nambiar f = fl_lookup(mask, &skb_mkey, &skb_key); 298e8eb36cdSAmir Vadai if (f && !tc_skip_sw(f->flags)) { 29977b9900eSJiri Pirko *res = f->res; 30077b9900eSJiri Pirko return tcf_exts_exec(skb, &f->exts, res); 30177b9900eSJiri Pirko } 30205cd271fSPaul Blakey } 30377b9900eSJiri Pirko return -1; 30477b9900eSJiri Pirko } 30577b9900eSJiri Pirko 30677b9900eSJiri Pirko static int fl_init(struct tcf_proto *tp) 30777b9900eSJiri Pirko { 30877b9900eSJiri Pirko struct cls_fl_head *head; 30977b9900eSJiri Pirko 31077b9900eSJiri Pirko head = kzalloc(sizeof(*head), GFP_KERNEL); 31177b9900eSJiri Pirko if (!head) 31277b9900eSJiri Pirko return -ENOBUFS; 31377b9900eSJiri Pirko 314259e60f9SVlad Buslov spin_lock_init(&head->masks_lock); 31505cd271fSPaul Blakey INIT_LIST_HEAD_RCU(&head->masks); 316c049d56eSVlad Buslov INIT_LIST_HEAD(&head->hw_filters); 31777b9900eSJiri Pirko rcu_assign_pointer(tp->root, head); 318c15ab236SChris Mi idr_init(&head->handle_idr); 31977b9900eSJiri Pirko 32005cd271fSPaul Blakey return rhashtable_init(&head->ht, &mask_ht_params); 32105cd271fSPaul Blakey } 32205cd271fSPaul Blakey 32399815f50SVlad Buslov static void fl_mask_free(struct fl_flow_mask *mask, bool mask_init_done) 32444a5cd43SPaolo Abeni { 32599815f50SVlad Buslov /* temporary masks don't have their filters list and ht initialized */ 32699815f50SVlad Buslov if (mask_init_done) { 327f48ef4d5SVlad Buslov WARN_ON(!list_empty(&mask->filters)); 32844a5cd43SPaolo Abeni rhashtable_destroy(&mask->ht); 32999815f50SVlad Buslov } 33044a5cd43SPaolo Abeni kfree(mask); 33144a5cd43SPaolo Abeni } 33244a5cd43SPaolo Abeni 33344a5cd43SPaolo Abeni static void fl_mask_free_work(struct work_struct *work) 33444a5cd43SPaolo Abeni { 33544a5cd43SPaolo Abeni struct fl_flow_mask *mask = container_of(to_rcu_work(work), 33644a5cd43SPaolo Abeni struct fl_flow_mask, rwork); 33744a5cd43SPaolo Abeni 33899815f50SVlad Buslov fl_mask_free(mask, true); 33999815f50SVlad Buslov } 34099815f50SVlad Buslov 34199815f50SVlad Buslov static void fl_uninit_mask_free_work(struct work_struct *work) 34299815f50SVlad Buslov { 34399815f50SVlad Buslov struct fl_flow_mask *mask = container_of(to_rcu_work(work), 34499815f50SVlad Buslov struct fl_flow_mask, rwork); 34599815f50SVlad Buslov 34699815f50SVlad Buslov fl_mask_free(mask, false); 34744a5cd43SPaolo Abeni } 34844a5cd43SPaolo Abeni 3499994677cSVlad Buslov static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask) 35005cd271fSPaul Blakey { 351f48ef4d5SVlad Buslov if (!refcount_dec_and_test(&mask->refcnt)) 35205cd271fSPaul Blakey return false; 35305cd271fSPaul Blakey 35405cd271fSPaul Blakey rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params); 355259e60f9SVlad Buslov 356259e60f9SVlad Buslov spin_lock(&head->masks_lock); 35705cd271fSPaul Blakey list_del_rcu(&mask->list); 358259e60f9SVlad Buslov spin_unlock(&head->masks_lock); 359259e60f9SVlad Buslov 36044a5cd43SPaolo Abeni tcf_queue_work(&mask->rwork, fl_mask_free_work); 36105cd271fSPaul Blakey 36205cd271fSPaul Blakey return true; 36377b9900eSJiri Pirko } 36477b9900eSJiri Pirko 365c049d56eSVlad Buslov static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp) 366c049d56eSVlad Buslov { 367c049d56eSVlad Buslov /* Flower classifier only changes root pointer during init and destroy. 368c049d56eSVlad Buslov * Users must obtain reference to tcf_proto instance before calling its 369c049d56eSVlad Buslov * API, so tp->root pointer is protected from concurrent call to 370c049d56eSVlad Buslov * fl_destroy() by reference counting. 371c049d56eSVlad Buslov */ 372c049d56eSVlad Buslov return rcu_dereference_raw(tp->root); 373c049d56eSVlad Buslov } 374c049d56eSVlad Buslov 3750dadc117SCong Wang static void __fl_destroy_filter(struct cls_fl_filter *f) 3760dadc117SCong Wang { 3770dadc117SCong Wang tcf_exts_destroy(&f->exts); 3780dadc117SCong Wang tcf_exts_put_net(&f->exts); 3790dadc117SCong Wang kfree(f); 3800dadc117SCong Wang } 3810dadc117SCong Wang 3820552c8afSCong Wang static void fl_destroy_filter_work(struct work_struct *work) 3830552c8afSCong Wang { 384aaa908ffSCong Wang struct cls_fl_filter *f = container_of(to_rcu_work(work), 385aaa908ffSCong Wang struct cls_fl_filter, rwork); 3860552c8afSCong Wang 3870dadc117SCong Wang __fl_destroy_filter(f); 3880552c8afSCong Wang } 3890552c8afSCong Wang 3901b0f8037SJakub Kicinski static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f, 391c24e43d8SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack) 3925b33f488SAmir Vadai { 393de4784caSJiri Pirko struct tc_cls_flower_offload cls_flower = {}; 394208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 3955b33f488SAmir Vadai 396c24e43d8SVlad Buslov if (!rtnl_held) 397c24e43d8SVlad Buslov rtnl_lock(); 398c24e43d8SVlad Buslov 399d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 400de4784caSJiri Pirko cls_flower.command = TC_CLSFLOWER_DESTROY; 401de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 4025b33f488SAmir Vadai 403aeb3fecdSCong Wang tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 4043d81e711SVlad Buslov spin_lock(&tp->lock); 405c049d56eSVlad Buslov list_del_init(&f->hw_list); 406caa72601SJiri Pirko tcf_block_offload_dec(block, &f->flags); 4073d81e711SVlad Buslov spin_unlock(&tp->lock); 408c24e43d8SVlad Buslov 409c24e43d8SVlad Buslov if (!rtnl_held) 410c24e43d8SVlad Buslov rtnl_unlock(); 4115b33f488SAmir Vadai } 4125b33f488SAmir Vadai 413e8eb36cdSAmir Vadai static int fl_hw_replace_filter(struct tcf_proto *tp, 414c24e43d8SVlad Buslov struct cls_fl_filter *f, bool rtnl_held, 41541002038SQuentin Monnet struct netlink_ext_ack *extack) 4165b33f488SAmir Vadai { 417c049d56eSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 418de4784caSJiri Pirko struct tc_cls_flower_offload cls_flower = {}; 419208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 420717503b9SJiri Pirko bool skip_sw = tc_skip_sw(f->flags); 421c24e43d8SVlad Buslov int err = 0; 422c24e43d8SVlad Buslov 423c24e43d8SVlad Buslov if (!rtnl_held) 424c24e43d8SVlad Buslov rtnl_lock(); 4255b33f488SAmir Vadai 426e3ab786bSPablo Neira Ayuso cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 427c24e43d8SVlad Buslov if (!cls_flower.rule) { 428c24e43d8SVlad Buslov err = -ENOMEM; 429c24e43d8SVlad Buslov goto errout; 430c24e43d8SVlad Buslov } 4318f256622SPablo Neira Ayuso 432d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 433de4784caSJiri Pirko cls_flower.command = TC_CLSFLOWER_REPLACE; 434de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 4358f256622SPablo Neira Ayuso cls_flower.rule->match.dissector = &f->mask->dissector; 4368f256622SPablo Neira Ayuso cls_flower.rule->match.mask = &f->mask->key; 4378f256622SPablo Neira Ayuso cls_flower.rule->match.key = &f->mkey; 438384c181eSAmritha Nambiar cls_flower.classid = f->res.classid; 4395b33f488SAmir Vadai 4403a7b6861SPablo Neira Ayuso err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 4413a7b6861SPablo Neira Ayuso if (err) { 4423a7b6861SPablo Neira Ayuso kfree(cls_flower.rule); 443c24e43d8SVlad Buslov if (skip_sw) 4441f15bb4fSVlad Buslov NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 445c24e43d8SVlad Buslov else 446c24e43d8SVlad Buslov err = 0; 447c24e43d8SVlad Buslov goto errout; 4481f15bb4fSVlad Buslov } 4493a7b6861SPablo Neira Ayuso 450aeb3fecdSCong Wang err = tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, skip_sw); 4518f256622SPablo Neira Ayuso kfree(cls_flower.rule); 4528f256622SPablo Neira Ayuso 453717503b9SJiri Pirko if (err < 0) { 454c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, f, true, NULL); 455c24e43d8SVlad Buslov goto errout; 456717503b9SJiri Pirko } else if (err > 0) { 45731533cbaSJohn Hurley f->in_hw_count = err; 458c24e43d8SVlad Buslov err = 0; 4593d81e711SVlad Buslov spin_lock(&tp->lock); 460caa72601SJiri Pirko tcf_block_offload_inc(block, &f->flags); 4613d81e711SVlad Buslov spin_unlock(&tp->lock); 462717503b9SJiri Pirko } 463717503b9SJiri Pirko 464c24e43d8SVlad Buslov if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW)) { 465c24e43d8SVlad Buslov err = -EINVAL; 466c24e43d8SVlad Buslov goto errout; 4675b33f488SAmir Vadai } 4685b33f488SAmir Vadai 469c049d56eSVlad Buslov spin_lock(&tp->lock); 470c049d56eSVlad Buslov list_add(&f->hw_list, &head->hw_filters); 471c049d56eSVlad Buslov spin_unlock(&tp->lock); 472c24e43d8SVlad Buslov errout: 473c24e43d8SVlad Buslov if (!rtnl_held) 474c24e43d8SVlad Buslov rtnl_unlock(); 475c24e43d8SVlad Buslov 476c24e43d8SVlad Buslov return err; 477c24e43d8SVlad Buslov } 478c24e43d8SVlad Buslov 479c24e43d8SVlad Buslov static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f, 480c24e43d8SVlad Buslov bool rtnl_held) 48110cbc684SAmir Vadai { 482de4784caSJiri Pirko struct tc_cls_flower_offload cls_flower = {}; 483208c0f4bSJiri Pirko struct tcf_block *block = tp->chain->block; 48410cbc684SAmir Vadai 485c24e43d8SVlad Buslov if (!rtnl_held) 486c24e43d8SVlad Buslov rtnl_lock(); 487c24e43d8SVlad Buslov 488d6787147SPieter Jansen van Vuuren tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL); 489de4784caSJiri Pirko cls_flower.command = TC_CLSFLOWER_STATS; 490de4784caSJiri Pirko cls_flower.cookie = (unsigned long) f; 491384c181eSAmritha Nambiar cls_flower.classid = f->res.classid; 49210cbc684SAmir Vadai 493aeb3fecdSCong Wang tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 4943b1903efSPablo Neira Ayuso 4953b1903efSPablo Neira Ayuso tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes, 4963b1903efSPablo Neira Ayuso cls_flower.stats.pkts, 4973b1903efSPablo Neira Ayuso cls_flower.stats.lastused); 498c24e43d8SVlad Buslov 499c24e43d8SVlad Buslov if (!rtnl_held) 500c24e43d8SVlad Buslov rtnl_unlock(); 50110cbc684SAmir Vadai } 50210cbc684SAmir Vadai 50306177558SVlad Buslov static void __fl_put(struct cls_fl_filter *f) 50406177558SVlad Buslov { 50506177558SVlad Buslov if (!refcount_dec_and_test(&f->refcnt)) 50606177558SVlad Buslov return; 50706177558SVlad Buslov 50806177558SVlad Buslov if (tcf_exts_get_net(&f->exts)) 50906177558SVlad Buslov tcf_queue_work(&f->rwork, fl_destroy_filter_work); 51006177558SVlad Buslov else 51106177558SVlad Buslov __fl_destroy_filter(f); 51206177558SVlad Buslov } 51306177558SVlad Buslov 51406177558SVlad Buslov static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle) 51506177558SVlad Buslov { 51606177558SVlad Buslov struct cls_fl_filter *f; 51706177558SVlad Buslov 51806177558SVlad Buslov rcu_read_lock(); 51906177558SVlad Buslov f = idr_find(&head->handle_idr, handle); 52006177558SVlad Buslov if (f && !refcount_inc_not_zero(&f->refcnt)) 52106177558SVlad Buslov f = NULL; 52206177558SVlad Buslov rcu_read_unlock(); 52306177558SVlad Buslov 52406177558SVlad Buslov return f; 52506177558SVlad Buslov } 52606177558SVlad Buslov 527b2552b8cSVlad Buslov static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f, 528c24e43d8SVlad Buslov bool *last, bool rtnl_held, 529c24e43d8SVlad Buslov struct netlink_ext_ack *extack) 53013fa876eSRoi Dayan { 531e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 532c15ab236SChris Mi 533b2552b8cSVlad Buslov *last = false; 534b2552b8cSVlad Buslov 5353d81e711SVlad Buslov spin_lock(&tp->lock); 5363d81e711SVlad Buslov if (f->deleted) { 5373d81e711SVlad Buslov spin_unlock(&tp->lock); 538b2552b8cSVlad Buslov return -ENOENT; 5393d81e711SVlad Buslov } 540b2552b8cSVlad Buslov 541b2552b8cSVlad Buslov f->deleted = true; 542b2552b8cSVlad Buslov rhashtable_remove_fast(&f->mask->ht, &f->ht_node, 543b2552b8cSVlad Buslov f->mask->filter_ht_params); 5449c160941SMatthew Wilcox idr_remove(&head->handle_idr, f->handle); 54513fa876eSRoi Dayan list_del_rcu(&f->list); 5463d81e711SVlad Buslov spin_unlock(&tp->lock); 5473d81e711SVlad Buslov 5489994677cSVlad Buslov *last = fl_mask_put(head, f->mask); 54979685219SHadar Hen Zion if (!tc_skip_hw(f->flags)) 550c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, f, rtnl_held, extack); 55113fa876eSRoi Dayan tcf_unbind_filter(tp, &f->res); 55206177558SVlad Buslov __fl_put(f); 55305cd271fSPaul Blakey 554b2552b8cSVlad Buslov return 0; 55513fa876eSRoi Dayan } 55613fa876eSRoi Dayan 557d9363774SDaniel Borkmann static void fl_destroy_sleepable(struct work_struct *work) 558d9363774SDaniel Borkmann { 559aaa908ffSCong Wang struct cls_fl_head *head = container_of(to_rcu_work(work), 560aaa908ffSCong Wang struct cls_fl_head, 561aaa908ffSCong Wang rwork); 562de9dc650SPaul Blakey 563de9dc650SPaul Blakey rhashtable_destroy(&head->ht); 564d9363774SDaniel Borkmann kfree(head); 565d9363774SDaniel Borkmann module_put(THIS_MODULE); 566d9363774SDaniel Borkmann } 567d9363774SDaniel Borkmann 56812db03b6SVlad Buslov static void fl_destroy(struct tcf_proto *tp, bool rtnl_held, 56912db03b6SVlad Buslov struct netlink_ext_ack *extack) 57077b9900eSJiri Pirko { 571e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 57205cd271fSPaul Blakey struct fl_flow_mask *mask, *next_mask; 57377b9900eSJiri Pirko struct cls_fl_filter *f, *next; 574b2552b8cSVlad Buslov bool last; 57577b9900eSJiri Pirko 57605cd271fSPaul Blakey list_for_each_entry_safe(mask, next_mask, &head->masks, list) { 57705cd271fSPaul Blakey list_for_each_entry_safe(f, next, &mask->filters, list) { 578c24e43d8SVlad Buslov __fl_delete(tp, f, &last, rtnl_held, extack); 579b2552b8cSVlad Buslov if (last) 58005cd271fSPaul Blakey break; 58105cd271fSPaul Blakey } 58205cd271fSPaul Blakey } 583c15ab236SChris Mi idr_destroy(&head->handle_idr); 584d9363774SDaniel Borkmann 585d9363774SDaniel Borkmann __module_get(THIS_MODULE); 586aaa908ffSCong Wang tcf_queue_work(&head->rwork, fl_destroy_sleepable); 58777b9900eSJiri Pirko } 58877b9900eSJiri Pirko 58906177558SVlad Buslov static void fl_put(struct tcf_proto *tp, void *arg) 59006177558SVlad Buslov { 59106177558SVlad Buslov struct cls_fl_filter *f = arg; 59206177558SVlad Buslov 59306177558SVlad Buslov __fl_put(f); 59406177558SVlad Buslov } 59506177558SVlad Buslov 5968113c095SWANG Cong static void *fl_get(struct tcf_proto *tp, u32 handle) 59777b9900eSJiri Pirko { 598e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 59977b9900eSJiri Pirko 60006177558SVlad Buslov return __fl_get(head, handle); 60177b9900eSJiri Pirko } 60277b9900eSJiri Pirko 60377b9900eSJiri Pirko static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 60477b9900eSJiri Pirko [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 60577b9900eSJiri Pirko [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 60677b9900eSJiri Pirko [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 60777b9900eSJiri Pirko .len = IFNAMSIZ }, 60877b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 60977b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 61077b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 61177b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 61277b9900eSJiri Pirko [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 61377b9900eSJiri Pirko [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 61477b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 61577b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 61677b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 61777b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 61877b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 61977b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 62077b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 62177b9900eSJiri Pirko [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 62277b9900eSJiri Pirko [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 62377b9900eSJiri Pirko [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 624b175c3a4SJamal Hadi Salim [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 625b175c3a4SJamal Hadi Salim [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 6269399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 6279399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 6289399ae9aSHadar Hen Zion [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 629bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 630bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 631bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 632bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 633bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 634bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 635bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 636bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 637bc3103f1SAmir Vadai [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 638aa72d708SOr Gerlitz [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, 639aa72d708SOr Gerlitz [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, 640aa72d708SOr Gerlitz [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, 641aa72d708SOr Gerlitz [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, 6425976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, 6435976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, 6445976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, 6455976c5f4SSimon Horman [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, 646f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, 647f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, 648f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, 649f4d997fdSHadar Hen Zion [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, 650faa3ffceSOr Gerlitz [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 }, 651faa3ffceSOr Gerlitz [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 }, 6527b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 }, 6537b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 }, 6547b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 }, 6557b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 }, 6567b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 }, 6577b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 }, 6587b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 }, 6597b684884SSimon Horman [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 }, 66099d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 }, 66199d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 }, 66299d31326SSimon Horman [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 }, 66399d31326SSimon Horman [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 }, 66499d31326SSimon Horman [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 }, 66599d31326SSimon Horman [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 }, 66699d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN }, 66799d31326SSimon Horman [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN }, 66899d31326SSimon Horman [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN }, 66999d31326SSimon Horman [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN }, 670a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 }, 671a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 }, 672a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 }, 673a577d8f7SBenjamin LaHaise [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 }, 674fdfc7dd6SJiri Pirko [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 }, 675fdfc7dd6SJiri Pirko [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 }, 6764d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 }, 6774d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 }, 6784d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 }, 6794d80cc0aSOr Gerlitz [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 }, 680d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 }, 681d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 }, 682d64efd09SJianbo Liu [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 }, 6830e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 }, 6840e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 }, 6850e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 }, 6860e2c17b6SOr Gerlitz [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 }, 6870a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED }, 6880a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED }, 6890a6e7778SPieter Jansen van Vuuren }; 6900a6e7778SPieter Jansen van Vuuren 6910a6e7778SPieter Jansen van Vuuren static const struct nla_policy 6920a6e7778SPieter Jansen van Vuuren enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = { 6930a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED }, 6940a6e7778SPieter Jansen van Vuuren }; 6950a6e7778SPieter Jansen van Vuuren 6960a6e7778SPieter Jansen van Vuuren static const struct nla_policy 6970a6e7778SPieter Jansen van Vuuren geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = { 6980a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 }, 6990a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 }, 7000a6e7778SPieter Jansen van Vuuren [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY, 7010a6e7778SPieter Jansen van Vuuren .len = 128 }, 70277b9900eSJiri Pirko }; 70377b9900eSJiri Pirko 70477b9900eSJiri Pirko static void fl_set_key_val(struct nlattr **tb, 70577b9900eSJiri Pirko void *val, int val_type, 70677b9900eSJiri Pirko void *mask, int mask_type, int len) 70777b9900eSJiri Pirko { 70877b9900eSJiri Pirko if (!tb[val_type]) 70977b9900eSJiri Pirko return; 71077b9900eSJiri Pirko memcpy(val, nla_data(tb[val_type]), len); 71177b9900eSJiri Pirko if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 71277b9900eSJiri Pirko memset(mask, 0xff, len); 71377b9900eSJiri Pirko else 71477b9900eSJiri Pirko memcpy(mask, nla_data(tb[mask_type]), len); 71577b9900eSJiri Pirko } 71677b9900eSJiri Pirko 7175c72299fSAmritha Nambiar static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, 7185c72299fSAmritha Nambiar struct fl_flow_key *mask) 7195c72299fSAmritha Nambiar { 7205c72299fSAmritha Nambiar fl_set_key_val(tb, &key->tp_min.dst, 7215c72299fSAmritha Nambiar TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_min.dst, 7225c72299fSAmritha Nambiar TCA_FLOWER_UNSPEC, sizeof(key->tp_min.dst)); 7235c72299fSAmritha Nambiar fl_set_key_val(tb, &key->tp_max.dst, 7245c72299fSAmritha Nambiar TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_max.dst, 7255c72299fSAmritha Nambiar TCA_FLOWER_UNSPEC, sizeof(key->tp_max.dst)); 7265c72299fSAmritha Nambiar fl_set_key_val(tb, &key->tp_min.src, 7275c72299fSAmritha Nambiar TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_min.src, 7285c72299fSAmritha Nambiar TCA_FLOWER_UNSPEC, sizeof(key->tp_min.src)); 7295c72299fSAmritha Nambiar fl_set_key_val(tb, &key->tp_max.src, 7305c72299fSAmritha Nambiar TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_max.src, 7315c72299fSAmritha Nambiar TCA_FLOWER_UNSPEC, sizeof(key->tp_max.src)); 7325c72299fSAmritha Nambiar 7335c72299fSAmritha Nambiar if ((mask->tp_min.dst && mask->tp_max.dst && 7345c72299fSAmritha Nambiar htons(key->tp_max.dst) <= htons(key->tp_min.dst)) || 7355c72299fSAmritha Nambiar (mask->tp_min.src && mask->tp_max.src && 7365c72299fSAmritha Nambiar htons(key->tp_max.src) <= htons(key->tp_min.src))) 7375c72299fSAmritha Nambiar return -EINVAL; 7385c72299fSAmritha Nambiar 7395c72299fSAmritha Nambiar return 0; 7405c72299fSAmritha Nambiar } 7415c72299fSAmritha Nambiar 7421a7fca63SBenjamin LaHaise static int fl_set_key_mpls(struct nlattr **tb, 743a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *key_val, 744a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *key_mask) 745a577d8f7SBenjamin LaHaise { 746a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_TTL]) { 747a577d8f7SBenjamin LaHaise key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]); 748a577d8f7SBenjamin LaHaise key_mask->mpls_ttl = MPLS_TTL_MASK; 749a577d8f7SBenjamin LaHaise } 750a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_BOS]) { 7511a7fca63SBenjamin LaHaise u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]); 7521a7fca63SBenjamin LaHaise 7531a7fca63SBenjamin LaHaise if (bos & ~MPLS_BOS_MASK) 7541a7fca63SBenjamin LaHaise return -EINVAL; 7551a7fca63SBenjamin LaHaise key_val->mpls_bos = bos; 756a577d8f7SBenjamin LaHaise key_mask->mpls_bos = MPLS_BOS_MASK; 757a577d8f7SBenjamin LaHaise } 758a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_TC]) { 7591a7fca63SBenjamin LaHaise u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]); 7601a7fca63SBenjamin LaHaise 7611a7fca63SBenjamin LaHaise if (tc & ~MPLS_TC_MASK) 7621a7fca63SBenjamin LaHaise return -EINVAL; 7631a7fca63SBenjamin LaHaise key_val->mpls_tc = tc; 764a577d8f7SBenjamin LaHaise key_mask->mpls_tc = MPLS_TC_MASK; 765a577d8f7SBenjamin LaHaise } 766a577d8f7SBenjamin LaHaise if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) { 7671a7fca63SBenjamin LaHaise u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]); 7681a7fca63SBenjamin LaHaise 7691a7fca63SBenjamin LaHaise if (label & ~MPLS_LABEL_MASK) 7701a7fca63SBenjamin LaHaise return -EINVAL; 7711a7fca63SBenjamin LaHaise key_val->mpls_label = label; 772a577d8f7SBenjamin LaHaise key_mask->mpls_label = MPLS_LABEL_MASK; 773a577d8f7SBenjamin LaHaise } 7741a7fca63SBenjamin LaHaise return 0; 775a577d8f7SBenjamin LaHaise } 776a577d8f7SBenjamin LaHaise 7779399ae9aSHadar Hen Zion static void fl_set_key_vlan(struct nlattr **tb, 778aaab0834SJianbo Liu __be16 ethertype, 779d64efd09SJianbo Liu int vlan_id_key, int vlan_prio_key, 7809399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *key_val, 7819399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *key_mask) 7829399ae9aSHadar Hen Zion { 7839399ae9aSHadar Hen Zion #define VLAN_PRIORITY_MASK 0x7 7849399ae9aSHadar Hen Zion 785d64efd09SJianbo Liu if (tb[vlan_id_key]) { 7869399ae9aSHadar Hen Zion key_val->vlan_id = 787d64efd09SJianbo Liu nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK; 7889399ae9aSHadar Hen Zion key_mask->vlan_id = VLAN_VID_MASK; 7899399ae9aSHadar Hen Zion } 790d64efd09SJianbo Liu if (tb[vlan_prio_key]) { 7919399ae9aSHadar Hen Zion key_val->vlan_priority = 792d64efd09SJianbo Liu nla_get_u8(tb[vlan_prio_key]) & 7939399ae9aSHadar Hen Zion VLAN_PRIORITY_MASK; 7949399ae9aSHadar Hen Zion key_mask->vlan_priority = VLAN_PRIORITY_MASK; 7959399ae9aSHadar Hen Zion } 796aaab0834SJianbo Liu key_val->vlan_tpid = ethertype; 797aaab0834SJianbo Liu key_mask->vlan_tpid = cpu_to_be16(~0); 7989399ae9aSHadar Hen Zion } 7999399ae9aSHadar Hen Zion 800faa3ffceSOr Gerlitz static void fl_set_key_flag(u32 flower_key, u32 flower_mask, 801faa3ffceSOr Gerlitz u32 *dissector_key, u32 *dissector_mask, 802faa3ffceSOr Gerlitz u32 flower_flag_bit, u32 dissector_flag_bit) 803faa3ffceSOr Gerlitz { 804faa3ffceSOr Gerlitz if (flower_mask & flower_flag_bit) { 805faa3ffceSOr Gerlitz *dissector_mask |= dissector_flag_bit; 806faa3ffceSOr Gerlitz if (flower_key & flower_flag_bit) 807faa3ffceSOr Gerlitz *dissector_key |= dissector_flag_bit; 808faa3ffceSOr Gerlitz } 809faa3ffceSOr Gerlitz } 810faa3ffceSOr Gerlitz 811d9724772SOr Gerlitz static int fl_set_key_flags(struct nlattr **tb, 812faa3ffceSOr Gerlitz u32 *flags_key, u32 *flags_mask) 813faa3ffceSOr Gerlitz { 814faa3ffceSOr Gerlitz u32 key, mask; 815faa3ffceSOr Gerlitz 816d9724772SOr Gerlitz /* mask is mandatory for flags */ 817d9724772SOr Gerlitz if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) 818d9724772SOr Gerlitz return -EINVAL; 819faa3ffceSOr Gerlitz 820faa3ffceSOr Gerlitz key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS])); 821faa3ffceSOr Gerlitz mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK])); 822faa3ffceSOr Gerlitz 823faa3ffceSOr Gerlitz *flags_key = 0; 824faa3ffceSOr Gerlitz *flags_mask = 0; 825faa3ffceSOr Gerlitz 826faa3ffceSOr Gerlitz fl_set_key_flag(key, mask, flags_key, flags_mask, 827faa3ffceSOr Gerlitz TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 828459d153dSPieter Jansen van Vuuren fl_set_key_flag(key, mask, flags_key, flags_mask, 829459d153dSPieter Jansen van Vuuren TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 830459d153dSPieter Jansen van Vuuren FLOW_DIS_FIRST_FRAG); 831d9724772SOr Gerlitz 832d9724772SOr Gerlitz return 0; 833faa3ffceSOr Gerlitz } 834faa3ffceSOr Gerlitz 8350e2c17b6SOr Gerlitz static void fl_set_key_ip(struct nlattr **tb, bool encap, 8364d80cc0aSOr Gerlitz struct flow_dissector_key_ip *key, 8374d80cc0aSOr Gerlitz struct flow_dissector_key_ip *mask) 8384d80cc0aSOr Gerlitz { 8390e2c17b6SOr Gerlitz int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 8400e2c17b6SOr Gerlitz int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 8410e2c17b6SOr Gerlitz int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 8420e2c17b6SOr Gerlitz int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 8434d80cc0aSOr Gerlitz 8440e2c17b6SOr Gerlitz fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)); 8450e2c17b6SOr Gerlitz fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)); 8464d80cc0aSOr Gerlitz } 8474d80cc0aSOr Gerlitz 8480a6e7778SPieter Jansen van Vuuren static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key, 8490a6e7778SPieter Jansen van Vuuren int depth, int option_len, 8500a6e7778SPieter Jansen van Vuuren struct netlink_ext_ack *extack) 8510a6e7778SPieter Jansen van Vuuren { 8520a6e7778SPieter Jansen van Vuuren struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1]; 8530a6e7778SPieter Jansen van Vuuren struct nlattr *class = NULL, *type = NULL, *data = NULL; 8540a6e7778SPieter Jansen van Vuuren struct geneve_opt *opt; 8550a6e7778SPieter Jansen van Vuuren int err, data_len = 0; 8560a6e7778SPieter Jansen van Vuuren 8570a6e7778SPieter Jansen van Vuuren if (option_len > sizeof(struct geneve_opt)) 8580a6e7778SPieter Jansen van Vuuren data_len = option_len - sizeof(struct geneve_opt); 8590a6e7778SPieter Jansen van Vuuren 8600a6e7778SPieter Jansen van Vuuren opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len]; 8610a6e7778SPieter Jansen van Vuuren memset(opt, 0xff, option_len); 8620a6e7778SPieter Jansen van Vuuren opt->length = data_len / 4; 8630a6e7778SPieter Jansen van Vuuren opt->r1 = 0; 8640a6e7778SPieter Jansen van Vuuren opt->r2 = 0; 8650a6e7778SPieter Jansen van Vuuren opt->r3 = 0; 8660a6e7778SPieter Jansen van Vuuren 8670a6e7778SPieter Jansen van Vuuren /* If no mask has been prodived we assume an exact match. */ 8680a6e7778SPieter Jansen van Vuuren if (!depth) 8690a6e7778SPieter Jansen van Vuuren return sizeof(struct geneve_opt) + data_len; 8700a6e7778SPieter Jansen van Vuuren 8710a6e7778SPieter Jansen van Vuuren if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) { 8720a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Non-geneve option type for mask"); 8730a6e7778SPieter Jansen van Vuuren return -EINVAL; 8740a6e7778SPieter Jansen van Vuuren } 8750a6e7778SPieter Jansen van Vuuren 8768cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, 8778cb08174SJohannes Berg TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX, 8780a6e7778SPieter Jansen van Vuuren nla, geneve_opt_policy, extack); 8790a6e7778SPieter Jansen van Vuuren if (err < 0) 8800a6e7778SPieter Jansen van Vuuren return err; 8810a6e7778SPieter Jansen van Vuuren 8820a6e7778SPieter Jansen van Vuuren /* We are not allowed to omit any of CLASS, TYPE or DATA 8830a6e7778SPieter Jansen van Vuuren * fields from the key. 8840a6e7778SPieter Jansen van Vuuren */ 8850a6e7778SPieter Jansen van Vuuren if (!option_len && 8860a6e7778SPieter Jansen van Vuuren (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] || 8870a6e7778SPieter Jansen van Vuuren !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] || 8880a6e7778SPieter Jansen van Vuuren !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) { 8890a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data"); 8900a6e7778SPieter Jansen van Vuuren return -EINVAL; 8910a6e7778SPieter Jansen van Vuuren } 8920a6e7778SPieter Jansen van Vuuren 8930a6e7778SPieter Jansen van Vuuren /* Omitting any of CLASS, TYPE or DATA fields is allowed 8940a6e7778SPieter Jansen van Vuuren * for the mask. 8950a6e7778SPieter Jansen van Vuuren */ 8960a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) { 8970a6e7778SPieter Jansen van Vuuren int new_len = key->enc_opts.len; 8980a6e7778SPieter Jansen van Vuuren 8990a6e7778SPieter Jansen van Vuuren data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]; 9000a6e7778SPieter Jansen van Vuuren data_len = nla_len(data); 9010a6e7778SPieter Jansen van Vuuren if (data_len < 4) { 9020a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long"); 9030a6e7778SPieter Jansen van Vuuren return -ERANGE; 9040a6e7778SPieter Jansen van Vuuren } 9050a6e7778SPieter Jansen van Vuuren if (data_len % 4) { 9060a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long"); 9070a6e7778SPieter Jansen van Vuuren return -ERANGE; 9080a6e7778SPieter Jansen van Vuuren } 9090a6e7778SPieter Jansen van Vuuren 9100a6e7778SPieter Jansen van Vuuren new_len += sizeof(struct geneve_opt) + data_len; 9110a6e7778SPieter Jansen van Vuuren BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX); 9120a6e7778SPieter Jansen van Vuuren if (new_len > FLOW_DIS_TUN_OPTS_MAX) { 9130a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size"); 9140a6e7778SPieter Jansen van Vuuren return -ERANGE; 9150a6e7778SPieter Jansen van Vuuren } 9160a6e7778SPieter Jansen van Vuuren opt->length = data_len / 4; 9170a6e7778SPieter Jansen van Vuuren memcpy(opt->opt_data, nla_data(data), data_len); 9180a6e7778SPieter Jansen van Vuuren } 9190a6e7778SPieter Jansen van Vuuren 9200a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) { 9210a6e7778SPieter Jansen van Vuuren class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]; 9220a6e7778SPieter Jansen van Vuuren opt->opt_class = nla_get_be16(class); 9230a6e7778SPieter Jansen van Vuuren } 9240a6e7778SPieter Jansen van Vuuren 9250a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) { 9260a6e7778SPieter Jansen van Vuuren type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]; 9270a6e7778SPieter Jansen van Vuuren opt->type = nla_get_u8(type); 9280a6e7778SPieter Jansen van Vuuren } 9290a6e7778SPieter Jansen van Vuuren 9300a6e7778SPieter Jansen van Vuuren return sizeof(struct geneve_opt) + data_len; 9310a6e7778SPieter Jansen van Vuuren } 9320a6e7778SPieter Jansen van Vuuren 9330a6e7778SPieter Jansen van Vuuren static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key, 9340a6e7778SPieter Jansen van Vuuren struct fl_flow_key *mask, 9350a6e7778SPieter Jansen van Vuuren struct netlink_ext_ack *extack) 9360a6e7778SPieter Jansen van Vuuren { 9370a6e7778SPieter Jansen van Vuuren const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL; 93863c82997SJakub Kicinski int err, option_len, key_depth, msk_depth = 0; 93963c82997SJakub Kicinski 9408cb08174SJohannes Berg err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS], 94163c82997SJakub Kicinski TCA_FLOWER_KEY_ENC_OPTS_MAX, 94263c82997SJakub Kicinski enc_opts_policy, extack); 94363c82997SJakub Kicinski if (err) 94463c82997SJakub Kicinski return err; 9450a6e7778SPieter Jansen van Vuuren 9460a6e7778SPieter Jansen van Vuuren nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]); 9470a6e7778SPieter Jansen van Vuuren 9480a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) { 9498cb08174SJohannes Berg err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK], 95063c82997SJakub Kicinski TCA_FLOWER_KEY_ENC_OPTS_MAX, 95163c82997SJakub Kicinski enc_opts_policy, extack); 95263c82997SJakub Kicinski if (err) 95363c82997SJakub Kicinski return err; 95463c82997SJakub Kicinski 9550a6e7778SPieter Jansen van Vuuren nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 9560a6e7778SPieter Jansen van Vuuren msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 9570a6e7778SPieter Jansen van Vuuren } 9580a6e7778SPieter Jansen van Vuuren 9590a6e7778SPieter Jansen van Vuuren nla_for_each_attr(nla_opt_key, nla_enc_key, 9600a6e7778SPieter Jansen van Vuuren nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) { 9610a6e7778SPieter Jansen van Vuuren switch (nla_type(nla_opt_key)) { 9620a6e7778SPieter Jansen van Vuuren case TCA_FLOWER_KEY_ENC_OPTS_GENEVE: 9630a6e7778SPieter Jansen van Vuuren option_len = 0; 9640a6e7778SPieter Jansen van Vuuren key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 9650a6e7778SPieter Jansen van Vuuren option_len = fl_set_geneve_opt(nla_opt_key, key, 9660a6e7778SPieter Jansen van Vuuren key_depth, option_len, 9670a6e7778SPieter Jansen van Vuuren extack); 9680a6e7778SPieter Jansen van Vuuren if (option_len < 0) 9690a6e7778SPieter Jansen van Vuuren return option_len; 9700a6e7778SPieter Jansen van Vuuren 9710a6e7778SPieter Jansen van Vuuren key->enc_opts.len += option_len; 9720a6e7778SPieter Jansen van Vuuren /* At the same time we need to parse through the mask 9730a6e7778SPieter Jansen van Vuuren * in order to verify exact and mask attribute lengths. 9740a6e7778SPieter Jansen van Vuuren */ 9750a6e7778SPieter Jansen van Vuuren mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 9760a6e7778SPieter Jansen van Vuuren option_len = fl_set_geneve_opt(nla_opt_msk, mask, 9770a6e7778SPieter Jansen van Vuuren msk_depth, option_len, 9780a6e7778SPieter Jansen van Vuuren extack); 9790a6e7778SPieter Jansen van Vuuren if (option_len < 0) 9800a6e7778SPieter Jansen van Vuuren return option_len; 9810a6e7778SPieter Jansen van Vuuren 9820a6e7778SPieter Jansen van Vuuren mask->enc_opts.len += option_len; 9830a6e7778SPieter Jansen van Vuuren if (key->enc_opts.len != mask->enc_opts.len) { 9840a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 9850a6e7778SPieter Jansen van Vuuren return -EINVAL; 9860a6e7778SPieter Jansen van Vuuren } 9870a6e7778SPieter Jansen van Vuuren 9880a6e7778SPieter Jansen van Vuuren if (msk_depth) 9890a6e7778SPieter Jansen van Vuuren nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 9900a6e7778SPieter Jansen van Vuuren break; 9910a6e7778SPieter Jansen van Vuuren default: 9920a6e7778SPieter Jansen van Vuuren NL_SET_ERR_MSG(extack, "Unknown tunnel option type"); 9930a6e7778SPieter Jansen van Vuuren return -EINVAL; 9940a6e7778SPieter Jansen van Vuuren } 9950a6e7778SPieter Jansen van Vuuren } 9960a6e7778SPieter Jansen van Vuuren 9970a6e7778SPieter Jansen van Vuuren return 0; 9980a6e7778SPieter Jansen van Vuuren } 9990a6e7778SPieter Jansen van Vuuren 100077b9900eSJiri Pirko static int fl_set_key(struct net *net, struct nlattr **tb, 10011057c55fSAlexander Aring struct fl_flow_key *key, struct fl_flow_key *mask, 10021057c55fSAlexander Aring struct netlink_ext_ack *extack) 100377b9900eSJiri Pirko { 10049399ae9aSHadar Hen Zion __be16 ethertype; 1005d9724772SOr Gerlitz int ret = 0; 1006dd3aa3b5SBrian Haley #ifdef CONFIG_NET_CLS_IND 100777b9900eSJiri Pirko if (tb[TCA_FLOWER_INDEV]) { 10081057c55fSAlexander Aring int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack); 100977b9900eSJiri Pirko if (err < 0) 101077b9900eSJiri Pirko return err; 101177b9900eSJiri Pirko key->indev_ifindex = err; 101277b9900eSJiri Pirko mask->indev_ifindex = 0xffffffff; 101377b9900eSJiri Pirko } 1014dd3aa3b5SBrian Haley #endif 101577b9900eSJiri Pirko 101677b9900eSJiri Pirko fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 101777b9900eSJiri Pirko mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 101877b9900eSJiri Pirko sizeof(key->eth.dst)); 101977b9900eSJiri Pirko fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 102077b9900eSJiri Pirko mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 102177b9900eSJiri Pirko sizeof(key->eth.src)); 102266530bdfSJamal Hadi Salim 10230b498a52SArnd Bergmann if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 10249399ae9aSHadar Hen Zion ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 10259399ae9aSHadar Hen Zion 1026aaab0834SJianbo Liu if (eth_type_vlan(ethertype)) { 1027d64efd09SJianbo Liu fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID, 1028d64efd09SJianbo Liu TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, 1029d64efd09SJianbo Liu &mask->vlan); 1030d64efd09SJianbo Liu 10315e9a0fe4SJianbo Liu if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) { 1032d64efd09SJianbo Liu ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]); 1033d64efd09SJianbo Liu if (eth_type_vlan(ethertype)) { 1034d64efd09SJianbo Liu fl_set_key_vlan(tb, ethertype, 1035d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_ID, 1036d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_PRIO, 1037d64efd09SJianbo Liu &key->cvlan, &mask->cvlan); 10389399ae9aSHadar Hen Zion fl_set_key_val(tb, &key->basic.n_proto, 1039d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 1040d64efd09SJianbo Liu &mask->basic.n_proto, 1041d64efd09SJianbo Liu TCA_FLOWER_UNSPEC, 104277b9900eSJiri Pirko sizeof(key->basic.n_proto)); 10439399ae9aSHadar Hen Zion } else { 10449399ae9aSHadar Hen Zion key->basic.n_proto = ethertype; 10459399ae9aSHadar Hen Zion mask->basic.n_proto = cpu_to_be16(~0); 10469399ae9aSHadar Hen Zion } 10475e9a0fe4SJianbo Liu } 1048d64efd09SJianbo Liu } else { 1049d64efd09SJianbo Liu key->basic.n_proto = ethertype; 1050d64efd09SJianbo Liu mask->basic.n_proto = cpu_to_be16(~0); 1051d64efd09SJianbo Liu } 10520b498a52SArnd Bergmann } 105366530bdfSJamal Hadi Salim 105477b9900eSJiri Pirko if (key->basic.n_proto == htons(ETH_P_IP) || 105577b9900eSJiri Pirko key->basic.n_proto == htons(ETH_P_IPV6)) { 105677b9900eSJiri Pirko fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 105777b9900eSJiri Pirko &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 105877b9900eSJiri Pirko sizeof(key->basic.ip_proto)); 10590e2c17b6SOr Gerlitz fl_set_key_ip(tb, false, &key->ip, &mask->ip); 106077b9900eSJiri Pirko } 106166530bdfSJamal Hadi Salim 106266530bdfSJamal Hadi Salim if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 106366530bdfSJamal Hadi Salim key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1064970bfcd0SPaul Blakey mask->control.addr_type = ~0; 106577b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 106677b9900eSJiri Pirko &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 106777b9900eSJiri Pirko sizeof(key->ipv4.src)); 106877b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 106977b9900eSJiri Pirko &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 107077b9900eSJiri Pirko sizeof(key->ipv4.dst)); 107166530bdfSJamal Hadi Salim } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 107266530bdfSJamal Hadi Salim key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1073970bfcd0SPaul Blakey mask->control.addr_type = ~0; 107477b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 107577b9900eSJiri Pirko &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 107677b9900eSJiri Pirko sizeof(key->ipv6.src)); 107777b9900eSJiri Pirko fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 107877b9900eSJiri Pirko &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 107977b9900eSJiri Pirko sizeof(key->ipv6.dst)); 108077b9900eSJiri Pirko } 108166530bdfSJamal Hadi Salim 108277b9900eSJiri Pirko if (key->basic.ip_proto == IPPROTO_TCP) { 108377b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 1084aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 108577b9900eSJiri Pirko sizeof(key->tp.src)); 108677b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 1087aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 108877b9900eSJiri Pirko sizeof(key->tp.dst)); 1089fdfc7dd6SJiri Pirko fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 1090fdfc7dd6SJiri Pirko &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 1091fdfc7dd6SJiri Pirko sizeof(key->tcp.flags)); 109277b9900eSJiri Pirko } else if (key->basic.ip_proto == IPPROTO_UDP) { 109377b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 1094aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 109577b9900eSJiri Pirko sizeof(key->tp.src)); 109677b9900eSJiri Pirko fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 1097aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 109877b9900eSJiri Pirko sizeof(key->tp.dst)); 10995976c5f4SSimon Horman } else if (key->basic.ip_proto == IPPROTO_SCTP) { 11005976c5f4SSimon Horman fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 11015976c5f4SSimon Horman &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 11025976c5f4SSimon Horman sizeof(key->tp.src)); 11035976c5f4SSimon Horman fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 11045976c5f4SSimon Horman &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 11055976c5f4SSimon Horman sizeof(key->tp.dst)); 11067b684884SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_IP) && 11077b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMP) { 11087b684884SSimon Horman fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE, 11097b684884SSimon Horman &mask->icmp.type, 11107b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 11117b684884SSimon Horman sizeof(key->icmp.type)); 11127b684884SSimon Horman fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE, 11137b684884SSimon Horman &mask->icmp.code, 11147b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 11157b684884SSimon Horman sizeof(key->icmp.code)); 11167b684884SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_IPV6) && 11177b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMPV6) { 11187b684884SSimon Horman fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE, 11197b684884SSimon Horman &mask->icmp.type, 11207b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 11217b684884SSimon Horman sizeof(key->icmp.type)); 1122040587afSSimon Horman fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE, 11237b684884SSimon Horman &mask->icmp.code, 1124040587afSSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 11257b684884SSimon Horman sizeof(key->icmp.code)); 1126a577d8f7SBenjamin LaHaise } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) || 1127a577d8f7SBenjamin LaHaise key->basic.n_proto == htons(ETH_P_MPLS_MC)) { 11281a7fca63SBenjamin LaHaise ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls); 11291a7fca63SBenjamin LaHaise if (ret) 11301a7fca63SBenjamin LaHaise return ret; 113199d31326SSimon Horman } else if (key->basic.n_proto == htons(ETH_P_ARP) || 113299d31326SSimon Horman key->basic.n_proto == htons(ETH_P_RARP)) { 113399d31326SSimon Horman fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP, 113499d31326SSimon Horman &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK, 113599d31326SSimon Horman sizeof(key->arp.sip)); 113699d31326SSimon Horman fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP, 113799d31326SSimon Horman &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK, 113899d31326SSimon Horman sizeof(key->arp.tip)); 113999d31326SSimon Horman fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP, 114099d31326SSimon Horman &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK, 114199d31326SSimon Horman sizeof(key->arp.op)); 114299d31326SSimon Horman fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 114399d31326SSimon Horman mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 114499d31326SSimon Horman sizeof(key->arp.sha)); 114599d31326SSimon Horman fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 114699d31326SSimon Horman mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 114799d31326SSimon Horman sizeof(key->arp.tha)); 114877b9900eSJiri Pirko } 114977b9900eSJiri Pirko 11505c72299fSAmritha Nambiar if (key->basic.ip_proto == IPPROTO_TCP || 11515c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_UDP || 11525c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_SCTP) { 11535c72299fSAmritha Nambiar ret = fl_set_key_port_range(tb, key, mask); 11545c72299fSAmritha Nambiar if (ret) 11555c72299fSAmritha Nambiar return ret; 11565c72299fSAmritha Nambiar } 11575c72299fSAmritha Nambiar 1158bc3103f1SAmir Vadai if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 1159bc3103f1SAmir Vadai tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 1160bc3103f1SAmir Vadai key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1161970bfcd0SPaul Blakey mask->enc_control.addr_type = ~0; 1162bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv4.src, 1163bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC, 1164bc3103f1SAmir Vadai &mask->enc_ipv4.src, 1165bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 1166bc3103f1SAmir Vadai sizeof(key->enc_ipv4.src)); 1167bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv4.dst, 1168bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST, 1169bc3103f1SAmir Vadai &mask->enc_ipv4.dst, 1170bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 1171bc3103f1SAmir Vadai sizeof(key->enc_ipv4.dst)); 1172bc3103f1SAmir Vadai } 1173bc3103f1SAmir Vadai 1174bc3103f1SAmir Vadai if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 1175bc3103f1SAmir Vadai tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 1176bc3103f1SAmir Vadai key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1177970bfcd0SPaul Blakey mask->enc_control.addr_type = ~0; 1178bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv6.src, 1179bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC, 1180bc3103f1SAmir Vadai &mask->enc_ipv6.src, 1181bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 1182bc3103f1SAmir Vadai sizeof(key->enc_ipv6.src)); 1183bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_ipv6.dst, 1184bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST, 1185bc3103f1SAmir Vadai &mask->enc_ipv6.dst, 1186bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 1187bc3103f1SAmir Vadai sizeof(key->enc_ipv6.dst)); 1188bc3103f1SAmir Vadai } 1189bc3103f1SAmir Vadai 1190bc3103f1SAmir Vadai fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 1191eb523f42SHadar Hen Zion &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC, 1192bc3103f1SAmir Vadai sizeof(key->enc_key_id.keyid)); 1193bc3103f1SAmir Vadai 1194f4d997fdSHadar Hen Zion fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 1195f4d997fdSHadar Hen Zion &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 1196f4d997fdSHadar Hen Zion sizeof(key->enc_tp.src)); 1197f4d997fdSHadar Hen Zion 1198f4d997fdSHadar Hen Zion fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 1199f4d997fdSHadar Hen Zion &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 1200f4d997fdSHadar Hen Zion sizeof(key->enc_tp.dst)); 1201f4d997fdSHadar Hen Zion 12020e2c17b6SOr Gerlitz fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip); 12030e2c17b6SOr Gerlitz 12040a6e7778SPieter Jansen van Vuuren if (tb[TCA_FLOWER_KEY_ENC_OPTS]) { 12050a6e7778SPieter Jansen van Vuuren ret = fl_set_enc_opt(tb, key, mask, extack); 12060a6e7778SPieter Jansen van Vuuren if (ret) 12070a6e7778SPieter Jansen van Vuuren return ret; 12080a6e7778SPieter Jansen van Vuuren } 12090a6e7778SPieter Jansen van Vuuren 1210d9724772SOr Gerlitz if (tb[TCA_FLOWER_KEY_FLAGS]) 1211d9724772SOr Gerlitz ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags); 1212faa3ffceSOr Gerlitz 1213d9724772SOr Gerlitz return ret; 121477b9900eSJiri Pirko } 121577b9900eSJiri Pirko 121605cd271fSPaul Blakey static void fl_mask_copy(struct fl_flow_mask *dst, 121705cd271fSPaul Blakey struct fl_flow_mask *src) 121877b9900eSJiri Pirko { 121905cd271fSPaul Blakey const void *psrc = fl_key_get_start(&src->key, src); 122005cd271fSPaul Blakey void *pdst = fl_key_get_start(&dst->key, src); 122177b9900eSJiri Pirko 122205cd271fSPaul Blakey memcpy(pdst, psrc, fl_mask_range(src)); 122305cd271fSPaul Blakey dst->range = src->range; 122477b9900eSJiri Pirko } 122577b9900eSJiri Pirko 122677b9900eSJiri Pirko static const struct rhashtable_params fl_ht_params = { 122777b9900eSJiri Pirko .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 122877b9900eSJiri Pirko .head_offset = offsetof(struct cls_fl_filter, ht_node), 122977b9900eSJiri Pirko .automatic_shrinking = true, 123077b9900eSJiri Pirko }; 123177b9900eSJiri Pirko 123205cd271fSPaul Blakey static int fl_init_mask_hashtable(struct fl_flow_mask *mask) 123377b9900eSJiri Pirko { 123405cd271fSPaul Blakey mask->filter_ht_params = fl_ht_params; 123505cd271fSPaul Blakey mask->filter_ht_params.key_len = fl_mask_range(mask); 123605cd271fSPaul Blakey mask->filter_ht_params.key_offset += mask->range.start; 123777b9900eSJiri Pirko 123805cd271fSPaul Blakey return rhashtable_init(&mask->ht, &mask->filter_ht_params); 123977b9900eSJiri Pirko } 124077b9900eSJiri Pirko 124177b9900eSJiri Pirko #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 1242cb205a81Szhong jiang #define FL_KEY_MEMBER_SIZE(member) FIELD_SIZEOF(struct fl_flow_key, member) 124377b9900eSJiri Pirko 1244339ba878SHadar Hen Zion #define FL_KEY_IS_MASKED(mask, member) \ 1245339ba878SHadar Hen Zion memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 1246339ba878SHadar Hen Zion 0, FL_KEY_MEMBER_SIZE(member)) \ 124777b9900eSJiri Pirko 124877b9900eSJiri Pirko #define FL_KEY_SET(keys, cnt, id, member) \ 124977b9900eSJiri Pirko do { \ 125077b9900eSJiri Pirko keys[cnt].key_id = id; \ 125177b9900eSJiri Pirko keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 125277b9900eSJiri Pirko cnt++; \ 125377b9900eSJiri Pirko } while(0); 125477b9900eSJiri Pirko 1255339ba878SHadar Hen Zion #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 125677b9900eSJiri Pirko do { \ 1257339ba878SHadar Hen Zion if (FL_KEY_IS_MASKED(mask, member)) \ 125877b9900eSJiri Pirko FL_KEY_SET(keys, cnt, id, member); \ 125977b9900eSJiri Pirko } while(0); 126077b9900eSJiri Pirko 126133fb5cbaSJiri Pirko static void fl_init_dissector(struct flow_dissector *dissector, 126233fb5cbaSJiri Pirko struct fl_flow_key *mask) 126377b9900eSJiri Pirko { 126477b9900eSJiri Pirko struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 126577b9900eSJiri Pirko size_t cnt = 0; 126677b9900eSJiri Pirko 126742aecaa9STom Herbert FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 126877b9900eSJiri Pirko FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 126933fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 127077b9900eSJiri Pirko FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 127133fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 127277b9900eSJiri Pirko FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 127333fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 127477b9900eSJiri Pirko FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 12755c72299fSAmritha Nambiar if (FL_KEY_IS_MASKED(mask, tp) || 12765c72299fSAmritha Nambiar FL_KEY_IS_MASKED(mask, tp_min) || FL_KEY_IS_MASKED(mask, tp_max)) 12775c72299fSAmritha Nambiar FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_PORTS, tp); 127833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 12794d80cc0aSOr Gerlitz FLOW_DISSECTOR_KEY_IP, ip); 128033fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1281fdfc7dd6SJiri Pirko FLOW_DISSECTOR_KEY_TCP, tcp); 128233fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 12837b684884SSimon Horman FLOW_DISSECTOR_KEY_ICMP, icmp); 128433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 128599d31326SSimon Horman FLOW_DISSECTOR_KEY_ARP, arp); 128633fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1287a577d8f7SBenjamin LaHaise FLOW_DISSECTOR_KEY_MPLS, mpls); 128833fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 12899399ae9aSHadar Hen Zion FLOW_DISSECTOR_KEY_VLAN, vlan); 129033fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1291d64efd09SJianbo Liu FLOW_DISSECTOR_KEY_CVLAN, cvlan); 129233fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1293519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 129433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1295519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4); 129633fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1297519d1052SHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6); 129833fb5cbaSJiri Pirko if (FL_KEY_IS_MASKED(mask, enc_ipv4) || 129933fb5cbaSJiri Pirko FL_KEY_IS_MASKED(mask, enc_ipv6)) 1300519d1052SHadar Hen Zion FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL, 1301519d1052SHadar Hen Zion enc_control); 130233fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1303f4d997fdSHadar Hen Zion FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp); 130433fb5cbaSJiri Pirko FL_KEY_SET_IF_MASKED(mask, keys, cnt, 13050e2c17b6SOr Gerlitz FLOW_DISSECTOR_KEY_ENC_IP, enc_ip); 13060a6e7778SPieter Jansen van Vuuren FL_KEY_SET_IF_MASKED(mask, keys, cnt, 13070a6e7778SPieter Jansen van Vuuren FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts); 130877b9900eSJiri Pirko 130933fb5cbaSJiri Pirko skb_flow_dissector_init(dissector, keys, cnt); 131005cd271fSPaul Blakey } 131105cd271fSPaul Blakey 131205cd271fSPaul Blakey static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, 131305cd271fSPaul Blakey struct fl_flow_mask *mask) 131405cd271fSPaul Blakey { 131505cd271fSPaul Blakey struct fl_flow_mask *newmask; 131605cd271fSPaul Blakey int err; 131705cd271fSPaul Blakey 131805cd271fSPaul Blakey newmask = kzalloc(sizeof(*newmask), GFP_KERNEL); 131905cd271fSPaul Blakey if (!newmask) 132005cd271fSPaul Blakey return ERR_PTR(-ENOMEM); 132105cd271fSPaul Blakey 132205cd271fSPaul Blakey fl_mask_copy(newmask, mask); 132305cd271fSPaul Blakey 13245c72299fSAmritha Nambiar if ((newmask->key.tp_min.dst && newmask->key.tp_max.dst) || 13255c72299fSAmritha Nambiar (newmask->key.tp_min.src && newmask->key.tp_max.src)) 13265c72299fSAmritha Nambiar newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE; 13275c72299fSAmritha Nambiar 132805cd271fSPaul Blakey err = fl_init_mask_hashtable(newmask); 132905cd271fSPaul Blakey if (err) 133005cd271fSPaul Blakey goto errout_free; 133105cd271fSPaul Blakey 133233fb5cbaSJiri Pirko fl_init_dissector(&newmask->dissector, &newmask->key); 133305cd271fSPaul Blakey 133405cd271fSPaul Blakey INIT_LIST_HEAD_RCU(&newmask->filters); 133505cd271fSPaul Blakey 1336f48ef4d5SVlad Buslov refcount_set(&newmask->refcnt, 1); 1337195c234dSVlad Buslov err = rhashtable_replace_fast(&head->ht, &mask->ht_node, 1338195c234dSVlad Buslov &newmask->ht_node, mask_ht_params); 133905cd271fSPaul Blakey if (err) 134005cd271fSPaul Blakey goto errout_destroy; 134105cd271fSPaul Blakey 1342259e60f9SVlad Buslov spin_lock(&head->masks_lock); 134305cd271fSPaul Blakey list_add_tail_rcu(&newmask->list, &head->masks); 1344259e60f9SVlad Buslov spin_unlock(&head->masks_lock); 134505cd271fSPaul Blakey 134605cd271fSPaul Blakey return newmask; 134705cd271fSPaul Blakey 134805cd271fSPaul Blakey errout_destroy: 134905cd271fSPaul Blakey rhashtable_destroy(&newmask->ht); 135005cd271fSPaul Blakey errout_free: 135105cd271fSPaul Blakey kfree(newmask); 135205cd271fSPaul Blakey 135305cd271fSPaul Blakey return ERR_PTR(err); 135477b9900eSJiri Pirko } 135577b9900eSJiri Pirko 135677b9900eSJiri Pirko static int fl_check_assign_mask(struct cls_fl_head *head, 135705cd271fSPaul Blakey struct cls_fl_filter *fnew, 135805cd271fSPaul Blakey struct cls_fl_filter *fold, 135977b9900eSJiri Pirko struct fl_flow_mask *mask) 136077b9900eSJiri Pirko { 136105cd271fSPaul Blakey struct fl_flow_mask *newmask; 1362f48ef4d5SVlad Buslov int ret = 0; 136377b9900eSJiri Pirko 1364f48ef4d5SVlad Buslov rcu_read_lock(); 1365195c234dSVlad Buslov 1366195c234dSVlad Buslov /* Insert mask as temporary node to prevent concurrent creation of mask 1367195c234dSVlad Buslov * with same key. Any concurrent lookups with same key will return 136899815f50SVlad Buslov * -EAGAIN because mask's refcnt is zero. 1369195c234dSVlad Buslov */ 1370195c234dSVlad Buslov fnew->mask = rhashtable_lookup_get_insert_fast(&head->ht, 1371195c234dSVlad Buslov &mask->ht_node, 1372195c234dSVlad Buslov mask_ht_params); 137305cd271fSPaul Blakey if (!fnew->mask) { 1374f48ef4d5SVlad Buslov rcu_read_unlock(); 1375f48ef4d5SVlad Buslov 1376195c234dSVlad Buslov if (fold) { 1377195c234dSVlad Buslov ret = -EINVAL; 1378195c234dSVlad Buslov goto errout_cleanup; 1379195c234dSVlad Buslov } 138005cd271fSPaul Blakey 138105cd271fSPaul Blakey newmask = fl_create_new_mask(head, mask); 1382195c234dSVlad Buslov if (IS_ERR(newmask)) { 1383195c234dSVlad Buslov ret = PTR_ERR(newmask); 1384195c234dSVlad Buslov goto errout_cleanup; 1385195c234dSVlad Buslov } 138605cd271fSPaul Blakey 138705cd271fSPaul Blakey fnew->mask = newmask; 138877b9900eSJiri Pirko return 0; 1389195c234dSVlad Buslov } else if (IS_ERR(fnew->mask)) { 1390195c234dSVlad Buslov ret = PTR_ERR(fnew->mask); 1391f48ef4d5SVlad Buslov } else if (fold && fold->mask != fnew->mask) { 1392f48ef4d5SVlad Buslov ret = -EINVAL; 1393f48ef4d5SVlad Buslov } else if (!refcount_inc_not_zero(&fnew->mask->refcnt)) { 1394f48ef4d5SVlad Buslov /* Mask was deleted concurrently, try again */ 1395f48ef4d5SVlad Buslov ret = -EAGAIN; 1396f48ef4d5SVlad Buslov } 1397f48ef4d5SVlad Buslov rcu_read_unlock(); 1398f48ef4d5SVlad Buslov return ret; 1399195c234dSVlad Buslov 1400195c234dSVlad Buslov errout_cleanup: 1401195c234dSVlad Buslov rhashtable_remove_fast(&head->ht, &mask->ht_node, 1402195c234dSVlad Buslov mask_ht_params); 1403195c234dSVlad Buslov return ret; 140477b9900eSJiri Pirko } 140577b9900eSJiri Pirko 140677b9900eSJiri Pirko static int fl_set_parms(struct net *net, struct tcf_proto *tp, 140777b9900eSJiri Pirko struct cls_fl_filter *f, struct fl_flow_mask *mask, 140877b9900eSJiri Pirko unsigned long base, struct nlattr **tb, 140950a56190SAlexander Aring struct nlattr *est, bool ovr, 1410c24e43d8SVlad Buslov struct fl_flow_tmplt *tmplt, bool rtnl_held, 141150a56190SAlexander Aring struct netlink_ext_ack *extack) 141277b9900eSJiri Pirko { 141377b9900eSJiri Pirko int err; 141477b9900eSJiri Pirko 1415c24e43d8SVlad Buslov err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, rtnl_held, 1416ec6743a1SVlad Buslov extack); 141777b9900eSJiri Pirko if (err < 0) 141877b9900eSJiri Pirko return err; 141977b9900eSJiri Pirko 142077b9900eSJiri Pirko if (tb[TCA_FLOWER_CLASSID]) { 142177b9900eSJiri Pirko f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 1422c24e43d8SVlad Buslov if (!rtnl_held) 1423c24e43d8SVlad Buslov rtnl_lock(); 142477b9900eSJiri Pirko tcf_bind_filter(tp, &f->res, base); 1425c24e43d8SVlad Buslov if (!rtnl_held) 1426c24e43d8SVlad Buslov rtnl_unlock(); 142777b9900eSJiri Pirko } 142877b9900eSJiri Pirko 14291057c55fSAlexander Aring err = fl_set_key(net, tb, &f->key, &mask->key, extack); 143077b9900eSJiri Pirko if (err) 143145507529SJiri Pirko return err; 143277b9900eSJiri Pirko 143377b9900eSJiri Pirko fl_mask_update_range(mask); 143477b9900eSJiri Pirko fl_set_masked_key(&f->mkey, &f->key, mask); 143577b9900eSJiri Pirko 1436b95ec7ebSJiri Pirko if (!fl_mask_fits_tmplt(tmplt, mask)) { 1437b95ec7ebSJiri Pirko NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template"); 1438b95ec7ebSJiri Pirko return -EINVAL; 1439b95ec7ebSJiri Pirko } 1440b95ec7ebSJiri Pirko 144177b9900eSJiri Pirko return 0; 144277b9900eSJiri Pirko } 144377b9900eSJiri Pirko 14441f17f774SVlad Buslov static int fl_ht_insert_unique(struct cls_fl_filter *fnew, 14451f17f774SVlad Buslov struct cls_fl_filter *fold, 14461f17f774SVlad Buslov bool *in_ht) 14471f17f774SVlad Buslov { 14481f17f774SVlad Buslov struct fl_flow_mask *mask = fnew->mask; 14491f17f774SVlad Buslov int err; 14501f17f774SVlad Buslov 14519e35552aSVlad Buslov err = rhashtable_lookup_insert_fast(&mask->ht, 14521f17f774SVlad Buslov &fnew->ht_node, 14531f17f774SVlad Buslov mask->filter_ht_params); 14541f17f774SVlad Buslov if (err) { 14551f17f774SVlad Buslov *in_ht = false; 14561f17f774SVlad Buslov /* It is okay if filter with same key exists when 14571f17f774SVlad Buslov * overwriting. 14581f17f774SVlad Buslov */ 14591f17f774SVlad Buslov return fold && err == -EEXIST ? 0 : err; 14601f17f774SVlad Buslov } 14611f17f774SVlad Buslov 14621f17f774SVlad Buslov *in_ht = true; 14631f17f774SVlad Buslov return 0; 14641f17f774SVlad Buslov } 14651f17f774SVlad Buslov 146677b9900eSJiri Pirko static int fl_change(struct net *net, struct sk_buff *in_skb, 146777b9900eSJiri Pirko struct tcf_proto *tp, unsigned long base, 146877b9900eSJiri Pirko u32 handle, struct nlattr **tca, 146912db03b6SVlad Buslov void **arg, bool ovr, bool rtnl_held, 147012db03b6SVlad Buslov struct netlink_ext_ack *extack) 147177b9900eSJiri Pirko { 1472e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 14738113c095SWANG Cong struct cls_fl_filter *fold = *arg; 147477b9900eSJiri Pirko struct cls_fl_filter *fnew; 14752cddd201SIvan Vecera struct fl_flow_mask *mask; 147639b7b6a6SArnd Bergmann struct nlattr **tb; 14771f17f774SVlad Buslov bool in_ht; 147877b9900eSJiri Pirko int err; 147977b9900eSJiri Pirko 148006177558SVlad Buslov if (!tca[TCA_OPTIONS]) { 148106177558SVlad Buslov err = -EINVAL; 148206177558SVlad Buslov goto errout_fold; 148306177558SVlad Buslov } 148477b9900eSJiri Pirko 14852cddd201SIvan Vecera mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL); 148606177558SVlad Buslov if (!mask) { 148706177558SVlad Buslov err = -ENOBUFS; 148806177558SVlad Buslov goto errout_fold; 148906177558SVlad Buslov } 149039b7b6a6SArnd Bergmann 14912cddd201SIvan Vecera tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 14922cddd201SIvan Vecera if (!tb) { 14932cddd201SIvan Vecera err = -ENOBUFS; 14942cddd201SIvan Vecera goto errout_mask_alloc; 14952cddd201SIvan Vecera } 14962cddd201SIvan Vecera 14978cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 14988cb08174SJohannes Berg tca[TCA_OPTIONS], fl_policy, NULL); 149977b9900eSJiri Pirko if (err < 0) 150039b7b6a6SArnd Bergmann goto errout_tb; 150177b9900eSJiri Pirko 150239b7b6a6SArnd Bergmann if (fold && handle && fold->handle != handle) { 150339b7b6a6SArnd Bergmann err = -EINVAL; 150439b7b6a6SArnd Bergmann goto errout_tb; 150539b7b6a6SArnd Bergmann } 150677b9900eSJiri Pirko 150777b9900eSJiri Pirko fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 150839b7b6a6SArnd Bergmann if (!fnew) { 150939b7b6a6SArnd Bergmann err = -ENOBUFS; 151039b7b6a6SArnd Bergmann goto errout_tb; 151139b7b6a6SArnd Bergmann } 1512c049d56eSVlad Buslov INIT_LIST_HEAD(&fnew->hw_list); 151306177558SVlad Buslov refcount_set(&fnew->refcnt, 1); 151477b9900eSJiri Pirko 151514215108SCong Wang err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0); 1516b9a24bb7SWANG Cong if (err < 0) 1517b9a24bb7SWANG Cong goto errout; 151877b9900eSJiri Pirko 1519ecb3dea4SVlad Buslov if (tb[TCA_FLOWER_FLAGS]) { 1520ecb3dea4SVlad Buslov fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 1521ecb3dea4SVlad Buslov 1522ecb3dea4SVlad Buslov if (!tc_flags_valid(fnew->flags)) { 1523ecb3dea4SVlad Buslov err = -EINVAL; 1524ecb3dea4SVlad Buslov goto errout; 1525ecb3dea4SVlad Buslov } 1526ecb3dea4SVlad Buslov } 1527ecb3dea4SVlad Buslov 1528ecb3dea4SVlad Buslov err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr, 1529c24e43d8SVlad Buslov tp->chain->tmplt_priv, rtnl_held, extack); 1530ecb3dea4SVlad Buslov if (err) 1531ecb3dea4SVlad Buslov goto errout; 1532ecb3dea4SVlad Buslov 1533ecb3dea4SVlad Buslov err = fl_check_assign_mask(head, fnew, fold, mask); 1534ecb3dea4SVlad Buslov if (err) 1535ecb3dea4SVlad Buslov goto errout; 1536ecb3dea4SVlad Buslov 15371f17f774SVlad Buslov err = fl_ht_insert_unique(fnew, fold, &in_ht); 15381f17f774SVlad Buslov if (err) 15391f17f774SVlad Buslov goto errout_mask; 15401f17f774SVlad Buslov 154179685219SHadar Hen Zion if (!tc_skip_hw(fnew->flags)) { 1542c24e43d8SVlad Buslov err = fl_hw_replace_filter(tp, fnew, rtnl_held, extack); 1543e8eb36cdSAmir Vadai if (err) 15441f17f774SVlad Buslov goto errout_ht; 154579685219SHadar Hen Zion } 15465b33f488SAmir Vadai 154755593960SOr Gerlitz if (!tc_in_hw(fnew->flags)) 154855593960SOr Gerlitz fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW; 154955593960SOr Gerlitz 15503d81e711SVlad Buslov spin_lock(&tp->lock); 15513d81e711SVlad Buslov 1552272ffaadSVlad Buslov /* tp was deleted concurrently. -EAGAIN will cause caller to lookup 1553272ffaadSVlad Buslov * proto again or create new one, if necessary. 1554272ffaadSVlad Buslov */ 1555272ffaadSVlad Buslov if (tp->deleting) { 1556272ffaadSVlad Buslov err = -EAGAIN; 1557272ffaadSVlad Buslov goto errout_hw; 1558272ffaadSVlad Buslov } 1559272ffaadSVlad Buslov 15605b33f488SAmir Vadai if (fold) { 1561b2552b8cSVlad Buslov /* Fold filter was deleted concurrently. Retry lookup. */ 1562b2552b8cSVlad Buslov if (fold->deleted) { 1563b2552b8cSVlad Buslov err = -EAGAIN; 1564b2552b8cSVlad Buslov goto errout_hw; 1565b2552b8cSVlad Buslov } 1566b2552b8cSVlad Buslov 1567620da486SVlad Buslov fnew->handle = handle; 1568620da486SVlad Buslov 15691f17f774SVlad Buslov if (!in_ht) { 15701f17f774SVlad Buslov struct rhashtable_params params = 15711f17f774SVlad Buslov fnew->mask->filter_ht_params; 15721f17f774SVlad Buslov 15731f17f774SVlad Buslov err = rhashtable_insert_fast(&fnew->mask->ht, 15741f17f774SVlad Buslov &fnew->ht_node, 15751f17f774SVlad Buslov params); 1576620da486SVlad Buslov if (err) 1577620da486SVlad Buslov goto errout_hw; 15781f17f774SVlad Buslov in_ht = true; 15791f17f774SVlad Buslov } 1580620da486SVlad Buslov 1581c049d56eSVlad Buslov refcount_inc(&fnew->refcnt); 158205cd271fSPaul Blakey rhashtable_remove_fast(&fold->mask->ht, 158305cd271fSPaul Blakey &fold->ht_node, 158405cd271fSPaul Blakey fold->mask->filter_ht_params); 1585234a4624SMatthew Wilcox idr_replace(&head->handle_idr, fnew, fnew->handle); 1586ff3532f2SDaniel Borkmann list_replace_rcu(&fold->list, &fnew->list); 1587b2552b8cSVlad Buslov fold->deleted = true; 1588620da486SVlad Buslov 15893d81e711SVlad Buslov spin_unlock(&tp->lock); 15903d81e711SVlad Buslov 15919994677cSVlad Buslov fl_mask_put(head, fold->mask); 1592620da486SVlad Buslov if (!tc_skip_hw(fold->flags)) 1593c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, fold, rtnl_held, NULL); 159477b9900eSJiri Pirko tcf_unbind_filter(tp, &fold->res); 159506177558SVlad Buslov /* Caller holds reference to fold, so refcnt is always > 0 159606177558SVlad Buslov * after this. 159706177558SVlad Buslov */ 159806177558SVlad Buslov refcount_dec(&fold->refcnt); 159906177558SVlad Buslov __fl_put(fold); 160077b9900eSJiri Pirko } else { 1601620da486SVlad Buslov if (handle) { 1602620da486SVlad Buslov /* user specifies a handle and it doesn't exist */ 1603620da486SVlad Buslov err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1604620da486SVlad Buslov handle, GFP_ATOMIC); 16059a2d9389SVlad Buslov 16069a2d9389SVlad Buslov /* Filter with specified handle was concurrently 16079a2d9389SVlad Buslov * inserted after initial check in cls_api. This is not 16089a2d9389SVlad Buslov * necessarily an error if NLM_F_EXCL is not set in 16099a2d9389SVlad Buslov * message flags. Returning EAGAIN will cause cls_api to 16109a2d9389SVlad Buslov * try to update concurrently inserted rule. 16119a2d9389SVlad Buslov */ 16129a2d9389SVlad Buslov if (err == -ENOSPC) 16139a2d9389SVlad Buslov err = -EAGAIN; 1614620da486SVlad Buslov } else { 1615620da486SVlad Buslov handle = 1; 1616620da486SVlad Buslov err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1617620da486SVlad Buslov INT_MAX, GFP_ATOMIC); 1618620da486SVlad Buslov } 1619620da486SVlad Buslov if (err) 1620620da486SVlad Buslov goto errout_hw; 1621620da486SVlad Buslov 1622c049d56eSVlad Buslov refcount_inc(&fnew->refcnt); 1623620da486SVlad Buslov fnew->handle = handle; 162405cd271fSPaul Blakey list_add_tail_rcu(&fnew->list, &fnew->mask->filters); 16253d81e711SVlad Buslov spin_unlock(&tp->lock); 162677b9900eSJiri Pirko } 162777b9900eSJiri Pirko 1628620da486SVlad Buslov *arg = fnew; 1629620da486SVlad Buslov 163039b7b6a6SArnd Bergmann kfree(tb); 163199815f50SVlad Buslov tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 163277b9900eSJiri Pirko return 0; 163377b9900eSJiri Pirko 1634c049d56eSVlad Buslov errout_ht: 1635c049d56eSVlad Buslov spin_lock(&tp->lock); 1636620da486SVlad Buslov errout_hw: 1637c049d56eSVlad Buslov fnew->deleted = true; 16383d81e711SVlad Buslov spin_unlock(&tp->lock); 1639620da486SVlad Buslov if (!tc_skip_hw(fnew->flags)) 1640c24e43d8SVlad Buslov fl_hw_destroy_filter(tp, fnew, rtnl_held, NULL); 16411f17f774SVlad Buslov if (in_ht) 16421f17f774SVlad Buslov rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node, 16431f17f774SVlad Buslov fnew->mask->filter_ht_params); 1644ecb3dea4SVlad Buslov errout_mask: 16459994677cSVlad Buslov fl_mask_put(head, fnew->mask); 164677b9900eSJiri Pirko errout: 1647c049d56eSVlad Buslov __fl_put(fnew); 164839b7b6a6SArnd Bergmann errout_tb: 164939b7b6a6SArnd Bergmann kfree(tb); 16502cddd201SIvan Vecera errout_mask_alloc: 165199815f50SVlad Buslov tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 165206177558SVlad Buslov errout_fold: 165306177558SVlad Buslov if (fold) 165406177558SVlad Buslov __fl_put(fold); 165577b9900eSJiri Pirko return err; 165677b9900eSJiri Pirko } 165777b9900eSJiri Pirko 1658571acf21SAlexander Aring static int fl_delete(struct tcf_proto *tp, void *arg, bool *last, 165912db03b6SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack) 166077b9900eSJiri Pirko { 1661e474619aSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 16628113c095SWANG Cong struct cls_fl_filter *f = arg; 1663b2552b8cSVlad Buslov bool last_on_mask; 1664b2552b8cSVlad Buslov int err = 0; 166577b9900eSJiri Pirko 1666c24e43d8SVlad Buslov err = __fl_delete(tp, f, &last_on_mask, rtnl_held, extack); 166705cd271fSPaul Blakey *last = list_empty(&head->masks); 166806177558SVlad Buslov __fl_put(f); 166906177558SVlad Buslov 1670b2552b8cSVlad Buslov return err; 167177b9900eSJiri Pirko } 167277b9900eSJiri Pirko 167312db03b6SVlad Buslov static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg, 167412db03b6SVlad Buslov bool rtnl_held) 167577b9900eSJiri Pirko { 1676*d39d7149SCong Wang struct cls_fl_head *head = fl_head_dereference(tp); 1677*d39d7149SCong Wang unsigned long id = arg->cookie, tmp; 167877b9900eSJiri Pirko struct cls_fl_filter *f; 167977b9900eSJiri Pirko 168001683a14SVlad Buslov arg->count = arg->skip; 168101683a14SVlad Buslov 1682*d39d7149SCong Wang idr_for_each_entry_continue_ul(&head->handle_idr, f, tmp, id) { 1683*d39d7149SCong Wang /* don't return filters that are being deleted */ 1684*d39d7149SCong Wang if (!refcount_inc_not_zero(&f->refcnt)) 1685*d39d7149SCong Wang continue; 16868113c095SWANG Cong if (arg->fn(tp, f, arg) < 0) { 168706177558SVlad Buslov __fl_put(f); 168877b9900eSJiri Pirko arg->stop = 1; 168977b9900eSJiri Pirko break; 169077b9900eSJiri Pirko } 169106177558SVlad Buslov __fl_put(f); 169277b9900eSJiri Pirko arg->count++; 169377b9900eSJiri Pirko } 1694*d39d7149SCong Wang arg->cookie = id; 169577b9900eSJiri Pirko } 169677b9900eSJiri Pirko 1697c049d56eSVlad Buslov static struct cls_fl_filter * 1698c049d56eSVlad Buslov fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add) 1699c049d56eSVlad Buslov { 1700c049d56eSVlad Buslov struct cls_fl_head *head = fl_head_dereference(tp); 1701c049d56eSVlad Buslov 1702c049d56eSVlad Buslov spin_lock(&tp->lock); 1703c049d56eSVlad Buslov if (list_empty(&head->hw_filters)) { 1704c049d56eSVlad Buslov spin_unlock(&tp->lock); 1705c049d56eSVlad Buslov return NULL; 1706c049d56eSVlad Buslov } 1707c049d56eSVlad Buslov 1708c049d56eSVlad Buslov if (!f) 1709c049d56eSVlad Buslov f = list_entry(&head->hw_filters, struct cls_fl_filter, 1710c049d56eSVlad Buslov hw_list); 1711c049d56eSVlad Buslov list_for_each_entry_continue(f, &head->hw_filters, hw_list) { 1712c049d56eSVlad Buslov if (!(add && f->deleted) && refcount_inc_not_zero(&f->refcnt)) { 1713c049d56eSVlad Buslov spin_unlock(&tp->lock); 1714c049d56eSVlad Buslov return f; 1715c049d56eSVlad Buslov } 1716c049d56eSVlad Buslov } 1717c049d56eSVlad Buslov 1718c049d56eSVlad Buslov spin_unlock(&tp->lock); 1719c049d56eSVlad Buslov return NULL; 1720c049d56eSVlad Buslov } 1721c049d56eSVlad Buslov 172231533cbaSJohn Hurley static int fl_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb, 172331533cbaSJohn Hurley void *cb_priv, struct netlink_ext_ack *extack) 172431533cbaSJohn Hurley { 172531533cbaSJohn Hurley struct tc_cls_flower_offload cls_flower = {}; 172631533cbaSJohn Hurley struct tcf_block *block = tp->chain->block; 1727c049d56eSVlad Buslov struct cls_fl_filter *f = NULL; 172831533cbaSJohn Hurley int err; 172931533cbaSJohn Hurley 1730c049d56eSVlad Buslov /* hw_filters list can only be changed by hw offload functions after 1731c049d56eSVlad Buslov * obtaining rtnl lock. Make sure it is not changed while reoffload is 1732c049d56eSVlad Buslov * iterating it. 1733c049d56eSVlad Buslov */ 1734c049d56eSVlad Buslov ASSERT_RTNL(); 173531533cbaSJohn Hurley 1736c049d56eSVlad Buslov while ((f = fl_get_next_hw_filter(tp, f, add))) { 1737e3ab786bSPablo Neira Ayuso cls_flower.rule = 1738e3ab786bSPablo Neira Ayuso flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 173995e27a4dSJohn Hurley if (!cls_flower.rule) { 174095e27a4dSJohn Hurley __fl_put(f); 17418f256622SPablo Neira Ayuso return -ENOMEM; 174295e27a4dSJohn Hurley } 17438f256622SPablo Neira Ayuso 174495e27a4dSJohn Hurley tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, 1745d6787147SPieter Jansen van Vuuren extack); 174631533cbaSJohn Hurley cls_flower.command = add ? 174731533cbaSJohn Hurley TC_CLSFLOWER_REPLACE : TC_CLSFLOWER_DESTROY; 174831533cbaSJohn Hurley cls_flower.cookie = (unsigned long)f; 174995e27a4dSJohn Hurley cls_flower.rule->match.dissector = &f->mask->dissector; 175095e27a4dSJohn Hurley cls_flower.rule->match.mask = &f->mask->key; 17518f256622SPablo Neira Ayuso cls_flower.rule->match.key = &f->mkey; 17523a7b6861SPablo Neira Ayuso 175395e27a4dSJohn Hurley err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 17543a7b6861SPablo Neira Ayuso if (err) { 17553a7b6861SPablo Neira Ayuso kfree(cls_flower.rule); 17561f15bb4fSVlad Buslov if (tc_skip_sw(f->flags)) { 17571f15bb4fSVlad Buslov NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 175895e27a4dSJohn Hurley __fl_put(f); 17593a7b6861SPablo Neira Ayuso return err; 17603a7b6861SPablo Neira Ayuso } 176195e27a4dSJohn Hurley goto next_flow; 17621f15bb4fSVlad Buslov } 17633a7b6861SPablo Neira Ayuso 176431533cbaSJohn Hurley cls_flower.classid = f->res.classid; 176531533cbaSJohn Hurley 176631533cbaSJohn Hurley err = cb(TC_SETUP_CLSFLOWER, &cls_flower, cb_priv); 17678f256622SPablo Neira Ayuso kfree(cls_flower.rule); 17688f256622SPablo Neira Ayuso 176931533cbaSJohn Hurley if (err) { 177095e27a4dSJohn Hurley if (add && tc_skip_sw(f->flags)) { 177195e27a4dSJohn Hurley __fl_put(f); 177231533cbaSJohn Hurley return err; 177395e27a4dSJohn Hurley } 177495e27a4dSJohn Hurley goto next_flow; 177531533cbaSJohn Hurley } 177631533cbaSJohn Hurley 17773d81e711SVlad Buslov spin_lock(&tp->lock); 177895e27a4dSJohn Hurley tc_cls_offload_cnt_update(block, &f->in_hw_count, &f->flags, 177995e27a4dSJohn Hurley add); 17803d81e711SVlad Buslov spin_unlock(&tp->lock); 178195e27a4dSJohn Hurley next_flow: 178295e27a4dSJohn Hurley __fl_put(f); 178331533cbaSJohn Hurley } 178431533cbaSJohn Hurley 178531533cbaSJohn Hurley return 0; 178631533cbaSJohn Hurley } 178731533cbaSJohn Hurley 17888f256622SPablo Neira Ayuso static int fl_hw_create_tmplt(struct tcf_chain *chain, 178934738452SJiri Pirko struct fl_flow_tmplt *tmplt) 179034738452SJiri Pirko { 179134738452SJiri Pirko struct tc_cls_flower_offload cls_flower = {}; 179234738452SJiri Pirko struct tcf_block *block = chain->block; 179334738452SJiri Pirko 1794e3ab786bSPablo Neira Ayuso cls_flower.rule = flow_rule_alloc(0); 17958f256622SPablo Neira Ayuso if (!cls_flower.rule) 17968f256622SPablo Neira Ayuso return -ENOMEM; 17978f256622SPablo Neira Ayuso 179834738452SJiri Pirko cls_flower.common.chain_index = chain->index; 179934738452SJiri Pirko cls_flower.command = TC_CLSFLOWER_TMPLT_CREATE; 180034738452SJiri Pirko cls_flower.cookie = (unsigned long) tmplt; 18018f256622SPablo Neira Ayuso cls_flower.rule->match.dissector = &tmplt->dissector; 18028f256622SPablo Neira Ayuso cls_flower.rule->match.mask = &tmplt->mask; 18038f256622SPablo Neira Ayuso cls_flower.rule->match.key = &tmplt->dummy_key; 180434738452SJiri Pirko 180534738452SJiri Pirko /* We don't care if driver (any of them) fails to handle this 180634738452SJiri Pirko * call. It serves just as a hint for it. 180734738452SJiri Pirko */ 1808aeb3fecdSCong Wang tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 18098f256622SPablo Neira Ayuso kfree(cls_flower.rule); 18108f256622SPablo Neira Ayuso 18118f256622SPablo Neira Ayuso return 0; 181234738452SJiri Pirko } 181334738452SJiri Pirko 181434738452SJiri Pirko static void fl_hw_destroy_tmplt(struct tcf_chain *chain, 181534738452SJiri Pirko struct fl_flow_tmplt *tmplt) 181634738452SJiri Pirko { 181734738452SJiri Pirko struct tc_cls_flower_offload cls_flower = {}; 181834738452SJiri Pirko struct tcf_block *block = chain->block; 181934738452SJiri Pirko 182034738452SJiri Pirko cls_flower.common.chain_index = chain->index; 182134738452SJiri Pirko cls_flower.command = TC_CLSFLOWER_TMPLT_DESTROY; 182234738452SJiri Pirko cls_flower.cookie = (unsigned long) tmplt; 182334738452SJiri Pirko 1824aeb3fecdSCong Wang tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 182534738452SJiri Pirko } 182634738452SJiri Pirko 1827b95ec7ebSJiri Pirko static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain, 1828b95ec7ebSJiri Pirko struct nlattr **tca, 1829b95ec7ebSJiri Pirko struct netlink_ext_ack *extack) 1830b95ec7ebSJiri Pirko { 1831b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt; 1832b95ec7ebSJiri Pirko struct nlattr **tb; 1833b95ec7ebSJiri Pirko int err; 1834b95ec7ebSJiri Pirko 1835b95ec7ebSJiri Pirko if (!tca[TCA_OPTIONS]) 1836b95ec7ebSJiri Pirko return ERR_PTR(-EINVAL); 1837b95ec7ebSJiri Pirko 1838b95ec7ebSJiri Pirko tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 1839b95ec7ebSJiri Pirko if (!tb) 1840b95ec7ebSJiri Pirko return ERR_PTR(-ENOBUFS); 18418cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 18428cb08174SJohannes Berg tca[TCA_OPTIONS], fl_policy, NULL); 1843b95ec7ebSJiri Pirko if (err) 1844b95ec7ebSJiri Pirko goto errout_tb; 1845b95ec7ebSJiri Pirko 1846b95ec7ebSJiri Pirko tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL); 18471cbc36a5SDan Carpenter if (!tmplt) { 18481cbc36a5SDan Carpenter err = -ENOMEM; 1849b95ec7ebSJiri Pirko goto errout_tb; 18501cbc36a5SDan Carpenter } 1851b95ec7ebSJiri Pirko tmplt->chain = chain; 1852b95ec7ebSJiri Pirko err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack); 1853b95ec7ebSJiri Pirko if (err) 1854b95ec7ebSJiri Pirko goto errout_tmplt; 1855b95ec7ebSJiri Pirko 1856b95ec7ebSJiri Pirko fl_init_dissector(&tmplt->dissector, &tmplt->mask); 1857b95ec7ebSJiri Pirko 18588f256622SPablo Neira Ayuso err = fl_hw_create_tmplt(chain, tmplt); 18598f256622SPablo Neira Ayuso if (err) 18608f256622SPablo Neira Ayuso goto errout_tmplt; 186134738452SJiri Pirko 18628f256622SPablo Neira Ayuso kfree(tb); 1863b95ec7ebSJiri Pirko return tmplt; 1864b95ec7ebSJiri Pirko 1865b95ec7ebSJiri Pirko errout_tmplt: 1866b95ec7ebSJiri Pirko kfree(tmplt); 1867b95ec7ebSJiri Pirko errout_tb: 1868b95ec7ebSJiri Pirko kfree(tb); 1869b95ec7ebSJiri Pirko return ERR_PTR(err); 1870b95ec7ebSJiri Pirko } 1871b95ec7ebSJiri Pirko 1872b95ec7ebSJiri Pirko static void fl_tmplt_destroy(void *tmplt_priv) 1873b95ec7ebSJiri Pirko { 1874b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt = tmplt_priv; 1875b95ec7ebSJiri Pirko 187695278ddaSCong Wang fl_hw_destroy_tmplt(tmplt->chain, tmplt); 187795278ddaSCong Wang kfree(tmplt); 1878b95ec7ebSJiri Pirko } 1879b95ec7ebSJiri Pirko 188077b9900eSJiri Pirko static int fl_dump_key_val(struct sk_buff *skb, 188177b9900eSJiri Pirko void *val, int val_type, 188277b9900eSJiri Pirko void *mask, int mask_type, int len) 188377b9900eSJiri Pirko { 188477b9900eSJiri Pirko int err; 188577b9900eSJiri Pirko 188677b9900eSJiri Pirko if (!memchr_inv(mask, 0, len)) 188777b9900eSJiri Pirko return 0; 188877b9900eSJiri Pirko err = nla_put(skb, val_type, len, val); 188977b9900eSJiri Pirko if (err) 189077b9900eSJiri Pirko return err; 189177b9900eSJiri Pirko if (mask_type != TCA_FLOWER_UNSPEC) { 189277b9900eSJiri Pirko err = nla_put(skb, mask_type, len, mask); 189377b9900eSJiri Pirko if (err) 189477b9900eSJiri Pirko return err; 189577b9900eSJiri Pirko } 189677b9900eSJiri Pirko return 0; 189777b9900eSJiri Pirko } 189877b9900eSJiri Pirko 18995c72299fSAmritha Nambiar static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key, 19005c72299fSAmritha Nambiar struct fl_flow_key *mask) 19015c72299fSAmritha Nambiar { 19025c72299fSAmritha Nambiar if (fl_dump_key_val(skb, &key->tp_min.dst, TCA_FLOWER_KEY_PORT_DST_MIN, 19035c72299fSAmritha Nambiar &mask->tp_min.dst, TCA_FLOWER_UNSPEC, 19045c72299fSAmritha Nambiar sizeof(key->tp_min.dst)) || 19055c72299fSAmritha Nambiar fl_dump_key_val(skb, &key->tp_max.dst, TCA_FLOWER_KEY_PORT_DST_MAX, 19065c72299fSAmritha Nambiar &mask->tp_max.dst, TCA_FLOWER_UNSPEC, 19075c72299fSAmritha Nambiar sizeof(key->tp_max.dst)) || 19085c72299fSAmritha Nambiar fl_dump_key_val(skb, &key->tp_min.src, TCA_FLOWER_KEY_PORT_SRC_MIN, 19095c72299fSAmritha Nambiar &mask->tp_min.src, TCA_FLOWER_UNSPEC, 19105c72299fSAmritha Nambiar sizeof(key->tp_min.src)) || 19115c72299fSAmritha Nambiar fl_dump_key_val(skb, &key->tp_max.src, TCA_FLOWER_KEY_PORT_SRC_MAX, 19125c72299fSAmritha Nambiar &mask->tp_max.src, TCA_FLOWER_UNSPEC, 19135c72299fSAmritha Nambiar sizeof(key->tp_max.src))) 19145c72299fSAmritha Nambiar return -1; 19155c72299fSAmritha Nambiar 19165c72299fSAmritha Nambiar return 0; 19175c72299fSAmritha Nambiar } 19185c72299fSAmritha Nambiar 1919a577d8f7SBenjamin LaHaise static int fl_dump_key_mpls(struct sk_buff *skb, 1920a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *mpls_key, 1921a577d8f7SBenjamin LaHaise struct flow_dissector_key_mpls *mpls_mask) 1922a577d8f7SBenjamin LaHaise { 1923a577d8f7SBenjamin LaHaise int err; 1924a577d8f7SBenjamin LaHaise 1925a577d8f7SBenjamin LaHaise if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask))) 1926a577d8f7SBenjamin LaHaise return 0; 1927a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_ttl) { 1928a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL, 1929a577d8f7SBenjamin LaHaise mpls_key->mpls_ttl); 1930a577d8f7SBenjamin LaHaise if (err) 1931a577d8f7SBenjamin LaHaise return err; 1932a577d8f7SBenjamin LaHaise } 1933a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_tc) { 1934a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC, 1935a577d8f7SBenjamin LaHaise mpls_key->mpls_tc); 1936a577d8f7SBenjamin LaHaise if (err) 1937a577d8f7SBenjamin LaHaise return err; 1938a577d8f7SBenjamin LaHaise } 1939a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_label) { 1940a577d8f7SBenjamin LaHaise err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL, 1941a577d8f7SBenjamin LaHaise mpls_key->mpls_label); 1942a577d8f7SBenjamin LaHaise if (err) 1943a577d8f7SBenjamin LaHaise return err; 1944a577d8f7SBenjamin LaHaise } 1945a577d8f7SBenjamin LaHaise if (mpls_mask->mpls_bos) { 1946a577d8f7SBenjamin LaHaise err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS, 1947a577d8f7SBenjamin LaHaise mpls_key->mpls_bos); 1948a577d8f7SBenjamin LaHaise if (err) 1949a577d8f7SBenjamin LaHaise return err; 1950a577d8f7SBenjamin LaHaise } 1951a577d8f7SBenjamin LaHaise return 0; 1952a577d8f7SBenjamin LaHaise } 1953a577d8f7SBenjamin LaHaise 19540e2c17b6SOr Gerlitz static int fl_dump_key_ip(struct sk_buff *skb, bool encap, 19554d80cc0aSOr Gerlitz struct flow_dissector_key_ip *key, 19564d80cc0aSOr Gerlitz struct flow_dissector_key_ip *mask) 19574d80cc0aSOr Gerlitz { 19580e2c17b6SOr Gerlitz int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 19590e2c17b6SOr Gerlitz int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 19600e2c17b6SOr Gerlitz int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 19610e2c17b6SOr Gerlitz int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 19620e2c17b6SOr Gerlitz 19630e2c17b6SOr Gerlitz if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) || 19640e2c17b6SOr Gerlitz fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl))) 19654d80cc0aSOr Gerlitz return -1; 19664d80cc0aSOr Gerlitz 19674d80cc0aSOr Gerlitz return 0; 19684d80cc0aSOr Gerlitz } 19694d80cc0aSOr Gerlitz 19709399ae9aSHadar Hen Zion static int fl_dump_key_vlan(struct sk_buff *skb, 1971d64efd09SJianbo Liu int vlan_id_key, int vlan_prio_key, 19729399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *vlan_key, 19739399ae9aSHadar Hen Zion struct flow_dissector_key_vlan *vlan_mask) 19749399ae9aSHadar Hen Zion { 19759399ae9aSHadar Hen Zion int err; 19769399ae9aSHadar Hen Zion 19779399ae9aSHadar Hen Zion if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 19789399ae9aSHadar Hen Zion return 0; 19799399ae9aSHadar Hen Zion if (vlan_mask->vlan_id) { 1980d64efd09SJianbo Liu err = nla_put_u16(skb, vlan_id_key, 19819399ae9aSHadar Hen Zion vlan_key->vlan_id); 19829399ae9aSHadar Hen Zion if (err) 19839399ae9aSHadar Hen Zion return err; 19849399ae9aSHadar Hen Zion } 19859399ae9aSHadar Hen Zion if (vlan_mask->vlan_priority) { 1986d64efd09SJianbo Liu err = nla_put_u8(skb, vlan_prio_key, 19879399ae9aSHadar Hen Zion vlan_key->vlan_priority); 19889399ae9aSHadar Hen Zion if (err) 19899399ae9aSHadar Hen Zion return err; 19909399ae9aSHadar Hen Zion } 19919399ae9aSHadar Hen Zion return 0; 19929399ae9aSHadar Hen Zion } 19939399ae9aSHadar Hen Zion 1994faa3ffceSOr Gerlitz static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask, 1995faa3ffceSOr Gerlitz u32 *flower_key, u32 *flower_mask, 1996faa3ffceSOr Gerlitz u32 flower_flag_bit, u32 dissector_flag_bit) 1997faa3ffceSOr Gerlitz { 1998faa3ffceSOr Gerlitz if (dissector_mask & dissector_flag_bit) { 1999faa3ffceSOr Gerlitz *flower_mask |= flower_flag_bit; 2000faa3ffceSOr Gerlitz if (dissector_key & dissector_flag_bit) 2001faa3ffceSOr Gerlitz *flower_key |= flower_flag_bit; 2002faa3ffceSOr Gerlitz } 2003faa3ffceSOr Gerlitz } 2004faa3ffceSOr Gerlitz 2005faa3ffceSOr Gerlitz static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask) 2006faa3ffceSOr Gerlitz { 2007faa3ffceSOr Gerlitz u32 key, mask; 2008faa3ffceSOr Gerlitz __be32 _key, _mask; 2009faa3ffceSOr Gerlitz int err; 2010faa3ffceSOr Gerlitz 2011faa3ffceSOr Gerlitz if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask))) 2012faa3ffceSOr Gerlitz return 0; 2013faa3ffceSOr Gerlitz 2014faa3ffceSOr Gerlitz key = 0; 2015faa3ffceSOr Gerlitz mask = 0; 2016faa3ffceSOr Gerlitz 2017faa3ffceSOr Gerlitz fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2018faa3ffceSOr Gerlitz TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 2019459d153dSPieter Jansen van Vuuren fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2020459d153dSPieter Jansen van Vuuren TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 2021459d153dSPieter Jansen van Vuuren FLOW_DIS_FIRST_FRAG); 2022faa3ffceSOr Gerlitz 2023faa3ffceSOr Gerlitz _key = cpu_to_be32(key); 2024faa3ffceSOr Gerlitz _mask = cpu_to_be32(mask); 2025faa3ffceSOr Gerlitz 2026faa3ffceSOr Gerlitz err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key); 2027faa3ffceSOr Gerlitz if (err) 2028faa3ffceSOr Gerlitz return err; 2029faa3ffceSOr Gerlitz 2030faa3ffceSOr Gerlitz return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask); 2031faa3ffceSOr Gerlitz } 2032faa3ffceSOr Gerlitz 20330a6e7778SPieter Jansen van Vuuren static int fl_dump_key_geneve_opt(struct sk_buff *skb, 20340a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *enc_opts) 20350a6e7778SPieter Jansen van Vuuren { 20360a6e7778SPieter Jansen van Vuuren struct geneve_opt *opt; 20370a6e7778SPieter Jansen van Vuuren struct nlattr *nest; 20380a6e7778SPieter Jansen van Vuuren int opt_off = 0; 20390a6e7778SPieter Jansen van Vuuren 2040ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE); 20410a6e7778SPieter Jansen van Vuuren if (!nest) 20420a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20430a6e7778SPieter Jansen van Vuuren 20440a6e7778SPieter Jansen van Vuuren while (enc_opts->len > opt_off) { 20450a6e7778SPieter Jansen van Vuuren opt = (struct geneve_opt *)&enc_opts->data[opt_off]; 20460a6e7778SPieter Jansen van Vuuren 20470a6e7778SPieter Jansen van Vuuren if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS, 20480a6e7778SPieter Jansen van Vuuren opt->opt_class)) 20490a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20500a6e7778SPieter Jansen van Vuuren if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE, 20510a6e7778SPieter Jansen van Vuuren opt->type)) 20520a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20530a6e7778SPieter Jansen van Vuuren if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA, 20540a6e7778SPieter Jansen van Vuuren opt->length * 4, opt->opt_data)) 20550a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20560a6e7778SPieter Jansen van Vuuren 20570a6e7778SPieter Jansen van Vuuren opt_off += sizeof(struct geneve_opt) + opt->length * 4; 20580a6e7778SPieter Jansen van Vuuren } 20590a6e7778SPieter Jansen van Vuuren nla_nest_end(skb, nest); 20600a6e7778SPieter Jansen van Vuuren return 0; 20610a6e7778SPieter Jansen van Vuuren 20620a6e7778SPieter Jansen van Vuuren nla_put_failure: 20630a6e7778SPieter Jansen van Vuuren nla_nest_cancel(skb, nest); 20640a6e7778SPieter Jansen van Vuuren return -EMSGSIZE; 20650a6e7778SPieter Jansen van Vuuren } 20660a6e7778SPieter Jansen van Vuuren 20670a6e7778SPieter Jansen van Vuuren static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type, 20680a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *enc_opts) 20690a6e7778SPieter Jansen van Vuuren { 20700a6e7778SPieter Jansen van Vuuren struct nlattr *nest; 20710a6e7778SPieter Jansen van Vuuren int err; 20720a6e7778SPieter Jansen van Vuuren 20730a6e7778SPieter Jansen van Vuuren if (!enc_opts->len) 20740a6e7778SPieter Jansen van Vuuren return 0; 20750a6e7778SPieter Jansen van Vuuren 2076ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, enc_opt_type); 20770a6e7778SPieter Jansen van Vuuren if (!nest) 20780a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20790a6e7778SPieter Jansen van Vuuren 20800a6e7778SPieter Jansen van Vuuren switch (enc_opts->dst_opt_type) { 20810a6e7778SPieter Jansen van Vuuren case TUNNEL_GENEVE_OPT: 20820a6e7778SPieter Jansen van Vuuren err = fl_dump_key_geneve_opt(skb, enc_opts); 20830a6e7778SPieter Jansen van Vuuren if (err) 20840a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20850a6e7778SPieter Jansen van Vuuren break; 20860a6e7778SPieter Jansen van Vuuren default: 20870a6e7778SPieter Jansen van Vuuren goto nla_put_failure; 20880a6e7778SPieter Jansen van Vuuren } 20890a6e7778SPieter Jansen van Vuuren nla_nest_end(skb, nest); 20900a6e7778SPieter Jansen van Vuuren return 0; 20910a6e7778SPieter Jansen van Vuuren 20920a6e7778SPieter Jansen van Vuuren nla_put_failure: 20930a6e7778SPieter Jansen van Vuuren nla_nest_cancel(skb, nest); 20940a6e7778SPieter Jansen van Vuuren return -EMSGSIZE; 20950a6e7778SPieter Jansen van Vuuren } 20960a6e7778SPieter Jansen van Vuuren 20970a6e7778SPieter Jansen van Vuuren static int fl_dump_key_enc_opt(struct sk_buff *skb, 20980a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *key_opts, 20990a6e7778SPieter Jansen van Vuuren struct flow_dissector_key_enc_opts *msk_opts) 21000a6e7778SPieter Jansen van Vuuren { 21010a6e7778SPieter Jansen van Vuuren int err; 21020a6e7778SPieter Jansen van Vuuren 21030a6e7778SPieter Jansen van Vuuren err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts); 21040a6e7778SPieter Jansen van Vuuren if (err) 21050a6e7778SPieter Jansen van Vuuren return err; 21060a6e7778SPieter Jansen van Vuuren 21070a6e7778SPieter Jansen van Vuuren return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts); 21080a6e7778SPieter Jansen van Vuuren } 21090a6e7778SPieter Jansen van Vuuren 2110f5749081SJiri Pirko static int fl_dump_key(struct sk_buff *skb, struct net *net, 2111f5749081SJiri Pirko struct fl_flow_key *key, struct fl_flow_key *mask) 211277b9900eSJiri Pirko { 211377b9900eSJiri Pirko if (mask->indev_ifindex) { 211477b9900eSJiri Pirko struct net_device *dev; 211577b9900eSJiri Pirko 211677b9900eSJiri Pirko dev = __dev_get_by_index(net, key->indev_ifindex); 211777b9900eSJiri Pirko if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 211877b9900eSJiri Pirko goto nla_put_failure; 211977b9900eSJiri Pirko } 212077b9900eSJiri Pirko 212177b9900eSJiri Pirko if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 212277b9900eSJiri Pirko mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 212377b9900eSJiri Pirko sizeof(key->eth.dst)) || 212477b9900eSJiri Pirko fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 212577b9900eSJiri Pirko mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 212677b9900eSJiri Pirko sizeof(key->eth.src)) || 212777b9900eSJiri Pirko fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 212877b9900eSJiri Pirko &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 212977b9900eSJiri Pirko sizeof(key->basic.n_proto))) 213077b9900eSJiri Pirko goto nla_put_failure; 21319399ae9aSHadar Hen Zion 2132a577d8f7SBenjamin LaHaise if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls)) 2133a577d8f7SBenjamin LaHaise goto nla_put_failure; 2134a577d8f7SBenjamin LaHaise 2135d64efd09SJianbo Liu if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID, 2136d64efd09SJianbo Liu TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan)) 21379399ae9aSHadar Hen Zion goto nla_put_failure; 21389399ae9aSHadar Hen Zion 2139d64efd09SJianbo Liu if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID, 2140d64efd09SJianbo Liu TCA_FLOWER_KEY_CVLAN_PRIO, 2141d64efd09SJianbo Liu &key->cvlan, &mask->cvlan) || 2142d64efd09SJianbo Liu (mask->cvlan.vlan_tpid && 2143158abbf1SJianbo Liu nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2144d64efd09SJianbo Liu key->cvlan.vlan_tpid))) 2145d3069512SJianbo Liu goto nla_put_failure; 2146d3069512SJianbo Liu 21475e9a0fe4SJianbo Liu if (mask->basic.n_proto) { 2148d64efd09SJianbo Liu if (mask->cvlan.vlan_tpid) { 2149d64efd09SJianbo Liu if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 2150d64efd09SJianbo Liu key->basic.n_proto)) 2151d64efd09SJianbo Liu goto nla_put_failure; 2152d64efd09SJianbo Liu } else if (mask->vlan.vlan_tpid) { 2153d64efd09SJianbo Liu if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2154d64efd09SJianbo Liu key->basic.n_proto)) 2155d64efd09SJianbo Liu goto nla_put_failure; 2156d64efd09SJianbo Liu } 21575e9a0fe4SJianbo Liu } 2158d64efd09SJianbo Liu 215977b9900eSJiri Pirko if ((key->basic.n_proto == htons(ETH_P_IP) || 216077b9900eSJiri Pirko key->basic.n_proto == htons(ETH_P_IPV6)) && 21614d80cc0aSOr Gerlitz (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 216277b9900eSJiri Pirko &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 21634d80cc0aSOr Gerlitz sizeof(key->basic.ip_proto)) || 21640e2c17b6SOr Gerlitz fl_dump_key_ip(skb, false, &key->ip, &mask->ip))) 216577b9900eSJiri Pirko goto nla_put_failure; 216677b9900eSJiri Pirko 2167c3f83241STom Herbert if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 216877b9900eSJiri Pirko (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 216977b9900eSJiri Pirko &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 217077b9900eSJiri Pirko sizeof(key->ipv4.src)) || 217177b9900eSJiri Pirko fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 217277b9900eSJiri Pirko &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 217377b9900eSJiri Pirko sizeof(key->ipv4.dst)))) 217477b9900eSJiri Pirko goto nla_put_failure; 2175c3f83241STom Herbert else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 217677b9900eSJiri Pirko (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 217777b9900eSJiri Pirko &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 217877b9900eSJiri Pirko sizeof(key->ipv6.src)) || 217977b9900eSJiri Pirko fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 218077b9900eSJiri Pirko &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 218177b9900eSJiri Pirko sizeof(key->ipv6.dst)))) 218277b9900eSJiri Pirko goto nla_put_failure; 218377b9900eSJiri Pirko 218477b9900eSJiri Pirko if (key->basic.ip_proto == IPPROTO_TCP && 218577b9900eSJiri Pirko (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 2186aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 218777b9900eSJiri Pirko sizeof(key->tp.src)) || 218877b9900eSJiri Pirko fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 2189aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 2190fdfc7dd6SJiri Pirko sizeof(key->tp.dst)) || 2191fdfc7dd6SJiri Pirko fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 2192fdfc7dd6SJiri Pirko &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 2193fdfc7dd6SJiri Pirko sizeof(key->tcp.flags)))) 219477b9900eSJiri Pirko goto nla_put_failure; 219577b9900eSJiri Pirko else if (key->basic.ip_proto == IPPROTO_UDP && 219677b9900eSJiri Pirko (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 2197aa72d708SOr Gerlitz &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 219877b9900eSJiri Pirko sizeof(key->tp.src)) || 219977b9900eSJiri Pirko fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 2200aa72d708SOr Gerlitz &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 220177b9900eSJiri Pirko sizeof(key->tp.dst)))) 220277b9900eSJiri Pirko goto nla_put_failure; 22035976c5f4SSimon Horman else if (key->basic.ip_proto == IPPROTO_SCTP && 22045976c5f4SSimon Horman (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 22055976c5f4SSimon Horman &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 22065976c5f4SSimon Horman sizeof(key->tp.src)) || 22075976c5f4SSimon Horman fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 22085976c5f4SSimon Horman &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 22095976c5f4SSimon Horman sizeof(key->tp.dst)))) 22105976c5f4SSimon Horman goto nla_put_failure; 22117b684884SSimon Horman else if (key->basic.n_proto == htons(ETH_P_IP) && 22127b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMP && 22137b684884SSimon Horman (fl_dump_key_val(skb, &key->icmp.type, 22147b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type, 22157b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 22167b684884SSimon Horman sizeof(key->icmp.type)) || 22177b684884SSimon Horman fl_dump_key_val(skb, &key->icmp.code, 22187b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code, 22197b684884SSimon Horman TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 22207b684884SSimon Horman sizeof(key->icmp.code)))) 22217b684884SSimon Horman goto nla_put_failure; 22227b684884SSimon Horman else if (key->basic.n_proto == htons(ETH_P_IPV6) && 22237b684884SSimon Horman key->basic.ip_proto == IPPROTO_ICMPV6 && 22247b684884SSimon Horman (fl_dump_key_val(skb, &key->icmp.type, 22257b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type, 22267b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 22277b684884SSimon Horman sizeof(key->icmp.type)) || 22287b684884SSimon Horman fl_dump_key_val(skb, &key->icmp.code, 22297b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code, 22307b684884SSimon Horman TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 22317b684884SSimon Horman sizeof(key->icmp.code)))) 22327b684884SSimon Horman goto nla_put_failure; 223399d31326SSimon Horman else if ((key->basic.n_proto == htons(ETH_P_ARP) || 223499d31326SSimon Horman key->basic.n_proto == htons(ETH_P_RARP)) && 223599d31326SSimon Horman (fl_dump_key_val(skb, &key->arp.sip, 223699d31326SSimon Horman TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip, 223799d31326SSimon Horman TCA_FLOWER_KEY_ARP_SIP_MASK, 223899d31326SSimon Horman sizeof(key->arp.sip)) || 223999d31326SSimon Horman fl_dump_key_val(skb, &key->arp.tip, 224099d31326SSimon Horman TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip, 224199d31326SSimon Horman TCA_FLOWER_KEY_ARP_TIP_MASK, 224299d31326SSimon Horman sizeof(key->arp.tip)) || 224399d31326SSimon Horman fl_dump_key_val(skb, &key->arp.op, 224499d31326SSimon Horman TCA_FLOWER_KEY_ARP_OP, &mask->arp.op, 224599d31326SSimon Horman TCA_FLOWER_KEY_ARP_OP_MASK, 224699d31326SSimon Horman sizeof(key->arp.op)) || 224799d31326SSimon Horman fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 224899d31326SSimon Horman mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 224999d31326SSimon Horman sizeof(key->arp.sha)) || 225099d31326SSimon Horman fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 225199d31326SSimon Horman mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 225299d31326SSimon Horman sizeof(key->arp.tha)))) 225399d31326SSimon Horman goto nla_put_failure; 225477b9900eSJiri Pirko 22555c72299fSAmritha Nambiar if ((key->basic.ip_proto == IPPROTO_TCP || 22565c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_UDP || 22575c72299fSAmritha Nambiar key->basic.ip_proto == IPPROTO_SCTP) && 22585c72299fSAmritha Nambiar fl_dump_key_port_range(skb, key, mask)) 22595c72299fSAmritha Nambiar goto nla_put_failure; 22605c72299fSAmritha Nambiar 2261bc3103f1SAmir Vadai if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2262bc3103f1SAmir Vadai (fl_dump_key_val(skb, &key->enc_ipv4.src, 2263bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 2264bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 2265bc3103f1SAmir Vadai sizeof(key->enc_ipv4.src)) || 2266bc3103f1SAmir Vadai fl_dump_key_val(skb, &key->enc_ipv4.dst, 2267bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 2268bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 2269bc3103f1SAmir Vadai sizeof(key->enc_ipv4.dst)))) 2270bc3103f1SAmir Vadai goto nla_put_failure; 2271bc3103f1SAmir Vadai else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2272bc3103f1SAmir Vadai (fl_dump_key_val(skb, &key->enc_ipv6.src, 2273bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 2274bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 2275bc3103f1SAmir Vadai sizeof(key->enc_ipv6.src)) || 2276bc3103f1SAmir Vadai fl_dump_key_val(skb, &key->enc_ipv6.dst, 2277bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST, 2278bc3103f1SAmir Vadai &mask->enc_ipv6.dst, 2279bc3103f1SAmir Vadai TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 2280bc3103f1SAmir Vadai sizeof(key->enc_ipv6.dst)))) 2281bc3103f1SAmir Vadai goto nla_put_failure; 2282bc3103f1SAmir Vadai 2283bc3103f1SAmir Vadai if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 2284eb523f42SHadar Hen Zion &mask->enc_key_id, TCA_FLOWER_UNSPEC, 2285f4d997fdSHadar Hen Zion sizeof(key->enc_key_id)) || 2286f4d997fdSHadar Hen Zion fl_dump_key_val(skb, &key->enc_tp.src, 2287f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 2288f4d997fdSHadar Hen Zion &mask->enc_tp.src, 2289f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 2290f4d997fdSHadar Hen Zion sizeof(key->enc_tp.src)) || 2291f4d997fdSHadar Hen Zion fl_dump_key_val(skb, &key->enc_tp.dst, 2292f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 2293f4d997fdSHadar Hen Zion &mask->enc_tp.dst, 2294f4d997fdSHadar Hen Zion TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 22950e2c17b6SOr Gerlitz sizeof(key->enc_tp.dst)) || 22960a6e7778SPieter Jansen van Vuuren fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) || 22970a6e7778SPieter Jansen van Vuuren fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts)) 2298bc3103f1SAmir Vadai goto nla_put_failure; 2299bc3103f1SAmir Vadai 2300faa3ffceSOr Gerlitz if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags)) 2301faa3ffceSOr Gerlitz goto nla_put_failure; 2302faa3ffceSOr Gerlitz 2303f5749081SJiri Pirko return 0; 2304f5749081SJiri Pirko 2305f5749081SJiri Pirko nla_put_failure: 2306f5749081SJiri Pirko return -EMSGSIZE; 2307f5749081SJiri Pirko } 2308f5749081SJiri Pirko 2309f5749081SJiri Pirko static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh, 231012db03b6SVlad Buslov struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 2311f5749081SJiri Pirko { 2312f5749081SJiri Pirko struct cls_fl_filter *f = fh; 2313f5749081SJiri Pirko struct nlattr *nest; 2314f5749081SJiri Pirko struct fl_flow_key *key, *mask; 23153d81e711SVlad Buslov bool skip_hw; 2316f5749081SJiri Pirko 2317f5749081SJiri Pirko if (!f) 2318f5749081SJiri Pirko return skb->len; 2319f5749081SJiri Pirko 2320f5749081SJiri Pirko t->tcm_handle = f->handle; 2321f5749081SJiri Pirko 2322ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 2323f5749081SJiri Pirko if (!nest) 2324f5749081SJiri Pirko goto nla_put_failure; 2325f5749081SJiri Pirko 23263d81e711SVlad Buslov spin_lock(&tp->lock); 23273d81e711SVlad Buslov 2328f5749081SJiri Pirko if (f->res.classid && 2329f5749081SJiri Pirko nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 23303d81e711SVlad Buslov goto nla_put_failure_locked; 2331f5749081SJiri Pirko 2332f5749081SJiri Pirko key = &f->key; 2333f5749081SJiri Pirko mask = &f->mask->key; 23343d81e711SVlad Buslov skip_hw = tc_skip_hw(f->flags); 2335f5749081SJiri Pirko 2336f5749081SJiri Pirko if (fl_dump_key(skb, net, key, mask)) 23373d81e711SVlad Buslov goto nla_put_failure_locked; 2338f5749081SJiri Pirko 2339749e6720SOr Gerlitz if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags)) 23403d81e711SVlad Buslov goto nla_put_failure_locked; 23413d81e711SVlad Buslov 23423d81e711SVlad Buslov spin_unlock(&tp->lock); 23433d81e711SVlad Buslov 23443d81e711SVlad Buslov if (!skip_hw) 2345c24e43d8SVlad Buslov fl_hw_update_stats(tp, f, rtnl_held); 2346e69985c6SAmir Vadai 234786c55361SVlad Buslov if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count)) 234886c55361SVlad Buslov goto nla_put_failure; 234986c55361SVlad Buslov 235077b9900eSJiri Pirko if (tcf_exts_dump(skb, &f->exts)) 235177b9900eSJiri Pirko goto nla_put_failure; 235277b9900eSJiri Pirko 235377b9900eSJiri Pirko nla_nest_end(skb, nest); 235477b9900eSJiri Pirko 235577b9900eSJiri Pirko if (tcf_exts_dump_stats(skb, &f->exts) < 0) 235677b9900eSJiri Pirko goto nla_put_failure; 235777b9900eSJiri Pirko 235877b9900eSJiri Pirko return skb->len; 235977b9900eSJiri Pirko 23603d81e711SVlad Buslov nla_put_failure_locked: 23613d81e711SVlad Buslov spin_unlock(&tp->lock); 236277b9900eSJiri Pirko nla_put_failure: 236377b9900eSJiri Pirko nla_nest_cancel(skb, nest); 236477b9900eSJiri Pirko return -1; 236577b9900eSJiri Pirko } 236677b9900eSJiri Pirko 2367b95ec7ebSJiri Pirko static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv) 2368b95ec7ebSJiri Pirko { 2369b95ec7ebSJiri Pirko struct fl_flow_tmplt *tmplt = tmplt_priv; 2370b95ec7ebSJiri Pirko struct fl_flow_key *key, *mask; 2371b95ec7ebSJiri Pirko struct nlattr *nest; 2372b95ec7ebSJiri Pirko 2373ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 2374b95ec7ebSJiri Pirko if (!nest) 2375b95ec7ebSJiri Pirko goto nla_put_failure; 2376b95ec7ebSJiri Pirko 2377b95ec7ebSJiri Pirko key = &tmplt->dummy_key; 2378b95ec7ebSJiri Pirko mask = &tmplt->mask; 2379b95ec7ebSJiri Pirko 2380b95ec7ebSJiri Pirko if (fl_dump_key(skb, net, key, mask)) 2381b95ec7ebSJiri Pirko goto nla_put_failure; 2382b95ec7ebSJiri Pirko 2383b95ec7ebSJiri Pirko nla_nest_end(skb, nest); 2384b95ec7ebSJiri Pirko 2385b95ec7ebSJiri Pirko return skb->len; 2386b95ec7ebSJiri Pirko 2387b95ec7ebSJiri Pirko nla_put_failure: 2388b95ec7ebSJiri Pirko nla_nest_cancel(skb, nest); 2389b95ec7ebSJiri Pirko return -EMSGSIZE; 2390b95ec7ebSJiri Pirko } 2391b95ec7ebSJiri Pirko 239207d79fc7SCong Wang static void fl_bind_class(void *fh, u32 classid, unsigned long cl) 239307d79fc7SCong Wang { 239407d79fc7SCong Wang struct cls_fl_filter *f = fh; 239507d79fc7SCong Wang 239607d79fc7SCong Wang if (f && f->res.classid == classid) 239707d79fc7SCong Wang f->res.class = cl; 239807d79fc7SCong Wang } 239907d79fc7SCong Wang 240077b9900eSJiri Pirko static struct tcf_proto_ops cls_fl_ops __read_mostly = { 240177b9900eSJiri Pirko .kind = "flower", 240277b9900eSJiri Pirko .classify = fl_classify, 240377b9900eSJiri Pirko .init = fl_init, 240477b9900eSJiri Pirko .destroy = fl_destroy, 240577b9900eSJiri Pirko .get = fl_get, 240606177558SVlad Buslov .put = fl_put, 240777b9900eSJiri Pirko .change = fl_change, 240877b9900eSJiri Pirko .delete = fl_delete, 240977b9900eSJiri Pirko .walk = fl_walk, 241031533cbaSJohn Hurley .reoffload = fl_reoffload, 241177b9900eSJiri Pirko .dump = fl_dump, 241207d79fc7SCong Wang .bind_class = fl_bind_class, 2413b95ec7ebSJiri Pirko .tmplt_create = fl_tmplt_create, 2414b95ec7ebSJiri Pirko .tmplt_destroy = fl_tmplt_destroy, 2415b95ec7ebSJiri Pirko .tmplt_dump = fl_tmplt_dump, 241677b9900eSJiri Pirko .owner = THIS_MODULE, 241792149190SVlad Buslov .flags = TCF_PROTO_OPS_DOIT_UNLOCKED, 241877b9900eSJiri Pirko }; 241977b9900eSJiri Pirko 242077b9900eSJiri Pirko static int __init cls_fl_init(void) 242177b9900eSJiri Pirko { 242277b9900eSJiri Pirko return register_tcf_proto_ops(&cls_fl_ops); 242377b9900eSJiri Pirko } 242477b9900eSJiri Pirko 242577b9900eSJiri Pirko static void __exit cls_fl_exit(void) 242677b9900eSJiri Pirko { 242777b9900eSJiri Pirko unregister_tcf_proto_ops(&cls_fl_ops); 242877b9900eSJiri Pirko } 242977b9900eSJiri Pirko 243077b9900eSJiri Pirko module_init(cls_fl_init); 243177b9900eSJiri Pirko module_exit(cls_fl_exit); 243277b9900eSJiri Pirko 243377b9900eSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 243477b9900eSJiri Pirko MODULE_DESCRIPTION("Flower classifier"); 243577b9900eSJiri Pirko MODULE_LICENSE("GPL v2"); 2436