1 /* 2 * net/sched/cls_flower.c Flower classifier 3 * 4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/rhashtable.h> 16 #include <linux/workqueue.h> 17 #include <linux/refcount.h> 18 19 #include <linux/if_ether.h> 20 #include <linux/in6.h> 21 #include <linux/ip.h> 22 #include <linux/mpls.h> 23 24 #include <net/sch_generic.h> 25 #include <net/pkt_cls.h> 26 #include <net/ip.h> 27 #include <net/flow_dissector.h> 28 #include <net/geneve.h> 29 30 #include <net/dst.h> 31 #include <net/dst_metadata.h> 32 33 struct fl_flow_key { 34 int indev_ifindex; 35 struct flow_dissector_key_control control; 36 struct flow_dissector_key_control enc_control; 37 struct flow_dissector_key_basic basic; 38 struct flow_dissector_key_eth_addrs eth; 39 struct flow_dissector_key_vlan vlan; 40 struct flow_dissector_key_vlan cvlan; 41 union { 42 struct flow_dissector_key_ipv4_addrs ipv4; 43 struct flow_dissector_key_ipv6_addrs ipv6; 44 }; 45 struct flow_dissector_key_ports tp; 46 struct flow_dissector_key_icmp icmp; 47 struct flow_dissector_key_arp arp; 48 struct flow_dissector_key_keyid enc_key_id; 49 union { 50 struct flow_dissector_key_ipv4_addrs enc_ipv4; 51 struct flow_dissector_key_ipv6_addrs enc_ipv6; 52 }; 53 struct flow_dissector_key_ports enc_tp; 54 struct flow_dissector_key_mpls mpls; 55 struct flow_dissector_key_tcp tcp; 56 struct flow_dissector_key_ip ip; 57 struct flow_dissector_key_ip enc_ip; 58 struct flow_dissector_key_enc_opts enc_opts; 59 struct flow_dissector_key_ports tp_min; 60 struct flow_dissector_key_ports tp_max; 61 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 62 63 struct fl_flow_mask_range { 64 unsigned short int start; 65 unsigned short int end; 66 }; 67 68 struct fl_flow_mask { 69 struct fl_flow_key key; 70 struct fl_flow_mask_range range; 71 u32 flags; 72 struct rhash_head ht_node; 73 struct rhashtable ht; 74 struct rhashtable_params filter_ht_params; 75 struct flow_dissector dissector; 76 struct list_head filters; 77 struct rcu_work rwork; 78 struct list_head list; 79 refcount_t refcnt; 80 }; 81 82 struct fl_flow_tmplt { 83 struct fl_flow_key dummy_key; 84 struct fl_flow_key mask; 85 struct flow_dissector dissector; 86 struct tcf_chain *chain; 87 }; 88 89 struct cls_fl_head { 90 struct rhashtable ht; 91 spinlock_t masks_lock; /* Protect masks list */ 92 struct list_head masks; 93 struct list_head hw_filters; 94 struct rcu_work rwork; 95 struct idr handle_idr; 96 }; 97 98 struct cls_fl_filter { 99 struct fl_flow_mask *mask; 100 struct rhash_head ht_node; 101 struct fl_flow_key mkey; 102 struct tcf_exts exts; 103 struct tcf_result res; 104 struct fl_flow_key key; 105 struct list_head list; 106 struct list_head hw_list; 107 u32 handle; 108 u32 flags; 109 u32 in_hw_count; 110 struct rcu_work rwork; 111 struct net_device *hw_dev; 112 /* Flower classifier is unlocked, which means that its reference counter 113 * can be changed concurrently without any kind of external 114 * synchronization. Use atomic reference counter to be concurrency-safe. 115 */ 116 refcount_t refcnt; 117 bool deleted; 118 }; 119 120 static const struct rhashtable_params mask_ht_params = { 121 .key_offset = offsetof(struct fl_flow_mask, key), 122 .key_len = sizeof(struct fl_flow_key), 123 .head_offset = offsetof(struct fl_flow_mask, ht_node), 124 .automatic_shrinking = true, 125 }; 126 127 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 128 { 129 return mask->range.end - mask->range.start; 130 } 131 132 static void fl_mask_update_range(struct fl_flow_mask *mask) 133 { 134 const u8 *bytes = (const u8 *) &mask->key; 135 size_t size = sizeof(mask->key); 136 size_t i, first = 0, last; 137 138 for (i = 0; i < size; i++) { 139 if (bytes[i]) { 140 first = i; 141 break; 142 } 143 } 144 last = first; 145 for (i = size - 1; i != first; i--) { 146 if (bytes[i]) { 147 last = i; 148 break; 149 } 150 } 151 mask->range.start = rounddown(first, sizeof(long)); 152 mask->range.end = roundup(last + 1, sizeof(long)); 153 } 154 155 static void *fl_key_get_start(struct fl_flow_key *key, 156 const struct fl_flow_mask *mask) 157 { 158 return (u8 *) key + mask->range.start; 159 } 160 161 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 162 struct fl_flow_mask *mask) 163 { 164 const long *lkey = fl_key_get_start(key, mask); 165 const long *lmask = fl_key_get_start(&mask->key, mask); 166 long *lmkey = fl_key_get_start(mkey, mask); 167 int i; 168 169 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 170 *lmkey++ = *lkey++ & *lmask++; 171 } 172 173 static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt, 174 struct fl_flow_mask *mask) 175 { 176 const long *lmask = fl_key_get_start(&mask->key, mask); 177 const long *ltmplt; 178 int i; 179 180 if (!tmplt) 181 return true; 182 ltmplt = fl_key_get_start(&tmplt->mask, mask); 183 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) { 184 if (~*ltmplt++ & *lmask++) 185 return false; 186 } 187 return true; 188 } 189 190 static void fl_clear_masked_range(struct fl_flow_key *key, 191 struct fl_flow_mask *mask) 192 { 193 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 194 } 195 196 static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter, 197 struct fl_flow_key *key, 198 struct fl_flow_key *mkey) 199 { 200 __be16 min_mask, max_mask, min_val, max_val; 201 202 min_mask = htons(filter->mask->key.tp_min.dst); 203 max_mask = htons(filter->mask->key.tp_max.dst); 204 min_val = htons(filter->key.tp_min.dst); 205 max_val = htons(filter->key.tp_max.dst); 206 207 if (min_mask && max_mask) { 208 if (htons(key->tp.dst) < min_val || 209 htons(key->tp.dst) > max_val) 210 return false; 211 212 /* skb does not have min and max values */ 213 mkey->tp_min.dst = filter->mkey.tp_min.dst; 214 mkey->tp_max.dst = filter->mkey.tp_max.dst; 215 } 216 return true; 217 } 218 219 static bool fl_range_port_src_cmp(struct cls_fl_filter *filter, 220 struct fl_flow_key *key, 221 struct fl_flow_key *mkey) 222 { 223 __be16 min_mask, max_mask, min_val, max_val; 224 225 min_mask = htons(filter->mask->key.tp_min.src); 226 max_mask = htons(filter->mask->key.tp_max.src); 227 min_val = htons(filter->key.tp_min.src); 228 max_val = htons(filter->key.tp_max.src); 229 230 if (min_mask && max_mask) { 231 if (htons(key->tp.src) < min_val || 232 htons(key->tp.src) > max_val) 233 return false; 234 235 /* skb does not have min and max values */ 236 mkey->tp_min.src = filter->mkey.tp_min.src; 237 mkey->tp_max.src = filter->mkey.tp_max.src; 238 } 239 return true; 240 } 241 242 static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask, 243 struct fl_flow_key *mkey) 244 { 245 return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask), 246 mask->filter_ht_params); 247 } 248 249 static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask, 250 struct fl_flow_key *mkey, 251 struct fl_flow_key *key) 252 { 253 struct cls_fl_filter *filter, *f; 254 255 list_for_each_entry_rcu(filter, &mask->filters, list) { 256 if (!fl_range_port_dst_cmp(filter, key, mkey)) 257 continue; 258 259 if (!fl_range_port_src_cmp(filter, key, mkey)) 260 continue; 261 262 f = __fl_lookup(mask, mkey); 263 if (f) 264 return f; 265 } 266 return NULL; 267 } 268 269 static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask, 270 struct fl_flow_key *mkey, 271 struct fl_flow_key *key) 272 { 273 if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE)) 274 return fl_lookup_range(mask, mkey, key); 275 276 return __fl_lookup(mask, mkey); 277 } 278 279 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 280 struct tcf_result *res) 281 { 282 struct cls_fl_head *head = rcu_dereference_bh(tp->root); 283 struct cls_fl_filter *f; 284 struct fl_flow_mask *mask; 285 struct fl_flow_key skb_key; 286 struct fl_flow_key skb_mkey; 287 288 list_for_each_entry_rcu(mask, &head->masks, list) { 289 fl_clear_masked_range(&skb_key, mask); 290 291 skb_key.indev_ifindex = skb->skb_iif; 292 /* skb_flow_dissect() does not set n_proto in case an unknown 293 * protocol, so do it rather here. 294 */ 295 skb_key.basic.n_proto = skb->protocol; 296 skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key); 297 skb_flow_dissect(skb, &mask->dissector, &skb_key, 0); 298 299 fl_set_masked_key(&skb_mkey, &skb_key, mask); 300 301 f = fl_lookup(mask, &skb_mkey, &skb_key); 302 if (f && !tc_skip_sw(f->flags)) { 303 *res = f->res; 304 return tcf_exts_exec(skb, &f->exts, res); 305 } 306 } 307 return -1; 308 } 309 310 static int fl_init(struct tcf_proto *tp) 311 { 312 struct cls_fl_head *head; 313 314 head = kzalloc(sizeof(*head), GFP_KERNEL); 315 if (!head) 316 return -ENOBUFS; 317 318 spin_lock_init(&head->masks_lock); 319 INIT_LIST_HEAD_RCU(&head->masks); 320 INIT_LIST_HEAD(&head->hw_filters); 321 rcu_assign_pointer(tp->root, head); 322 idr_init(&head->handle_idr); 323 324 return rhashtable_init(&head->ht, &mask_ht_params); 325 } 326 327 static void fl_mask_free(struct fl_flow_mask *mask) 328 { 329 WARN_ON(!list_empty(&mask->filters)); 330 rhashtable_destroy(&mask->ht); 331 kfree(mask); 332 } 333 334 static void fl_mask_free_work(struct work_struct *work) 335 { 336 struct fl_flow_mask *mask = container_of(to_rcu_work(work), 337 struct fl_flow_mask, rwork); 338 339 fl_mask_free(mask); 340 } 341 342 static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask) 343 { 344 if (!refcount_dec_and_test(&mask->refcnt)) 345 return false; 346 347 rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params); 348 349 spin_lock(&head->masks_lock); 350 list_del_rcu(&mask->list); 351 spin_unlock(&head->masks_lock); 352 353 tcf_queue_work(&mask->rwork, fl_mask_free_work); 354 355 return true; 356 } 357 358 static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp) 359 { 360 /* Flower classifier only changes root pointer during init and destroy. 361 * Users must obtain reference to tcf_proto instance before calling its 362 * API, so tp->root pointer is protected from concurrent call to 363 * fl_destroy() by reference counting. 364 */ 365 return rcu_dereference_raw(tp->root); 366 } 367 368 static void __fl_destroy_filter(struct cls_fl_filter *f) 369 { 370 tcf_exts_destroy(&f->exts); 371 tcf_exts_put_net(&f->exts); 372 kfree(f); 373 } 374 375 static void fl_destroy_filter_work(struct work_struct *work) 376 { 377 struct cls_fl_filter *f = container_of(to_rcu_work(work), 378 struct cls_fl_filter, rwork); 379 380 __fl_destroy_filter(f); 381 } 382 383 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f, 384 bool rtnl_held, struct netlink_ext_ack *extack) 385 { 386 struct tc_cls_flower_offload cls_flower = {}; 387 struct tcf_block *block = tp->chain->block; 388 389 if (!rtnl_held) 390 rtnl_lock(); 391 392 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 393 cls_flower.command = TC_CLSFLOWER_DESTROY; 394 cls_flower.cookie = (unsigned long) f; 395 396 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 397 spin_lock(&tp->lock); 398 list_del_init(&f->hw_list); 399 tcf_block_offload_dec(block, &f->flags); 400 spin_unlock(&tp->lock); 401 402 if (!rtnl_held) 403 rtnl_unlock(); 404 } 405 406 static int fl_hw_replace_filter(struct tcf_proto *tp, 407 struct cls_fl_filter *f, bool rtnl_held, 408 struct netlink_ext_ack *extack) 409 { 410 struct cls_fl_head *head = fl_head_dereference(tp); 411 struct tc_cls_flower_offload cls_flower = {}; 412 struct tcf_block *block = tp->chain->block; 413 bool skip_sw = tc_skip_sw(f->flags); 414 int err = 0; 415 416 if (!rtnl_held) 417 rtnl_lock(); 418 419 cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 420 if (!cls_flower.rule) { 421 err = -ENOMEM; 422 goto errout; 423 } 424 425 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 426 cls_flower.command = TC_CLSFLOWER_REPLACE; 427 cls_flower.cookie = (unsigned long) f; 428 cls_flower.rule->match.dissector = &f->mask->dissector; 429 cls_flower.rule->match.mask = &f->mask->key; 430 cls_flower.rule->match.key = &f->mkey; 431 cls_flower.classid = f->res.classid; 432 433 err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 434 if (err) { 435 kfree(cls_flower.rule); 436 if (skip_sw) 437 NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 438 else 439 err = 0; 440 goto errout; 441 } 442 443 err = tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, skip_sw); 444 kfree(cls_flower.rule); 445 446 if (err < 0) { 447 fl_hw_destroy_filter(tp, f, true, NULL); 448 goto errout; 449 } else if (err > 0) { 450 f->in_hw_count = err; 451 err = 0; 452 spin_lock(&tp->lock); 453 tcf_block_offload_inc(block, &f->flags); 454 spin_unlock(&tp->lock); 455 } 456 457 if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW)) { 458 err = -EINVAL; 459 goto errout; 460 } 461 462 spin_lock(&tp->lock); 463 list_add(&f->hw_list, &head->hw_filters); 464 spin_unlock(&tp->lock); 465 errout: 466 if (!rtnl_held) 467 rtnl_unlock(); 468 469 return err; 470 } 471 472 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f, 473 bool rtnl_held) 474 { 475 struct tc_cls_flower_offload cls_flower = {}; 476 struct tcf_block *block = tp->chain->block; 477 478 if (!rtnl_held) 479 rtnl_lock(); 480 481 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL); 482 cls_flower.command = TC_CLSFLOWER_STATS; 483 cls_flower.cookie = (unsigned long) f; 484 cls_flower.classid = f->res.classid; 485 486 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 487 488 tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes, 489 cls_flower.stats.pkts, 490 cls_flower.stats.lastused); 491 492 if (!rtnl_held) 493 rtnl_unlock(); 494 } 495 496 static void __fl_put(struct cls_fl_filter *f) 497 { 498 if (!refcount_dec_and_test(&f->refcnt)) 499 return; 500 501 if (tcf_exts_get_net(&f->exts)) 502 tcf_queue_work(&f->rwork, fl_destroy_filter_work); 503 else 504 __fl_destroy_filter(f); 505 } 506 507 static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle) 508 { 509 struct cls_fl_filter *f; 510 511 rcu_read_lock(); 512 f = idr_find(&head->handle_idr, handle); 513 if (f && !refcount_inc_not_zero(&f->refcnt)) 514 f = NULL; 515 rcu_read_unlock(); 516 517 return f; 518 } 519 520 static struct cls_fl_filter *fl_get_next_filter(struct tcf_proto *tp, 521 unsigned long *handle) 522 { 523 struct cls_fl_head *head = fl_head_dereference(tp); 524 struct cls_fl_filter *f; 525 526 rcu_read_lock(); 527 while ((f = idr_get_next_ul(&head->handle_idr, handle))) { 528 /* don't return filters that are being deleted */ 529 if (refcount_inc_not_zero(&f->refcnt)) 530 break; 531 ++(*handle); 532 } 533 rcu_read_unlock(); 534 535 return f; 536 } 537 538 static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f, 539 bool *last, bool rtnl_held, 540 struct netlink_ext_ack *extack) 541 { 542 struct cls_fl_head *head = fl_head_dereference(tp); 543 544 *last = false; 545 546 spin_lock(&tp->lock); 547 if (f->deleted) { 548 spin_unlock(&tp->lock); 549 return -ENOENT; 550 } 551 552 f->deleted = true; 553 rhashtable_remove_fast(&f->mask->ht, &f->ht_node, 554 f->mask->filter_ht_params); 555 idr_remove(&head->handle_idr, f->handle); 556 list_del_rcu(&f->list); 557 spin_unlock(&tp->lock); 558 559 *last = fl_mask_put(head, f->mask); 560 if (!tc_skip_hw(f->flags)) 561 fl_hw_destroy_filter(tp, f, rtnl_held, extack); 562 tcf_unbind_filter(tp, &f->res); 563 __fl_put(f); 564 565 return 0; 566 } 567 568 static void fl_destroy_sleepable(struct work_struct *work) 569 { 570 struct cls_fl_head *head = container_of(to_rcu_work(work), 571 struct cls_fl_head, 572 rwork); 573 574 rhashtable_destroy(&head->ht); 575 kfree(head); 576 module_put(THIS_MODULE); 577 } 578 579 static void fl_destroy(struct tcf_proto *tp, bool rtnl_held, 580 struct netlink_ext_ack *extack) 581 { 582 struct cls_fl_head *head = fl_head_dereference(tp); 583 struct fl_flow_mask *mask, *next_mask; 584 struct cls_fl_filter *f, *next; 585 bool last; 586 587 list_for_each_entry_safe(mask, next_mask, &head->masks, list) { 588 list_for_each_entry_safe(f, next, &mask->filters, list) { 589 __fl_delete(tp, f, &last, rtnl_held, extack); 590 if (last) 591 break; 592 } 593 } 594 idr_destroy(&head->handle_idr); 595 596 __module_get(THIS_MODULE); 597 tcf_queue_work(&head->rwork, fl_destroy_sleepable); 598 } 599 600 static void fl_put(struct tcf_proto *tp, void *arg) 601 { 602 struct cls_fl_filter *f = arg; 603 604 __fl_put(f); 605 } 606 607 static void *fl_get(struct tcf_proto *tp, u32 handle) 608 { 609 struct cls_fl_head *head = fl_head_dereference(tp); 610 611 return __fl_get(head, handle); 612 } 613 614 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 615 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 616 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 617 [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 618 .len = IFNAMSIZ }, 619 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 620 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 621 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 622 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 623 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 624 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 625 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 626 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 627 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 628 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 629 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 630 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 631 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 632 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 633 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 634 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 635 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 636 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 637 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 638 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 639 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 640 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 641 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 642 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 643 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 644 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 645 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 646 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 647 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 648 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 649 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, 650 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, 651 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, 652 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, 653 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, 654 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, 655 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, 656 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, 657 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, 658 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, 659 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, 660 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, 661 [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 }, 662 [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 }, 663 [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 }, 664 [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 }, 665 [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 }, 666 [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 }, 667 [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 }, 668 [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 }, 669 [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 }, 670 [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 }, 671 [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 }, 672 [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 }, 673 [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 }, 674 [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 }, 675 [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 }, 676 [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 }, 677 [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN }, 678 [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN }, 679 [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN }, 680 [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN }, 681 [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 }, 682 [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 }, 683 [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 }, 684 [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 }, 685 [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 }, 686 [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 }, 687 [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 }, 688 [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 }, 689 [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 }, 690 [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 }, 691 [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 }, 692 [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 }, 693 [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 }, 694 [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 }, 695 [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 }, 696 [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 }, 697 [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 }, 698 [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED }, 699 [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED }, 700 }; 701 702 static const struct nla_policy 703 enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = { 704 [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED }, 705 }; 706 707 static const struct nla_policy 708 geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = { 709 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 }, 710 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 }, 711 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY, 712 .len = 128 }, 713 }; 714 715 static void fl_set_key_val(struct nlattr **tb, 716 void *val, int val_type, 717 void *mask, int mask_type, int len) 718 { 719 if (!tb[val_type]) 720 return; 721 memcpy(val, nla_data(tb[val_type]), len); 722 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 723 memset(mask, 0xff, len); 724 else 725 memcpy(mask, nla_data(tb[mask_type]), len); 726 } 727 728 static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, 729 struct fl_flow_key *mask) 730 { 731 fl_set_key_val(tb, &key->tp_min.dst, 732 TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_min.dst, 733 TCA_FLOWER_UNSPEC, sizeof(key->tp_min.dst)); 734 fl_set_key_val(tb, &key->tp_max.dst, 735 TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_max.dst, 736 TCA_FLOWER_UNSPEC, sizeof(key->tp_max.dst)); 737 fl_set_key_val(tb, &key->tp_min.src, 738 TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_min.src, 739 TCA_FLOWER_UNSPEC, sizeof(key->tp_min.src)); 740 fl_set_key_val(tb, &key->tp_max.src, 741 TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_max.src, 742 TCA_FLOWER_UNSPEC, sizeof(key->tp_max.src)); 743 744 if ((mask->tp_min.dst && mask->tp_max.dst && 745 htons(key->tp_max.dst) <= htons(key->tp_min.dst)) || 746 (mask->tp_min.src && mask->tp_max.src && 747 htons(key->tp_max.src) <= htons(key->tp_min.src))) 748 return -EINVAL; 749 750 return 0; 751 } 752 753 static int fl_set_key_mpls(struct nlattr **tb, 754 struct flow_dissector_key_mpls *key_val, 755 struct flow_dissector_key_mpls *key_mask) 756 { 757 if (tb[TCA_FLOWER_KEY_MPLS_TTL]) { 758 key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]); 759 key_mask->mpls_ttl = MPLS_TTL_MASK; 760 } 761 if (tb[TCA_FLOWER_KEY_MPLS_BOS]) { 762 u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]); 763 764 if (bos & ~MPLS_BOS_MASK) 765 return -EINVAL; 766 key_val->mpls_bos = bos; 767 key_mask->mpls_bos = MPLS_BOS_MASK; 768 } 769 if (tb[TCA_FLOWER_KEY_MPLS_TC]) { 770 u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]); 771 772 if (tc & ~MPLS_TC_MASK) 773 return -EINVAL; 774 key_val->mpls_tc = tc; 775 key_mask->mpls_tc = MPLS_TC_MASK; 776 } 777 if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) { 778 u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]); 779 780 if (label & ~MPLS_LABEL_MASK) 781 return -EINVAL; 782 key_val->mpls_label = label; 783 key_mask->mpls_label = MPLS_LABEL_MASK; 784 } 785 return 0; 786 } 787 788 static void fl_set_key_vlan(struct nlattr **tb, 789 __be16 ethertype, 790 int vlan_id_key, int vlan_prio_key, 791 struct flow_dissector_key_vlan *key_val, 792 struct flow_dissector_key_vlan *key_mask) 793 { 794 #define VLAN_PRIORITY_MASK 0x7 795 796 if (tb[vlan_id_key]) { 797 key_val->vlan_id = 798 nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK; 799 key_mask->vlan_id = VLAN_VID_MASK; 800 } 801 if (tb[vlan_prio_key]) { 802 key_val->vlan_priority = 803 nla_get_u8(tb[vlan_prio_key]) & 804 VLAN_PRIORITY_MASK; 805 key_mask->vlan_priority = VLAN_PRIORITY_MASK; 806 } 807 key_val->vlan_tpid = ethertype; 808 key_mask->vlan_tpid = cpu_to_be16(~0); 809 } 810 811 static void fl_set_key_flag(u32 flower_key, u32 flower_mask, 812 u32 *dissector_key, u32 *dissector_mask, 813 u32 flower_flag_bit, u32 dissector_flag_bit) 814 { 815 if (flower_mask & flower_flag_bit) { 816 *dissector_mask |= dissector_flag_bit; 817 if (flower_key & flower_flag_bit) 818 *dissector_key |= dissector_flag_bit; 819 } 820 } 821 822 static int fl_set_key_flags(struct nlattr **tb, 823 u32 *flags_key, u32 *flags_mask) 824 { 825 u32 key, mask; 826 827 /* mask is mandatory for flags */ 828 if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) 829 return -EINVAL; 830 831 key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS])); 832 mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK])); 833 834 *flags_key = 0; 835 *flags_mask = 0; 836 837 fl_set_key_flag(key, mask, flags_key, flags_mask, 838 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 839 fl_set_key_flag(key, mask, flags_key, flags_mask, 840 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 841 FLOW_DIS_FIRST_FRAG); 842 843 return 0; 844 } 845 846 static void fl_set_key_ip(struct nlattr **tb, bool encap, 847 struct flow_dissector_key_ip *key, 848 struct flow_dissector_key_ip *mask) 849 { 850 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 851 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 852 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 853 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 854 855 fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)); 856 fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)); 857 } 858 859 static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key, 860 int depth, int option_len, 861 struct netlink_ext_ack *extack) 862 { 863 struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1]; 864 struct nlattr *class = NULL, *type = NULL, *data = NULL; 865 struct geneve_opt *opt; 866 int err, data_len = 0; 867 868 if (option_len > sizeof(struct geneve_opt)) 869 data_len = option_len - sizeof(struct geneve_opt); 870 871 opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len]; 872 memset(opt, 0xff, option_len); 873 opt->length = data_len / 4; 874 opt->r1 = 0; 875 opt->r2 = 0; 876 opt->r3 = 0; 877 878 /* If no mask has been prodived we assume an exact match. */ 879 if (!depth) 880 return sizeof(struct geneve_opt) + data_len; 881 882 if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) { 883 NL_SET_ERR_MSG(extack, "Non-geneve option type for mask"); 884 return -EINVAL; 885 } 886 887 err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX, 888 nla, geneve_opt_policy, extack); 889 if (err < 0) 890 return err; 891 892 /* We are not allowed to omit any of CLASS, TYPE or DATA 893 * fields from the key. 894 */ 895 if (!option_len && 896 (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] || 897 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] || 898 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) { 899 NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data"); 900 return -EINVAL; 901 } 902 903 /* Omitting any of CLASS, TYPE or DATA fields is allowed 904 * for the mask. 905 */ 906 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) { 907 int new_len = key->enc_opts.len; 908 909 data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]; 910 data_len = nla_len(data); 911 if (data_len < 4) { 912 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long"); 913 return -ERANGE; 914 } 915 if (data_len % 4) { 916 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long"); 917 return -ERANGE; 918 } 919 920 new_len += sizeof(struct geneve_opt) + data_len; 921 BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX); 922 if (new_len > FLOW_DIS_TUN_OPTS_MAX) { 923 NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size"); 924 return -ERANGE; 925 } 926 opt->length = data_len / 4; 927 memcpy(opt->opt_data, nla_data(data), data_len); 928 } 929 930 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) { 931 class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]; 932 opt->opt_class = nla_get_be16(class); 933 } 934 935 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) { 936 type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]; 937 opt->type = nla_get_u8(type); 938 } 939 940 return sizeof(struct geneve_opt) + data_len; 941 } 942 943 static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key, 944 struct fl_flow_key *mask, 945 struct netlink_ext_ack *extack) 946 { 947 const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL; 948 int err, option_len, key_depth, msk_depth = 0; 949 950 err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS], 951 TCA_FLOWER_KEY_ENC_OPTS_MAX, 952 enc_opts_policy, extack); 953 if (err) 954 return err; 955 956 nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]); 957 958 if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) { 959 err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK], 960 TCA_FLOWER_KEY_ENC_OPTS_MAX, 961 enc_opts_policy, extack); 962 if (err) 963 return err; 964 965 nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 966 msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 967 } 968 969 nla_for_each_attr(nla_opt_key, nla_enc_key, 970 nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) { 971 switch (nla_type(nla_opt_key)) { 972 case TCA_FLOWER_KEY_ENC_OPTS_GENEVE: 973 option_len = 0; 974 key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 975 option_len = fl_set_geneve_opt(nla_opt_key, key, 976 key_depth, option_len, 977 extack); 978 if (option_len < 0) 979 return option_len; 980 981 key->enc_opts.len += option_len; 982 /* At the same time we need to parse through the mask 983 * in order to verify exact and mask attribute lengths. 984 */ 985 mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 986 option_len = fl_set_geneve_opt(nla_opt_msk, mask, 987 msk_depth, option_len, 988 extack); 989 if (option_len < 0) 990 return option_len; 991 992 mask->enc_opts.len += option_len; 993 if (key->enc_opts.len != mask->enc_opts.len) { 994 NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 995 return -EINVAL; 996 } 997 998 if (msk_depth) 999 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 1000 break; 1001 default: 1002 NL_SET_ERR_MSG(extack, "Unknown tunnel option type"); 1003 return -EINVAL; 1004 } 1005 } 1006 1007 return 0; 1008 } 1009 1010 static int fl_set_key(struct net *net, struct nlattr **tb, 1011 struct fl_flow_key *key, struct fl_flow_key *mask, 1012 struct netlink_ext_ack *extack) 1013 { 1014 __be16 ethertype; 1015 int ret = 0; 1016 #ifdef CONFIG_NET_CLS_IND 1017 if (tb[TCA_FLOWER_INDEV]) { 1018 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack); 1019 if (err < 0) 1020 return err; 1021 key->indev_ifindex = err; 1022 mask->indev_ifindex = 0xffffffff; 1023 } 1024 #endif 1025 1026 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 1027 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 1028 sizeof(key->eth.dst)); 1029 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 1030 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 1031 sizeof(key->eth.src)); 1032 1033 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 1034 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 1035 1036 if (eth_type_vlan(ethertype)) { 1037 fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID, 1038 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, 1039 &mask->vlan); 1040 1041 if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) { 1042 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]); 1043 if (eth_type_vlan(ethertype)) { 1044 fl_set_key_vlan(tb, ethertype, 1045 TCA_FLOWER_KEY_CVLAN_ID, 1046 TCA_FLOWER_KEY_CVLAN_PRIO, 1047 &key->cvlan, &mask->cvlan); 1048 fl_set_key_val(tb, &key->basic.n_proto, 1049 TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 1050 &mask->basic.n_proto, 1051 TCA_FLOWER_UNSPEC, 1052 sizeof(key->basic.n_proto)); 1053 } else { 1054 key->basic.n_proto = ethertype; 1055 mask->basic.n_proto = cpu_to_be16(~0); 1056 } 1057 } 1058 } else { 1059 key->basic.n_proto = ethertype; 1060 mask->basic.n_proto = cpu_to_be16(~0); 1061 } 1062 } 1063 1064 if (key->basic.n_proto == htons(ETH_P_IP) || 1065 key->basic.n_proto == htons(ETH_P_IPV6)) { 1066 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 1067 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 1068 sizeof(key->basic.ip_proto)); 1069 fl_set_key_ip(tb, false, &key->ip, &mask->ip); 1070 } 1071 1072 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 1073 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1074 mask->control.addr_type = ~0; 1075 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 1076 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 1077 sizeof(key->ipv4.src)); 1078 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 1079 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 1080 sizeof(key->ipv4.dst)); 1081 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 1082 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1083 mask->control.addr_type = ~0; 1084 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 1085 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 1086 sizeof(key->ipv6.src)); 1087 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 1088 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 1089 sizeof(key->ipv6.dst)); 1090 } 1091 1092 if (key->basic.ip_proto == IPPROTO_TCP) { 1093 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 1094 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 1095 sizeof(key->tp.src)); 1096 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 1097 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 1098 sizeof(key->tp.dst)); 1099 fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 1100 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 1101 sizeof(key->tcp.flags)); 1102 } else if (key->basic.ip_proto == IPPROTO_UDP) { 1103 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 1104 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 1105 sizeof(key->tp.src)); 1106 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 1107 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 1108 sizeof(key->tp.dst)); 1109 } else if (key->basic.ip_proto == IPPROTO_SCTP) { 1110 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 1111 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 1112 sizeof(key->tp.src)); 1113 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 1114 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 1115 sizeof(key->tp.dst)); 1116 } else if (key->basic.n_proto == htons(ETH_P_IP) && 1117 key->basic.ip_proto == IPPROTO_ICMP) { 1118 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE, 1119 &mask->icmp.type, 1120 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 1121 sizeof(key->icmp.type)); 1122 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE, 1123 &mask->icmp.code, 1124 TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 1125 sizeof(key->icmp.code)); 1126 } else if (key->basic.n_proto == htons(ETH_P_IPV6) && 1127 key->basic.ip_proto == IPPROTO_ICMPV6) { 1128 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE, 1129 &mask->icmp.type, 1130 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 1131 sizeof(key->icmp.type)); 1132 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE, 1133 &mask->icmp.code, 1134 TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 1135 sizeof(key->icmp.code)); 1136 } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) || 1137 key->basic.n_proto == htons(ETH_P_MPLS_MC)) { 1138 ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls); 1139 if (ret) 1140 return ret; 1141 } else if (key->basic.n_proto == htons(ETH_P_ARP) || 1142 key->basic.n_proto == htons(ETH_P_RARP)) { 1143 fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP, 1144 &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK, 1145 sizeof(key->arp.sip)); 1146 fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP, 1147 &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK, 1148 sizeof(key->arp.tip)); 1149 fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP, 1150 &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK, 1151 sizeof(key->arp.op)); 1152 fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 1153 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 1154 sizeof(key->arp.sha)); 1155 fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 1156 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 1157 sizeof(key->arp.tha)); 1158 } 1159 1160 if (key->basic.ip_proto == IPPROTO_TCP || 1161 key->basic.ip_proto == IPPROTO_UDP || 1162 key->basic.ip_proto == IPPROTO_SCTP) { 1163 ret = fl_set_key_port_range(tb, key, mask); 1164 if (ret) 1165 return ret; 1166 } 1167 1168 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 1169 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 1170 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1171 mask->enc_control.addr_type = ~0; 1172 fl_set_key_val(tb, &key->enc_ipv4.src, 1173 TCA_FLOWER_KEY_ENC_IPV4_SRC, 1174 &mask->enc_ipv4.src, 1175 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 1176 sizeof(key->enc_ipv4.src)); 1177 fl_set_key_val(tb, &key->enc_ipv4.dst, 1178 TCA_FLOWER_KEY_ENC_IPV4_DST, 1179 &mask->enc_ipv4.dst, 1180 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 1181 sizeof(key->enc_ipv4.dst)); 1182 } 1183 1184 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 1185 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 1186 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1187 mask->enc_control.addr_type = ~0; 1188 fl_set_key_val(tb, &key->enc_ipv6.src, 1189 TCA_FLOWER_KEY_ENC_IPV6_SRC, 1190 &mask->enc_ipv6.src, 1191 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 1192 sizeof(key->enc_ipv6.src)); 1193 fl_set_key_val(tb, &key->enc_ipv6.dst, 1194 TCA_FLOWER_KEY_ENC_IPV6_DST, 1195 &mask->enc_ipv6.dst, 1196 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 1197 sizeof(key->enc_ipv6.dst)); 1198 } 1199 1200 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 1201 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC, 1202 sizeof(key->enc_key_id.keyid)); 1203 1204 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 1205 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 1206 sizeof(key->enc_tp.src)); 1207 1208 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 1209 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 1210 sizeof(key->enc_tp.dst)); 1211 1212 fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip); 1213 1214 if (tb[TCA_FLOWER_KEY_ENC_OPTS]) { 1215 ret = fl_set_enc_opt(tb, key, mask, extack); 1216 if (ret) 1217 return ret; 1218 } 1219 1220 if (tb[TCA_FLOWER_KEY_FLAGS]) 1221 ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags); 1222 1223 return ret; 1224 } 1225 1226 static void fl_mask_copy(struct fl_flow_mask *dst, 1227 struct fl_flow_mask *src) 1228 { 1229 const void *psrc = fl_key_get_start(&src->key, src); 1230 void *pdst = fl_key_get_start(&dst->key, src); 1231 1232 memcpy(pdst, psrc, fl_mask_range(src)); 1233 dst->range = src->range; 1234 } 1235 1236 static const struct rhashtable_params fl_ht_params = { 1237 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 1238 .head_offset = offsetof(struct cls_fl_filter, ht_node), 1239 .automatic_shrinking = true, 1240 }; 1241 1242 static int fl_init_mask_hashtable(struct fl_flow_mask *mask) 1243 { 1244 mask->filter_ht_params = fl_ht_params; 1245 mask->filter_ht_params.key_len = fl_mask_range(mask); 1246 mask->filter_ht_params.key_offset += mask->range.start; 1247 1248 return rhashtable_init(&mask->ht, &mask->filter_ht_params); 1249 } 1250 1251 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 1252 #define FL_KEY_MEMBER_SIZE(member) FIELD_SIZEOF(struct fl_flow_key, member) 1253 1254 #define FL_KEY_IS_MASKED(mask, member) \ 1255 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 1256 0, FL_KEY_MEMBER_SIZE(member)) \ 1257 1258 #define FL_KEY_SET(keys, cnt, id, member) \ 1259 do { \ 1260 keys[cnt].key_id = id; \ 1261 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 1262 cnt++; \ 1263 } while(0); 1264 1265 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 1266 do { \ 1267 if (FL_KEY_IS_MASKED(mask, member)) \ 1268 FL_KEY_SET(keys, cnt, id, member); \ 1269 } while(0); 1270 1271 static void fl_init_dissector(struct flow_dissector *dissector, 1272 struct fl_flow_key *mask) 1273 { 1274 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 1275 size_t cnt = 0; 1276 1277 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 1278 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 1279 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1280 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 1281 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1282 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 1283 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1284 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 1285 if (FL_KEY_IS_MASKED(mask, tp) || 1286 FL_KEY_IS_MASKED(mask, tp_min) || FL_KEY_IS_MASKED(mask, tp_max)) 1287 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_PORTS, tp); 1288 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1289 FLOW_DISSECTOR_KEY_IP, ip); 1290 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1291 FLOW_DISSECTOR_KEY_TCP, tcp); 1292 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1293 FLOW_DISSECTOR_KEY_ICMP, icmp); 1294 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1295 FLOW_DISSECTOR_KEY_ARP, arp); 1296 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1297 FLOW_DISSECTOR_KEY_MPLS, mpls); 1298 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1299 FLOW_DISSECTOR_KEY_VLAN, vlan); 1300 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1301 FLOW_DISSECTOR_KEY_CVLAN, cvlan); 1302 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1303 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 1304 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1305 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4); 1306 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1307 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6); 1308 if (FL_KEY_IS_MASKED(mask, enc_ipv4) || 1309 FL_KEY_IS_MASKED(mask, enc_ipv6)) 1310 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL, 1311 enc_control); 1312 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1313 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp); 1314 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1315 FLOW_DISSECTOR_KEY_ENC_IP, enc_ip); 1316 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1317 FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts); 1318 1319 skb_flow_dissector_init(dissector, keys, cnt); 1320 } 1321 1322 static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, 1323 struct fl_flow_mask *mask) 1324 { 1325 struct fl_flow_mask *newmask; 1326 int err; 1327 1328 newmask = kzalloc(sizeof(*newmask), GFP_KERNEL); 1329 if (!newmask) 1330 return ERR_PTR(-ENOMEM); 1331 1332 fl_mask_copy(newmask, mask); 1333 1334 if ((newmask->key.tp_min.dst && newmask->key.tp_max.dst) || 1335 (newmask->key.tp_min.src && newmask->key.tp_max.src)) 1336 newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE; 1337 1338 err = fl_init_mask_hashtable(newmask); 1339 if (err) 1340 goto errout_free; 1341 1342 fl_init_dissector(&newmask->dissector, &newmask->key); 1343 1344 INIT_LIST_HEAD_RCU(&newmask->filters); 1345 1346 refcount_set(&newmask->refcnt, 1); 1347 err = rhashtable_replace_fast(&head->ht, &mask->ht_node, 1348 &newmask->ht_node, mask_ht_params); 1349 if (err) 1350 goto errout_destroy; 1351 1352 /* Wait until any potential concurrent users of mask are finished */ 1353 synchronize_rcu(); 1354 1355 spin_lock(&head->masks_lock); 1356 list_add_tail_rcu(&newmask->list, &head->masks); 1357 spin_unlock(&head->masks_lock); 1358 1359 return newmask; 1360 1361 errout_destroy: 1362 rhashtable_destroy(&newmask->ht); 1363 errout_free: 1364 kfree(newmask); 1365 1366 return ERR_PTR(err); 1367 } 1368 1369 static int fl_check_assign_mask(struct cls_fl_head *head, 1370 struct cls_fl_filter *fnew, 1371 struct cls_fl_filter *fold, 1372 struct fl_flow_mask *mask) 1373 { 1374 struct fl_flow_mask *newmask; 1375 int ret = 0; 1376 1377 rcu_read_lock(); 1378 1379 /* Insert mask as temporary node to prevent concurrent creation of mask 1380 * with same key. Any concurrent lookups with same key will return 1381 * -EAGAIN because mask's refcnt is zero. It is safe to insert 1382 * stack-allocated 'mask' to masks hash table because we call 1383 * synchronize_rcu() before returning from this function (either in case 1384 * of error or after replacing it with heap-allocated mask in 1385 * fl_create_new_mask()). 1386 */ 1387 fnew->mask = rhashtable_lookup_get_insert_fast(&head->ht, 1388 &mask->ht_node, 1389 mask_ht_params); 1390 if (!fnew->mask) { 1391 rcu_read_unlock(); 1392 1393 if (fold) { 1394 ret = -EINVAL; 1395 goto errout_cleanup; 1396 } 1397 1398 newmask = fl_create_new_mask(head, mask); 1399 if (IS_ERR(newmask)) { 1400 ret = PTR_ERR(newmask); 1401 goto errout_cleanup; 1402 } 1403 1404 fnew->mask = newmask; 1405 return 0; 1406 } else if (IS_ERR(fnew->mask)) { 1407 ret = PTR_ERR(fnew->mask); 1408 } else if (fold && fold->mask != fnew->mask) { 1409 ret = -EINVAL; 1410 } else if (!refcount_inc_not_zero(&fnew->mask->refcnt)) { 1411 /* Mask was deleted concurrently, try again */ 1412 ret = -EAGAIN; 1413 } 1414 rcu_read_unlock(); 1415 return ret; 1416 1417 errout_cleanup: 1418 rhashtable_remove_fast(&head->ht, &mask->ht_node, 1419 mask_ht_params); 1420 /* Wait until any potential concurrent users of mask are finished */ 1421 synchronize_rcu(); 1422 return ret; 1423 } 1424 1425 static int fl_set_parms(struct net *net, struct tcf_proto *tp, 1426 struct cls_fl_filter *f, struct fl_flow_mask *mask, 1427 unsigned long base, struct nlattr **tb, 1428 struct nlattr *est, bool ovr, 1429 struct fl_flow_tmplt *tmplt, bool rtnl_held, 1430 struct netlink_ext_ack *extack) 1431 { 1432 int err; 1433 1434 err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, rtnl_held, 1435 extack); 1436 if (err < 0) 1437 return err; 1438 1439 if (tb[TCA_FLOWER_CLASSID]) { 1440 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 1441 if (!rtnl_held) 1442 rtnl_lock(); 1443 tcf_bind_filter(tp, &f->res, base); 1444 if (!rtnl_held) 1445 rtnl_unlock(); 1446 } 1447 1448 err = fl_set_key(net, tb, &f->key, &mask->key, extack); 1449 if (err) 1450 return err; 1451 1452 fl_mask_update_range(mask); 1453 fl_set_masked_key(&f->mkey, &f->key, mask); 1454 1455 if (!fl_mask_fits_tmplt(tmplt, mask)) { 1456 NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template"); 1457 return -EINVAL; 1458 } 1459 1460 return 0; 1461 } 1462 1463 static int fl_ht_insert_unique(struct cls_fl_filter *fnew, 1464 struct cls_fl_filter *fold, 1465 bool *in_ht) 1466 { 1467 struct fl_flow_mask *mask = fnew->mask; 1468 int err; 1469 1470 err = rhashtable_lookup_insert_fast(&mask->ht, 1471 &fnew->ht_node, 1472 mask->filter_ht_params); 1473 if (err) { 1474 *in_ht = false; 1475 /* It is okay if filter with same key exists when 1476 * overwriting. 1477 */ 1478 return fold && err == -EEXIST ? 0 : err; 1479 } 1480 1481 *in_ht = true; 1482 return 0; 1483 } 1484 1485 static int fl_change(struct net *net, struct sk_buff *in_skb, 1486 struct tcf_proto *tp, unsigned long base, 1487 u32 handle, struct nlattr **tca, 1488 void **arg, bool ovr, bool rtnl_held, 1489 struct netlink_ext_ack *extack) 1490 { 1491 struct cls_fl_head *head = fl_head_dereference(tp); 1492 struct cls_fl_filter *fold = *arg; 1493 struct cls_fl_filter *fnew; 1494 struct fl_flow_mask *mask; 1495 struct nlattr **tb; 1496 bool in_ht; 1497 int err; 1498 1499 if (!tca[TCA_OPTIONS]) { 1500 err = -EINVAL; 1501 goto errout_fold; 1502 } 1503 1504 mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL); 1505 if (!mask) { 1506 err = -ENOBUFS; 1507 goto errout_fold; 1508 } 1509 1510 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 1511 if (!tb) { 1512 err = -ENOBUFS; 1513 goto errout_mask_alloc; 1514 } 1515 1516 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], 1517 fl_policy, NULL); 1518 if (err < 0) 1519 goto errout_tb; 1520 1521 if (fold && handle && fold->handle != handle) { 1522 err = -EINVAL; 1523 goto errout_tb; 1524 } 1525 1526 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 1527 if (!fnew) { 1528 err = -ENOBUFS; 1529 goto errout_tb; 1530 } 1531 INIT_LIST_HEAD(&fnew->hw_list); 1532 refcount_set(&fnew->refcnt, 1); 1533 1534 err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0); 1535 if (err < 0) 1536 goto errout; 1537 1538 if (tb[TCA_FLOWER_FLAGS]) { 1539 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 1540 1541 if (!tc_flags_valid(fnew->flags)) { 1542 err = -EINVAL; 1543 goto errout; 1544 } 1545 } 1546 1547 err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr, 1548 tp->chain->tmplt_priv, rtnl_held, extack); 1549 if (err) 1550 goto errout; 1551 1552 err = fl_check_assign_mask(head, fnew, fold, mask); 1553 if (err) 1554 goto errout; 1555 1556 err = fl_ht_insert_unique(fnew, fold, &in_ht); 1557 if (err) 1558 goto errout_mask; 1559 1560 if (!tc_skip_hw(fnew->flags)) { 1561 err = fl_hw_replace_filter(tp, fnew, rtnl_held, extack); 1562 if (err) 1563 goto errout_ht; 1564 } 1565 1566 if (!tc_in_hw(fnew->flags)) 1567 fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW; 1568 1569 spin_lock(&tp->lock); 1570 1571 /* tp was deleted concurrently. -EAGAIN will cause caller to lookup 1572 * proto again or create new one, if necessary. 1573 */ 1574 if (tp->deleting) { 1575 err = -EAGAIN; 1576 goto errout_hw; 1577 } 1578 1579 if (fold) { 1580 /* Fold filter was deleted concurrently. Retry lookup. */ 1581 if (fold->deleted) { 1582 err = -EAGAIN; 1583 goto errout_hw; 1584 } 1585 1586 fnew->handle = handle; 1587 1588 if (!in_ht) { 1589 struct rhashtable_params params = 1590 fnew->mask->filter_ht_params; 1591 1592 err = rhashtable_insert_fast(&fnew->mask->ht, 1593 &fnew->ht_node, 1594 params); 1595 if (err) 1596 goto errout_hw; 1597 in_ht = true; 1598 } 1599 1600 refcount_inc(&fnew->refcnt); 1601 rhashtable_remove_fast(&fold->mask->ht, 1602 &fold->ht_node, 1603 fold->mask->filter_ht_params); 1604 idr_replace(&head->handle_idr, fnew, fnew->handle); 1605 list_replace_rcu(&fold->list, &fnew->list); 1606 fold->deleted = true; 1607 1608 spin_unlock(&tp->lock); 1609 1610 fl_mask_put(head, fold->mask); 1611 if (!tc_skip_hw(fold->flags)) 1612 fl_hw_destroy_filter(tp, fold, rtnl_held, NULL); 1613 tcf_unbind_filter(tp, &fold->res); 1614 /* Caller holds reference to fold, so refcnt is always > 0 1615 * after this. 1616 */ 1617 refcount_dec(&fold->refcnt); 1618 __fl_put(fold); 1619 } else { 1620 if (handle) { 1621 /* user specifies a handle and it doesn't exist */ 1622 err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1623 handle, GFP_ATOMIC); 1624 1625 /* Filter with specified handle was concurrently 1626 * inserted after initial check in cls_api. This is not 1627 * necessarily an error if NLM_F_EXCL is not set in 1628 * message flags. Returning EAGAIN will cause cls_api to 1629 * try to update concurrently inserted rule. 1630 */ 1631 if (err == -ENOSPC) 1632 err = -EAGAIN; 1633 } else { 1634 handle = 1; 1635 err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 1636 INT_MAX, GFP_ATOMIC); 1637 } 1638 if (err) 1639 goto errout_hw; 1640 1641 refcount_inc(&fnew->refcnt); 1642 fnew->handle = handle; 1643 list_add_tail_rcu(&fnew->list, &fnew->mask->filters); 1644 spin_unlock(&tp->lock); 1645 } 1646 1647 *arg = fnew; 1648 1649 kfree(tb); 1650 kfree(mask); 1651 return 0; 1652 1653 errout_ht: 1654 spin_lock(&tp->lock); 1655 errout_hw: 1656 fnew->deleted = true; 1657 spin_unlock(&tp->lock); 1658 if (!tc_skip_hw(fnew->flags)) 1659 fl_hw_destroy_filter(tp, fnew, rtnl_held, NULL); 1660 if (in_ht) 1661 rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node, 1662 fnew->mask->filter_ht_params); 1663 errout_mask: 1664 fl_mask_put(head, fnew->mask); 1665 errout: 1666 __fl_put(fnew); 1667 errout_tb: 1668 kfree(tb); 1669 errout_mask_alloc: 1670 kfree(mask); 1671 errout_fold: 1672 if (fold) 1673 __fl_put(fold); 1674 return err; 1675 } 1676 1677 static int fl_delete(struct tcf_proto *tp, void *arg, bool *last, 1678 bool rtnl_held, struct netlink_ext_ack *extack) 1679 { 1680 struct cls_fl_head *head = fl_head_dereference(tp); 1681 struct cls_fl_filter *f = arg; 1682 bool last_on_mask; 1683 int err = 0; 1684 1685 err = __fl_delete(tp, f, &last_on_mask, rtnl_held, extack); 1686 *last = list_empty(&head->masks); 1687 __fl_put(f); 1688 1689 return err; 1690 } 1691 1692 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg, 1693 bool rtnl_held) 1694 { 1695 struct cls_fl_filter *f; 1696 1697 arg->count = arg->skip; 1698 1699 while ((f = fl_get_next_filter(tp, &arg->cookie)) != NULL) { 1700 if (arg->fn(tp, f, arg) < 0) { 1701 __fl_put(f); 1702 arg->stop = 1; 1703 break; 1704 } 1705 __fl_put(f); 1706 arg->cookie++; 1707 arg->count++; 1708 } 1709 } 1710 1711 static struct cls_fl_filter * 1712 fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add) 1713 { 1714 struct cls_fl_head *head = fl_head_dereference(tp); 1715 1716 spin_lock(&tp->lock); 1717 if (list_empty(&head->hw_filters)) { 1718 spin_unlock(&tp->lock); 1719 return NULL; 1720 } 1721 1722 if (!f) 1723 f = list_entry(&head->hw_filters, struct cls_fl_filter, 1724 hw_list); 1725 list_for_each_entry_continue(f, &head->hw_filters, hw_list) { 1726 if (!(add && f->deleted) && refcount_inc_not_zero(&f->refcnt)) { 1727 spin_unlock(&tp->lock); 1728 return f; 1729 } 1730 } 1731 1732 spin_unlock(&tp->lock); 1733 return NULL; 1734 } 1735 1736 static int fl_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb, 1737 void *cb_priv, struct netlink_ext_ack *extack) 1738 { 1739 struct tc_cls_flower_offload cls_flower = {}; 1740 struct tcf_block *block = tp->chain->block; 1741 struct cls_fl_filter *f = NULL; 1742 int err; 1743 1744 /* hw_filters list can only be changed by hw offload functions after 1745 * obtaining rtnl lock. Make sure it is not changed while reoffload is 1746 * iterating it. 1747 */ 1748 ASSERT_RTNL(); 1749 1750 while ((f = fl_get_next_hw_filter(tp, f, add))) { 1751 cls_flower.rule = 1752 flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 1753 if (!cls_flower.rule) { 1754 __fl_put(f); 1755 return -ENOMEM; 1756 } 1757 1758 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, 1759 extack); 1760 cls_flower.command = add ? 1761 TC_CLSFLOWER_REPLACE : TC_CLSFLOWER_DESTROY; 1762 cls_flower.cookie = (unsigned long)f; 1763 cls_flower.rule->match.dissector = &f->mask->dissector; 1764 cls_flower.rule->match.mask = &f->mask->key; 1765 cls_flower.rule->match.key = &f->mkey; 1766 1767 err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 1768 if (err) { 1769 kfree(cls_flower.rule); 1770 if (tc_skip_sw(f->flags)) { 1771 NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 1772 __fl_put(f); 1773 return err; 1774 } 1775 goto next_flow; 1776 } 1777 1778 cls_flower.classid = f->res.classid; 1779 1780 err = cb(TC_SETUP_CLSFLOWER, &cls_flower, cb_priv); 1781 kfree(cls_flower.rule); 1782 1783 if (err) { 1784 if (add && tc_skip_sw(f->flags)) { 1785 __fl_put(f); 1786 return err; 1787 } 1788 goto next_flow; 1789 } 1790 1791 spin_lock(&tp->lock); 1792 tc_cls_offload_cnt_update(block, &f->in_hw_count, &f->flags, 1793 add); 1794 spin_unlock(&tp->lock); 1795 next_flow: 1796 __fl_put(f); 1797 } 1798 1799 return 0; 1800 } 1801 1802 static int fl_hw_create_tmplt(struct tcf_chain *chain, 1803 struct fl_flow_tmplt *tmplt) 1804 { 1805 struct tc_cls_flower_offload cls_flower = {}; 1806 struct tcf_block *block = chain->block; 1807 1808 cls_flower.rule = flow_rule_alloc(0); 1809 if (!cls_flower.rule) 1810 return -ENOMEM; 1811 1812 cls_flower.common.chain_index = chain->index; 1813 cls_flower.command = TC_CLSFLOWER_TMPLT_CREATE; 1814 cls_flower.cookie = (unsigned long) tmplt; 1815 cls_flower.rule->match.dissector = &tmplt->dissector; 1816 cls_flower.rule->match.mask = &tmplt->mask; 1817 cls_flower.rule->match.key = &tmplt->dummy_key; 1818 1819 /* We don't care if driver (any of them) fails to handle this 1820 * call. It serves just as a hint for it. 1821 */ 1822 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 1823 kfree(cls_flower.rule); 1824 1825 return 0; 1826 } 1827 1828 static void fl_hw_destroy_tmplt(struct tcf_chain *chain, 1829 struct fl_flow_tmplt *tmplt) 1830 { 1831 struct tc_cls_flower_offload cls_flower = {}; 1832 struct tcf_block *block = chain->block; 1833 1834 cls_flower.common.chain_index = chain->index; 1835 cls_flower.command = TC_CLSFLOWER_TMPLT_DESTROY; 1836 cls_flower.cookie = (unsigned long) tmplt; 1837 1838 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false); 1839 } 1840 1841 static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain, 1842 struct nlattr **tca, 1843 struct netlink_ext_ack *extack) 1844 { 1845 struct fl_flow_tmplt *tmplt; 1846 struct nlattr **tb; 1847 int err; 1848 1849 if (!tca[TCA_OPTIONS]) 1850 return ERR_PTR(-EINVAL); 1851 1852 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 1853 if (!tb) 1854 return ERR_PTR(-ENOBUFS); 1855 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], 1856 fl_policy, NULL); 1857 if (err) 1858 goto errout_tb; 1859 1860 tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL); 1861 if (!tmplt) { 1862 err = -ENOMEM; 1863 goto errout_tb; 1864 } 1865 tmplt->chain = chain; 1866 err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack); 1867 if (err) 1868 goto errout_tmplt; 1869 1870 fl_init_dissector(&tmplt->dissector, &tmplt->mask); 1871 1872 err = fl_hw_create_tmplt(chain, tmplt); 1873 if (err) 1874 goto errout_tmplt; 1875 1876 kfree(tb); 1877 return tmplt; 1878 1879 errout_tmplt: 1880 kfree(tmplt); 1881 errout_tb: 1882 kfree(tb); 1883 return ERR_PTR(err); 1884 } 1885 1886 static void fl_tmplt_destroy(void *tmplt_priv) 1887 { 1888 struct fl_flow_tmplt *tmplt = tmplt_priv; 1889 1890 fl_hw_destroy_tmplt(tmplt->chain, tmplt); 1891 kfree(tmplt); 1892 } 1893 1894 static int fl_dump_key_val(struct sk_buff *skb, 1895 void *val, int val_type, 1896 void *mask, int mask_type, int len) 1897 { 1898 int err; 1899 1900 if (!memchr_inv(mask, 0, len)) 1901 return 0; 1902 err = nla_put(skb, val_type, len, val); 1903 if (err) 1904 return err; 1905 if (mask_type != TCA_FLOWER_UNSPEC) { 1906 err = nla_put(skb, mask_type, len, mask); 1907 if (err) 1908 return err; 1909 } 1910 return 0; 1911 } 1912 1913 static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key, 1914 struct fl_flow_key *mask) 1915 { 1916 if (fl_dump_key_val(skb, &key->tp_min.dst, TCA_FLOWER_KEY_PORT_DST_MIN, 1917 &mask->tp_min.dst, TCA_FLOWER_UNSPEC, 1918 sizeof(key->tp_min.dst)) || 1919 fl_dump_key_val(skb, &key->tp_max.dst, TCA_FLOWER_KEY_PORT_DST_MAX, 1920 &mask->tp_max.dst, TCA_FLOWER_UNSPEC, 1921 sizeof(key->tp_max.dst)) || 1922 fl_dump_key_val(skb, &key->tp_min.src, TCA_FLOWER_KEY_PORT_SRC_MIN, 1923 &mask->tp_min.src, TCA_FLOWER_UNSPEC, 1924 sizeof(key->tp_min.src)) || 1925 fl_dump_key_val(skb, &key->tp_max.src, TCA_FLOWER_KEY_PORT_SRC_MAX, 1926 &mask->tp_max.src, TCA_FLOWER_UNSPEC, 1927 sizeof(key->tp_max.src))) 1928 return -1; 1929 1930 return 0; 1931 } 1932 1933 static int fl_dump_key_mpls(struct sk_buff *skb, 1934 struct flow_dissector_key_mpls *mpls_key, 1935 struct flow_dissector_key_mpls *mpls_mask) 1936 { 1937 int err; 1938 1939 if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask))) 1940 return 0; 1941 if (mpls_mask->mpls_ttl) { 1942 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL, 1943 mpls_key->mpls_ttl); 1944 if (err) 1945 return err; 1946 } 1947 if (mpls_mask->mpls_tc) { 1948 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC, 1949 mpls_key->mpls_tc); 1950 if (err) 1951 return err; 1952 } 1953 if (mpls_mask->mpls_label) { 1954 err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL, 1955 mpls_key->mpls_label); 1956 if (err) 1957 return err; 1958 } 1959 if (mpls_mask->mpls_bos) { 1960 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS, 1961 mpls_key->mpls_bos); 1962 if (err) 1963 return err; 1964 } 1965 return 0; 1966 } 1967 1968 static int fl_dump_key_ip(struct sk_buff *skb, bool encap, 1969 struct flow_dissector_key_ip *key, 1970 struct flow_dissector_key_ip *mask) 1971 { 1972 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 1973 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 1974 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 1975 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 1976 1977 if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) || 1978 fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl))) 1979 return -1; 1980 1981 return 0; 1982 } 1983 1984 static int fl_dump_key_vlan(struct sk_buff *skb, 1985 int vlan_id_key, int vlan_prio_key, 1986 struct flow_dissector_key_vlan *vlan_key, 1987 struct flow_dissector_key_vlan *vlan_mask) 1988 { 1989 int err; 1990 1991 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 1992 return 0; 1993 if (vlan_mask->vlan_id) { 1994 err = nla_put_u16(skb, vlan_id_key, 1995 vlan_key->vlan_id); 1996 if (err) 1997 return err; 1998 } 1999 if (vlan_mask->vlan_priority) { 2000 err = nla_put_u8(skb, vlan_prio_key, 2001 vlan_key->vlan_priority); 2002 if (err) 2003 return err; 2004 } 2005 return 0; 2006 } 2007 2008 static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask, 2009 u32 *flower_key, u32 *flower_mask, 2010 u32 flower_flag_bit, u32 dissector_flag_bit) 2011 { 2012 if (dissector_mask & dissector_flag_bit) { 2013 *flower_mask |= flower_flag_bit; 2014 if (dissector_key & dissector_flag_bit) 2015 *flower_key |= flower_flag_bit; 2016 } 2017 } 2018 2019 static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask) 2020 { 2021 u32 key, mask; 2022 __be32 _key, _mask; 2023 int err; 2024 2025 if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask))) 2026 return 0; 2027 2028 key = 0; 2029 mask = 0; 2030 2031 fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2032 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 2033 fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2034 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 2035 FLOW_DIS_FIRST_FRAG); 2036 2037 _key = cpu_to_be32(key); 2038 _mask = cpu_to_be32(mask); 2039 2040 err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key); 2041 if (err) 2042 return err; 2043 2044 return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask); 2045 } 2046 2047 static int fl_dump_key_geneve_opt(struct sk_buff *skb, 2048 struct flow_dissector_key_enc_opts *enc_opts) 2049 { 2050 struct geneve_opt *opt; 2051 struct nlattr *nest; 2052 int opt_off = 0; 2053 2054 nest = nla_nest_start(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE); 2055 if (!nest) 2056 goto nla_put_failure; 2057 2058 while (enc_opts->len > opt_off) { 2059 opt = (struct geneve_opt *)&enc_opts->data[opt_off]; 2060 2061 if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS, 2062 opt->opt_class)) 2063 goto nla_put_failure; 2064 if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE, 2065 opt->type)) 2066 goto nla_put_failure; 2067 if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA, 2068 opt->length * 4, opt->opt_data)) 2069 goto nla_put_failure; 2070 2071 opt_off += sizeof(struct geneve_opt) + opt->length * 4; 2072 } 2073 nla_nest_end(skb, nest); 2074 return 0; 2075 2076 nla_put_failure: 2077 nla_nest_cancel(skb, nest); 2078 return -EMSGSIZE; 2079 } 2080 2081 static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type, 2082 struct flow_dissector_key_enc_opts *enc_opts) 2083 { 2084 struct nlattr *nest; 2085 int err; 2086 2087 if (!enc_opts->len) 2088 return 0; 2089 2090 nest = nla_nest_start(skb, enc_opt_type); 2091 if (!nest) 2092 goto nla_put_failure; 2093 2094 switch (enc_opts->dst_opt_type) { 2095 case TUNNEL_GENEVE_OPT: 2096 err = fl_dump_key_geneve_opt(skb, enc_opts); 2097 if (err) 2098 goto nla_put_failure; 2099 break; 2100 default: 2101 goto nla_put_failure; 2102 } 2103 nla_nest_end(skb, nest); 2104 return 0; 2105 2106 nla_put_failure: 2107 nla_nest_cancel(skb, nest); 2108 return -EMSGSIZE; 2109 } 2110 2111 static int fl_dump_key_enc_opt(struct sk_buff *skb, 2112 struct flow_dissector_key_enc_opts *key_opts, 2113 struct flow_dissector_key_enc_opts *msk_opts) 2114 { 2115 int err; 2116 2117 err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts); 2118 if (err) 2119 return err; 2120 2121 return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts); 2122 } 2123 2124 static int fl_dump_key(struct sk_buff *skb, struct net *net, 2125 struct fl_flow_key *key, struct fl_flow_key *mask) 2126 { 2127 if (mask->indev_ifindex) { 2128 struct net_device *dev; 2129 2130 dev = __dev_get_by_index(net, key->indev_ifindex); 2131 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 2132 goto nla_put_failure; 2133 } 2134 2135 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 2136 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 2137 sizeof(key->eth.dst)) || 2138 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 2139 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 2140 sizeof(key->eth.src)) || 2141 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 2142 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 2143 sizeof(key->basic.n_proto))) 2144 goto nla_put_failure; 2145 2146 if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls)) 2147 goto nla_put_failure; 2148 2149 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID, 2150 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan)) 2151 goto nla_put_failure; 2152 2153 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID, 2154 TCA_FLOWER_KEY_CVLAN_PRIO, 2155 &key->cvlan, &mask->cvlan) || 2156 (mask->cvlan.vlan_tpid && 2157 nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2158 key->cvlan.vlan_tpid))) 2159 goto nla_put_failure; 2160 2161 if (mask->basic.n_proto) { 2162 if (mask->cvlan.vlan_tpid) { 2163 if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 2164 key->basic.n_proto)) 2165 goto nla_put_failure; 2166 } else if (mask->vlan.vlan_tpid) { 2167 if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2168 key->basic.n_proto)) 2169 goto nla_put_failure; 2170 } 2171 } 2172 2173 if ((key->basic.n_proto == htons(ETH_P_IP) || 2174 key->basic.n_proto == htons(ETH_P_IPV6)) && 2175 (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 2176 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 2177 sizeof(key->basic.ip_proto)) || 2178 fl_dump_key_ip(skb, false, &key->ip, &mask->ip))) 2179 goto nla_put_failure; 2180 2181 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2182 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 2183 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 2184 sizeof(key->ipv4.src)) || 2185 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 2186 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 2187 sizeof(key->ipv4.dst)))) 2188 goto nla_put_failure; 2189 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2190 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 2191 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 2192 sizeof(key->ipv6.src)) || 2193 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 2194 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 2195 sizeof(key->ipv6.dst)))) 2196 goto nla_put_failure; 2197 2198 if (key->basic.ip_proto == IPPROTO_TCP && 2199 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 2200 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 2201 sizeof(key->tp.src)) || 2202 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 2203 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 2204 sizeof(key->tp.dst)) || 2205 fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 2206 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 2207 sizeof(key->tcp.flags)))) 2208 goto nla_put_failure; 2209 else if (key->basic.ip_proto == IPPROTO_UDP && 2210 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 2211 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 2212 sizeof(key->tp.src)) || 2213 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 2214 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 2215 sizeof(key->tp.dst)))) 2216 goto nla_put_failure; 2217 else if (key->basic.ip_proto == IPPROTO_SCTP && 2218 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 2219 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 2220 sizeof(key->tp.src)) || 2221 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 2222 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 2223 sizeof(key->tp.dst)))) 2224 goto nla_put_failure; 2225 else if (key->basic.n_proto == htons(ETH_P_IP) && 2226 key->basic.ip_proto == IPPROTO_ICMP && 2227 (fl_dump_key_val(skb, &key->icmp.type, 2228 TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type, 2229 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 2230 sizeof(key->icmp.type)) || 2231 fl_dump_key_val(skb, &key->icmp.code, 2232 TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code, 2233 TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 2234 sizeof(key->icmp.code)))) 2235 goto nla_put_failure; 2236 else if (key->basic.n_proto == htons(ETH_P_IPV6) && 2237 key->basic.ip_proto == IPPROTO_ICMPV6 && 2238 (fl_dump_key_val(skb, &key->icmp.type, 2239 TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type, 2240 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 2241 sizeof(key->icmp.type)) || 2242 fl_dump_key_val(skb, &key->icmp.code, 2243 TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code, 2244 TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 2245 sizeof(key->icmp.code)))) 2246 goto nla_put_failure; 2247 else if ((key->basic.n_proto == htons(ETH_P_ARP) || 2248 key->basic.n_proto == htons(ETH_P_RARP)) && 2249 (fl_dump_key_val(skb, &key->arp.sip, 2250 TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip, 2251 TCA_FLOWER_KEY_ARP_SIP_MASK, 2252 sizeof(key->arp.sip)) || 2253 fl_dump_key_val(skb, &key->arp.tip, 2254 TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip, 2255 TCA_FLOWER_KEY_ARP_TIP_MASK, 2256 sizeof(key->arp.tip)) || 2257 fl_dump_key_val(skb, &key->arp.op, 2258 TCA_FLOWER_KEY_ARP_OP, &mask->arp.op, 2259 TCA_FLOWER_KEY_ARP_OP_MASK, 2260 sizeof(key->arp.op)) || 2261 fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 2262 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 2263 sizeof(key->arp.sha)) || 2264 fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 2265 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 2266 sizeof(key->arp.tha)))) 2267 goto nla_put_failure; 2268 2269 if ((key->basic.ip_proto == IPPROTO_TCP || 2270 key->basic.ip_proto == IPPROTO_UDP || 2271 key->basic.ip_proto == IPPROTO_SCTP) && 2272 fl_dump_key_port_range(skb, key, mask)) 2273 goto nla_put_failure; 2274 2275 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2276 (fl_dump_key_val(skb, &key->enc_ipv4.src, 2277 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 2278 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 2279 sizeof(key->enc_ipv4.src)) || 2280 fl_dump_key_val(skb, &key->enc_ipv4.dst, 2281 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 2282 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 2283 sizeof(key->enc_ipv4.dst)))) 2284 goto nla_put_failure; 2285 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2286 (fl_dump_key_val(skb, &key->enc_ipv6.src, 2287 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 2288 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 2289 sizeof(key->enc_ipv6.src)) || 2290 fl_dump_key_val(skb, &key->enc_ipv6.dst, 2291 TCA_FLOWER_KEY_ENC_IPV6_DST, 2292 &mask->enc_ipv6.dst, 2293 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 2294 sizeof(key->enc_ipv6.dst)))) 2295 goto nla_put_failure; 2296 2297 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 2298 &mask->enc_key_id, TCA_FLOWER_UNSPEC, 2299 sizeof(key->enc_key_id)) || 2300 fl_dump_key_val(skb, &key->enc_tp.src, 2301 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 2302 &mask->enc_tp.src, 2303 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 2304 sizeof(key->enc_tp.src)) || 2305 fl_dump_key_val(skb, &key->enc_tp.dst, 2306 TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 2307 &mask->enc_tp.dst, 2308 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 2309 sizeof(key->enc_tp.dst)) || 2310 fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) || 2311 fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts)) 2312 goto nla_put_failure; 2313 2314 if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags)) 2315 goto nla_put_failure; 2316 2317 return 0; 2318 2319 nla_put_failure: 2320 return -EMSGSIZE; 2321 } 2322 2323 static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh, 2324 struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 2325 { 2326 struct cls_fl_filter *f = fh; 2327 struct nlattr *nest; 2328 struct fl_flow_key *key, *mask; 2329 bool skip_hw; 2330 2331 if (!f) 2332 return skb->len; 2333 2334 t->tcm_handle = f->handle; 2335 2336 nest = nla_nest_start(skb, TCA_OPTIONS); 2337 if (!nest) 2338 goto nla_put_failure; 2339 2340 spin_lock(&tp->lock); 2341 2342 if (f->res.classid && 2343 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 2344 goto nla_put_failure_locked; 2345 2346 key = &f->key; 2347 mask = &f->mask->key; 2348 skip_hw = tc_skip_hw(f->flags); 2349 2350 if (fl_dump_key(skb, net, key, mask)) 2351 goto nla_put_failure_locked; 2352 2353 if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags)) 2354 goto nla_put_failure_locked; 2355 2356 spin_unlock(&tp->lock); 2357 2358 if (!skip_hw) 2359 fl_hw_update_stats(tp, f, rtnl_held); 2360 2361 if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count)) 2362 goto nla_put_failure; 2363 2364 if (tcf_exts_dump(skb, &f->exts)) 2365 goto nla_put_failure; 2366 2367 nla_nest_end(skb, nest); 2368 2369 if (tcf_exts_dump_stats(skb, &f->exts) < 0) 2370 goto nla_put_failure; 2371 2372 return skb->len; 2373 2374 nla_put_failure_locked: 2375 spin_unlock(&tp->lock); 2376 nla_put_failure: 2377 nla_nest_cancel(skb, nest); 2378 return -1; 2379 } 2380 2381 static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv) 2382 { 2383 struct fl_flow_tmplt *tmplt = tmplt_priv; 2384 struct fl_flow_key *key, *mask; 2385 struct nlattr *nest; 2386 2387 nest = nla_nest_start(skb, TCA_OPTIONS); 2388 if (!nest) 2389 goto nla_put_failure; 2390 2391 key = &tmplt->dummy_key; 2392 mask = &tmplt->mask; 2393 2394 if (fl_dump_key(skb, net, key, mask)) 2395 goto nla_put_failure; 2396 2397 nla_nest_end(skb, nest); 2398 2399 return skb->len; 2400 2401 nla_put_failure: 2402 nla_nest_cancel(skb, nest); 2403 return -EMSGSIZE; 2404 } 2405 2406 static void fl_bind_class(void *fh, u32 classid, unsigned long cl) 2407 { 2408 struct cls_fl_filter *f = fh; 2409 2410 if (f && f->res.classid == classid) 2411 f->res.class = cl; 2412 } 2413 2414 static struct tcf_proto_ops cls_fl_ops __read_mostly = { 2415 .kind = "flower", 2416 .classify = fl_classify, 2417 .init = fl_init, 2418 .destroy = fl_destroy, 2419 .get = fl_get, 2420 .put = fl_put, 2421 .change = fl_change, 2422 .delete = fl_delete, 2423 .walk = fl_walk, 2424 .reoffload = fl_reoffload, 2425 .dump = fl_dump, 2426 .bind_class = fl_bind_class, 2427 .tmplt_create = fl_tmplt_create, 2428 .tmplt_destroy = fl_tmplt_destroy, 2429 .tmplt_dump = fl_tmplt_dump, 2430 .owner = THIS_MODULE, 2431 .flags = TCF_PROTO_OPS_DOIT_UNLOCKED, 2432 }; 2433 2434 static int __init cls_fl_init(void) 2435 { 2436 return register_tcf_proto_ops(&cls_fl_ops); 2437 } 2438 2439 static void __exit cls_fl_exit(void) 2440 { 2441 unregister_tcf_proto_ops(&cls_fl_ops); 2442 } 2443 2444 module_init(cls_fl_init); 2445 module_exit(cls_fl_exit); 2446 2447 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 2448 MODULE_DESCRIPTION("Flower classifier"); 2449 MODULE_LICENSE("GPL v2"); 2450