1 /* 2 * net/sched/cls_flower.c Flower classifier 3 * 4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/rhashtable.h> 16 17 #include <linux/if_ether.h> 18 #include <linux/in6.h> 19 #include <linux/ip.h> 20 21 #include <net/sch_generic.h> 22 #include <net/pkt_cls.h> 23 #include <net/ip.h> 24 #include <net/flow_dissector.h> 25 26 #include <net/dst.h> 27 #include <net/dst_metadata.h> 28 29 struct fl_flow_key { 30 int indev_ifindex; 31 struct flow_dissector_key_control control; 32 struct flow_dissector_key_control enc_control; 33 struct flow_dissector_key_basic basic; 34 struct flow_dissector_key_eth_addrs eth; 35 struct flow_dissector_key_vlan vlan; 36 struct flow_dissector_key_addrs ipaddrs; 37 union { 38 struct flow_dissector_key_ipv4_addrs ipv4; 39 struct flow_dissector_key_ipv6_addrs ipv6; 40 }; 41 struct flow_dissector_key_ports tp; 42 struct flow_dissector_key_keyid enc_key_id; 43 union { 44 struct flow_dissector_key_ipv4_addrs enc_ipv4; 45 struct flow_dissector_key_ipv6_addrs enc_ipv6; 46 }; 47 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 48 49 struct fl_flow_mask_range { 50 unsigned short int start; 51 unsigned short int end; 52 }; 53 54 struct fl_flow_mask { 55 struct fl_flow_key key; 56 struct fl_flow_mask_range range; 57 struct rcu_head rcu; 58 }; 59 60 struct cls_fl_head { 61 struct rhashtable ht; 62 struct fl_flow_mask mask; 63 struct flow_dissector dissector; 64 u32 hgen; 65 bool mask_assigned; 66 struct list_head filters; 67 struct rhashtable_params ht_params; 68 struct rcu_head rcu; 69 }; 70 71 struct cls_fl_filter { 72 struct rhash_head ht_node; 73 struct fl_flow_key mkey; 74 struct tcf_exts exts; 75 struct tcf_result res; 76 struct fl_flow_key key; 77 struct list_head list; 78 u32 handle; 79 u32 flags; 80 struct rcu_head rcu; 81 }; 82 83 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 84 { 85 return mask->range.end - mask->range.start; 86 } 87 88 static void fl_mask_update_range(struct fl_flow_mask *mask) 89 { 90 const u8 *bytes = (const u8 *) &mask->key; 91 size_t size = sizeof(mask->key); 92 size_t i, first = 0, last = size - 1; 93 94 for (i = 0; i < sizeof(mask->key); i++) { 95 if (bytes[i]) { 96 if (!first && i) 97 first = i; 98 last = i; 99 } 100 } 101 mask->range.start = rounddown(first, sizeof(long)); 102 mask->range.end = roundup(last + 1, sizeof(long)); 103 } 104 105 static void *fl_key_get_start(struct fl_flow_key *key, 106 const struct fl_flow_mask *mask) 107 { 108 return (u8 *) key + mask->range.start; 109 } 110 111 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 112 struct fl_flow_mask *mask) 113 { 114 const long *lkey = fl_key_get_start(key, mask); 115 const long *lmask = fl_key_get_start(&mask->key, mask); 116 long *lmkey = fl_key_get_start(mkey, mask); 117 int i; 118 119 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 120 *lmkey++ = *lkey++ & *lmask++; 121 } 122 123 static void fl_clear_masked_range(struct fl_flow_key *key, 124 struct fl_flow_mask *mask) 125 { 126 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 127 } 128 129 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 130 struct tcf_result *res) 131 { 132 struct cls_fl_head *head = rcu_dereference_bh(tp->root); 133 struct cls_fl_filter *f; 134 struct fl_flow_key skb_key; 135 struct fl_flow_key skb_mkey; 136 struct ip_tunnel_info *info; 137 138 if (!atomic_read(&head->ht.nelems)) 139 return -1; 140 141 fl_clear_masked_range(&skb_key, &head->mask); 142 143 info = skb_tunnel_info(skb); 144 if (info) { 145 struct ip_tunnel_key *key = &info->key; 146 147 switch (ip_tunnel_info_af(info)) { 148 case AF_INET: 149 skb_key.enc_ipv4.src = key->u.ipv4.src; 150 skb_key.enc_ipv4.dst = key->u.ipv4.dst; 151 break; 152 case AF_INET6: 153 skb_key.enc_ipv6.src = key->u.ipv6.src; 154 skb_key.enc_ipv6.dst = key->u.ipv6.dst; 155 break; 156 } 157 158 skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id); 159 } 160 161 skb_key.indev_ifindex = skb->skb_iif; 162 /* skb_flow_dissect() does not set n_proto in case an unknown protocol, 163 * so do it rather here. 164 */ 165 skb_key.basic.n_proto = skb->protocol; 166 skb_flow_dissect(skb, &head->dissector, &skb_key, 0); 167 168 fl_set_masked_key(&skb_mkey, &skb_key, &head->mask); 169 170 f = rhashtable_lookup_fast(&head->ht, 171 fl_key_get_start(&skb_mkey, &head->mask), 172 head->ht_params); 173 if (f && !tc_skip_sw(f->flags)) { 174 *res = f->res; 175 return tcf_exts_exec(skb, &f->exts, res); 176 } 177 return -1; 178 } 179 180 static int fl_init(struct tcf_proto *tp) 181 { 182 struct cls_fl_head *head; 183 184 head = kzalloc(sizeof(*head), GFP_KERNEL); 185 if (!head) 186 return -ENOBUFS; 187 188 INIT_LIST_HEAD_RCU(&head->filters); 189 rcu_assign_pointer(tp->root, head); 190 191 return 0; 192 } 193 194 static void fl_destroy_filter(struct rcu_head *head) 195 { 196 struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu); 197 198 tcf_exts_destroy(&f->exts); 199 kfree(f); 200 } 201 202 static void fl_hw_destroy_filter(struct tcf_proto *tp, unsigned long cookie) 203 { 204 struct net_device *dev = tp->q->dev_queue->dev; 205 struct tc_cls_flower_offload offload = {0}; 206 struct tc_to_netdev tc; 207 208 if (!tc_should_offload(dev, tp, 0)) 209 return; 210 211 offload.command = TC_CLSFLOWER_DESTROY; 212 offload.cookie = cookie; 213 214 tc.type = TC_SETUP_CLSFLOWER; 215 tc.cls_flower = &offload; 216 217 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc); 218 } 219 220 static int fl_hw_replace_filter(struct tcf_proto *tp, 221 struct flow_dissector *dissector, 222 struct fl_flow_key *mask, 223 struct fl_flow_key *key, 224 struct tcf_exts *actions, 225 unsigned long cookie, u32 flags) 226 { 227 struct net_device *dev = tp->q->dev_queue->dev; 228 struct tc_cls_flower_offload offload = {0}; 229 struct tc_to_netdev tc; 230 int err; 231 232 if (!tc_should_offload(dev, tp, flags)) 233 return tc_skip_sw(flags) ? -EINVAL : 0; 234 235 offload.command = TC_CLSFLOWER_REPLACE; 236 offload.cookie = cookie; 237 offload.dissector = dissector; 238 offload.mask = mask; 239 offload.key = key; 240 offload.exts = actions; 241 242 tc.type = TC_SETUP_CLSFLOWER; 243 tc.cls_flower = &offload; 244 245 err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc); 246 247 if (tc_skip_sw(flags)) 248 return err; 249 250 return 0; 251 } 252 253 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f) 254 { 255 struct net_device *dev = tp->q->dev_queue->dev; 256 struct tc_cls_flower_offload offload = {0}; 257 struct tc_to_netdev tc; 258 259 if (!tc_should_offload(dev, tp, 0)) 260 return; 261 262 offload.command = TC_CLSFLOWER_STATS; 263 offload.cookie = (unsigned long)f; 264 offload.exts = &f->exts; 265 266 tc.type = TC_SETUP_CLSFLOWER; 267 tc.cls_flower = &offload; 268 269 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc); 270 } 271 272 static bool fl_destroy(struct tcf_proto *tp, bool force) 273 { 274 struct cls_fl_head *head = rtnl_dereference(tp->root); 275 struct cls_fl_filter *f, *next; 276 277 if (!force && !list_empty(&head->filters)) 278 return false; 279 280 list_for_each_entry_safe(f, next, &head->filters, list) { 281 fl_hw_destroy_filter(tp, (unsigned long)f); 282 list_del_rcu(&f->list); 283 call_rcu(&f->rcu, fl_destroy_filter); 284 } 285 RCU_INIT_POINTER(tp->root, NULL); 286 if (head->mask_assigned) 287 rhashtable_destroy(&head->ht); 288 kfree_rcu(head, rcu); 289 return true; 290 } 291 292 static unsigned long fl_get(struct tcf_proto *tp, u32 handle) 293 { 294 struct cls_fl_head *head = rtnl_dereference(tp->root); 295 struct cls_fl_filter *f; 296 297 list_for_each_entry(f, &head->filters, list) 298 if (f->handle == handle) 299 return (unsigned long) f; 300 return 0; 301 } 302 303 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 304 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 305 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 306 [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 307 .len = IFNAMSIZ }, 308 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 309 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 310 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 311 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 312 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 313 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 314 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 315 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 316 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 317 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 318 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 319 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 320 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 321 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 322 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 323 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 324 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 325 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 326 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 327 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 328 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 329 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 330 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 331 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 332 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 333 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 334 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 335 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 336 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 337 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 338 }; 339 340 static void fl_set_key_val(struct nlattr **tb, 341 void *val, int val_type, 342 void *mask, int mask_type, int len) 343 { 344 if (!tb[val_type]) 345 return; 346 memcpy(val, nla_data(tb[val_type]), len); 347 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 348 memset(mask, 0xff, len); 349 else 350 memcpy(mask, nla_data(tb[mask_type]), len); 351 } 352 353 static void fl_set_key_vlan(struct nlattr **tb, 354 struct flow_dissector_key_vlan *key_val, 355 struct flow_dissector_key_vlan *key_mask) 356 { 357 #define VLAN_PRIORITY_MASK 0x7 358 359 if (tb[TCA_FLOWER_KEY_VLAN_ID]) { 360 key_val->vlan_id = 361 nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK; 362 key_mask->vlan_id = VLAN_VID_MASK; 363 } 364 if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) { 365 key_val->vlan_priority = 366 nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) & 367 VLAN_PRIORITY_MASK; 368 key_mask->vlan_priority = VLAN_PRIORITY_MASK; 369 } 370 } 371 372 static int fl_set_key(struct net *net, struct nlattr **tb, 373 struct fl_flow_key *key, struct fl_flow_key *mask) 374 { 375 __be16 ethertype; 376 #ifdef CONFIG_NET_CLS_IND 377 if (tb[TCA_FLOWER_INDEV]) { 378 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV]); 379 if (err < 0) 380 return err; 381 key->indev_ifindex = err; 382 mask->indev_ifindex = 0xffffffff; 383 } 384 #endif 385 386 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 387 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 388 sizeof(key->eth.dst)); 389 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 390 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 391 sizeof(key->eth.src)); 392 393 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 394 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 395 396 if (ethertype == htons(ETH_P_8021Q)) { 397 fl_set_key_vlan(tb, &key->vlan, &mask->vlan); 398 fl_set_key_val(tb, &key->basic.n_proto, 399 TCA_FLOWER_KEY_VLAN_ETH_TYPE, 400 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 401 sizeof(key->basic.n_proto)); 402 } else { 403 key->basic.n_proto = ethertype; 404 mask->basic.n_proto = cpu_to_be16(~0); 405 } 406 } 407 408 if (key->basic.n_proto == htons(ETH_P_IP) || 409 key->basic.n_proto == htons(ETH_P_IPV6)) { 410 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 411 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 412 sizeof(key->basic.ip_proto)); 413 } 414 415 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 416 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 417 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 418 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 419 sizeof(key->ipv4.src)); 420 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 421 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 422 sizeof(key->ipv4.dst)); 423 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 424 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 425 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 426 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 427 sizeof(key->ipv6.src)); 428 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 429 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 430 sizeof(key->ipv6.dst)); 431 } 432 433 if (key->basic.ip_proto == IPPROTO_TCP) { 434 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 435 &mask->tp.src, TCA_FLOWER_UNSPEC, 436 sizeof(key->tp.src)); 437 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 438 &mask->tp.dst, TCA_FLOWER_UNSPEC, 439 sizeof(key->tp.dst)); 440 } else if (key->basic.ip_proto == IPPROTO_UDP) { 441 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 442 &mask->tp.src, TCA_FLOWER_UNSPEC, 443 sizeof(key->tp.src)); 444 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 445 &mask->tp.dst, TCA_FLOWER_UNSPEC, 446 sizeof(key->tp.dst)); 447 } 448 449 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 450 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 451 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 452 fl_set_key_val(tb, &key->enc_ipv4.src, 453 TCA_FLOWER_KEY_ENC_IPV4_SRC, 454 &mask->enc_ipv4.src, 455 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 456 sizeof(key->enc_ipv4.src)); 457 fl_set_key_val(tb, &key->enc_ipv4.dst, 458 TCA_FLOWER_KEY_ENC_IPV4_DST, 459 &mask->enc_ipv4.dst, 460 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 461 sizeof(key->enc_ipv4.dst)); 462 } 463 464 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 465 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 466 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 467 fl_set_key_val(tb, &key->enc_ipv6.src, 468 TCA_FLOWER_KEY_ENC_IPV6_SRC, 469 &mask->enc_ipv6.src, 470 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 471 sizeof(key->enc_ipv6.src)); 472 fl_set_key_val(tb, &key->enc_ipv6.dst, 473 TCA_FLOWER_KEY_ENC_IPV6_DST, 474 &mask->enc_ipv6.dst, 475 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 476 sizeof(key->enc_ipv6.dst)); 477 } 478 479 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 480 &mask->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 481 sizeof(key->enc_key_id.keyid)); 482 483 return 0; 484 } 485 486 static bool fl_mask_eq(struct fl_flow_mask *mask1, 487 struct fl_flow_mask *mask2) 488 { 489 const long *lmask1 = fl_key_get_start(&mask1->key, mask1); 490 const long *lmask2 = fl_key_get_start(&mask2->key, mask2); 491 492 return !memcmp(&mask1->range, &mask2->range, sizeof(mask1->range)) && 493 !memcmp(lmask1, lmask2, fl_mask_range(mask1)); 494 } 495 496 static const struct rhashtable_params fl_ht_params = { 497 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 498 .head_offset = offsetof(struct cls_fl_filter, ht_node), 499 .automatic_shrinking = true, 500 }; 501 502 static int fl_init_hashtable(struct cls_fl_head *head, 503 struct fl_flow_mask *mask) 504 { 505 head->ht_params = fl_ht_params; 506 head->ht_params.key_len = fl_mask_range(mask); 507 head->ht_params.key_offset += mask->range.start; 508 509 return rhashtable_init(&head->ht, &head->ht_params); 510 } 511 512 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 513 #define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member)) 514 515 #define FL_KEY_IS_MASKED(mask, member) \ 516 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 517 0, FL_KEY_MEMBER_SIZE(member)) \ 518 519 #define FL_KEY_SET(keys, cnt, id, member) \ 520 do { \ 521 keys[cnt].key_id = id; \ 522 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 523 cnt++; \ 524 } while(0); 525 526 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 527 do { \ 528 if (FL_KEY_IS_MASKED(mask, member)) \ 529 FL_KEY_SET(keys, cnt, id, member); \ 530 } while(0); 531 532 static void fl_init_dissector(struct cls_fl_head *head, 533 struct fl_flow_mask *mask) 534 { 535 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 536 size_t cnt = 0; 537 538 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 539 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 540 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 541 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 542 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 543 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 544 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 545 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 546 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 547 FLOW_DISSECTOR_KEY_PORTS, tp); 548 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 549 FLOW_DISSECTOR_KEY_VLAN, vlan); 550 551 skb_flow_dissector_init(&head->dissector, keys, cnt); 552 } 553 554 static int fl_check_assign_mask(struct cls_fl_head *head, 555 struct fl_flow_mask *mask) 556 { 557 int err; 558 559 if (head->mask_assigned) { 560 if (!fl_mask_eq(&head->mask, mask)) 561 return -EINVAL; 562 else 563 return 0; 564 } 565 566 /* Mask is not assigned yet. So assign it and init hashtable 567 * according to that. 568 */ 569 err = fl_init_hashtable(head, mask); 570 if (err) 571 return err; 572 memcpy(&head->mask, mask, sizeof(head->mask)); 573 head->mask_assigned = true; 574 575 fl_init_dissector(head, mask); 576 577 return 0; 578 } 579 580 static int fl_set_parms(struct net *net, struct tcf_proto *tp, 581 struct cls_fl_filter *f, struct fl_flow_mask *mask, 582 unsigned long base, struct nlattr **tb, 583 struct nlattr *est, bool ovr) 584 { 585 struct tcf_exts e; 586 int err; 587 588 err = tcf_exts_init(&e, TCA_FLOWER_ACT, 0); 589 if (err < 0) 590 return err; 591 err = tcf_exts_validate(net, tp, tb, est, &e, ovr); 592 if (err < 0) 593 goto errout; 594 595 if (tb[TCA_FLOWER_CLASSID]) { 596 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 597 tcf_bind_filter(tp, &f->res, base); 598 } 599 600 err = fl_set_key(net, tb, &f->key, &mask->key); 601 if (err) 602 goto errout; 603 604 fl_mask_update_range(mask); 605 fl_set_masked_key(&f->mkey, &f->key, mask); 606 607 tcf_exts_change(tp, &f->exts, &e); 608 609 return 0; 610 errout: 611 tcf_exts_destroy(&e); 612 return err; 613 } 614 615 static u32 fl_grab_new_handle(struct tcf_proto *tp, 616 struct cls_fl_head *head) 617 { 618 unsigned int i = 0x80000000; 619 u32 handle; 620 621 do { 622 if (++head->hgen == 0x7FFFFFFF) 623 head->hgen = 1; 624 } while (--i > 0 && fl_get(tp, head->hgen)); 625 626 if (unlikely(i == 0)) { 627 pr_err("Insufficient number of handles\n"); 628 handle = 0; 629 } else { 630 handle = head->hgen; 631 } 632 633 return handle; 634 } 635 636 static int fl_change(struct net *net, struct sk_buff *in_skb, 637 struct tcf_proto *tp, unsigned long base, 638 u32 handle, struct nlattr **tca, 639 unsigned long *arg, bool ovr) 640 { 641 struct cls_fl_head *head = rtnl_dereference(tp->root); 642 struct cls_fl_filter *fold = (struct cls_fl_filter *) *arg; 643 struct cls_fl_filter *fnew; 644 struct nlattr *tb[TCA_FLOWER_MAX + 1]; 645 struct fl_flow_mask mask = {}; 646 int err; 647 648 if (!tca[TCA_OPTIONS]) 649 return -EINVAL; 650 651 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], fl_policy); 652 if (err < 0) 653 return err; 654 655 if (fold && handle && fold->handle != handle) 656 return -EINVAL; 657 658 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 659 if (!fnew) 660 return -ENOBUFS; 661 662 err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0); 663 if (err < 0) 664 goto errout; 665 666 if (!handle) { 667 handle = fl_grab_new_handle(tp, head); 668 if (!handle) { 669 err = -EINVAL; 670 goto errout; 671 } 672 } 673 fnew->handle = handle; 674 675 if (tb[TCA_FLOWER_FLAGS]) { 676 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 677 678 if (!tc_flags_valid(fnew->flags)) { 679 err = -EINVAL; 680 goto errout; 681 } 682 } 683 684 err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr); 685 if (err) 686 goto errout; 687 688 err = fl_check_assign_mask(head, &mask); 689 if (err) 690 goto errout; 691 692 if (!tc_skip_sw(fnew->flags)) { 693 err = rhashtable_insert_fast(&head->ht, &fnew->ht_node, 694 head->ht_params); 695 if (err) 696 goto errout; 697 } 698 699 err = fl_hw_replace_filter(tp, 700 &head->dissector, 701 &mask.key, 702 &fnew->key, 703 &fnew->exts, 704 (unsigned long)fnew, 705 fnew->flags); 706 if (err) 707 goto errout; 708 709 if (fold) { 710 rhashtable_remove_fast(&head->ht, &fold->ht_node, 711 head->ht_params); 712 fl_hw_destroy_filter(tp, (unsigned long)fold); 713 } 714 715 *arg = (unsigned long) fnew; 716 717 if (fold) { 718 list_replace_rcu(&fold->list, &fnew->list); 719 tcf_unbind_filter(tp, &fold->res); 720 call_rcu(&fold->rcu, fl_destroy_filter); 721 } else { 722 list_add_tail_rcu(&fnew->list, &head->filters); 723 } 724 725 return 0; 726 727 errout: 728 tcf_exts_destroy(&fnew->exts); 729 kfree(fnew); 730 return err; 731 } 732 733 static int fl_delete(struct tcf_proto *tp, unsigned long arg) 734 { 735 struct cls_fl_head *head = rtnl_dereference(tp->root); 736 struct cls_fl_filter *f = (struct cls_fl_filter *) arg; 737 738 rhashtable_remove_fast(&head->ht, &f->ht_node, 739 head->ht_params); 740 list_del_rcu(&f->list); 741 fl_hw_destroy_filter(tp, (unsigned long)f); 742 tcf_unbind_filter(tp, &f->res); 743 call_rcu(&f->rcu, fl_destroy_filter); 744 return 0; 745 } 746 747 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg) 748 { 749 struct cls_fl_head *head = rtnl_dereference(tp->root); 750 struct cls_fl_filter *f; 751 752 list_for_each_entry_rcu(f, &head->filters, list) { 753 if (arg->count < arg->skip) 754 goto skip; 755 if (arg->fn(tp, (unsigned long) f, arg) < 0) { 756 arg->stop = 1; 757 break; 758 } 759 skip: 760 arg->count++; 761 } 762 } 763 764 static int fl_dump_key_val(struct sk_buff *skb, 765 void *val, int val_type, 766 void *mask, int mask_type, int len) 767 { 768 int err; 769 770 if (!memchr_inv(mask, 0, len)) 771 return 0; 772 err = nla_put(skb, val_type, len, val); 773 if (err) 774 return err; 775 if (mask_type != TCA_FLOWER_UNSPEC) { 776 err = nla_put(skb, mask_type, len, mask); 777 if (err) 778 return err; 779 } 780 return 0; 781 } 782 783 static int fl_dump_key_vlan(struct sk_buff *skb, 784 struct flow_dissector_key_vlan *vlan_key, 785 struct flow_dissector_key_vlan *vlan_mask) 786 { 787 int err; 788 789 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 790 return 0; 791 if (vlan_mask->vlan_id) { 792 err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID, 793 vlan_key->vlan_id); 794 if (err) 795 return err; 796 } 797 if (vlan_mask->vlan_priority) { 798 err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO, 799 vlan_key->vlan_priority); 800 if (err) 801 return err; 802 } 803 return 0; 804 } 805 806 static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh, 807 struct sk_buff *skb, struct tcmsg *t) 808 { 809 struct cls_fl_head *head = rtnl_dereference(tp->root); 810 struct cls_fl_filter *f = (struct cls_fl_filter *) fh; 811 struct nlattr *nest; 812 struct fl_flow_key *key, *mask; 813 814 if (!f) 815 return skb->len; 816 817 t->tcm_handle = f->handle; 818 819 nest = nla_nest_start(skb, TCA_OPTIONS); 820 if (!nest) 821 goto nla_put_failure; 822 823 if (f->res.classid && 824 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 825 goto nla_put_failure; 826 827 key = &f->key; 828 mask = &head->mask.key; 829 830 if (mask->indev_ifindex) { 831 struct net_device *dev; 832 833 dev = __dev_get_by_index(net, key->indev_ifindex); 834 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 835 goto nla_put_failure; 836 } 837 838 fl_hw_update_stats(tp, f); 839 840 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 841 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 842 sizeof(key->eth.dst)) || 843 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 844 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 845 sizeof(key->eth.src)) || 846 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 847 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 848 sizeof(key->basic.n_proto))) 849 goto nla_put_failure; 850 851 if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan)) 852 goto nla_put_failure; 853 854 if ((key->basic.n_proto == htons(ETH_P_IP) || 855 key->basic.n_proto == htons(ETH_P_IPV6)) && 856 fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 857 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 858 sizeof(key->basic.ip_proto))) 859 goto nla_put_failure; 860 861 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 862 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 863 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 864 sizeof(key->ipv4.src)) || 865 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 866 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 867 sizeof(key->ipv4.dst)))) 868 goto nla_put_failure; 869 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 870 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 871 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 872 sizeof(key->ipv6.src)) || 873 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 874 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 875 sizeof(key->ipv6.dst)))) 876 goto nla_put_failure; 877 878 if (key->basic.ip_proto == IPPROTO_TCP && 879 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 880 &mask->tp.src, TCA_FLOWER_UNSPEC, 881 sizeof(key->tp.src)) || 882 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 883 &mask->tp.dst, TCA_FLOWER_UNSPEC, 884 sizeof(key->tp.dst)))) 885 goto nla_put_failure; 886 else if (key->basic.ip_proto == IPPROTO_UDP && 887 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 888 &mask->tp.src, TCA_FLOWER_UNSPEC, 889 sizeof(key->tp.src)) || 890 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 891 &mask->tp.dst, TCA_FLOWER_UNSPEC, 892 sizeof(key->tp.dst)))) 893 goto nla_put_failure; 894 895 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 896 (fl_dump_key_val(skb, &key->enc_ipv4.src, 897 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 898 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 899 sizeof(key->enc_ipv4.src)) || 900 fl_dump_key_val(skb, &key->enc_ipv4.dst, 901 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 902 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 903 sizeof(key->enc_ipv4.dst)))) 904 goto nla_put_failure; 905 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 906 (fl_dump_key_val(skb, &key->enc_ipv6.src, 907 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 908 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 909 sizeof(key->enc_ipv6.src)) || 910 fl_dump_key_val(skb, &key->enc_ipv6.dst, 911 TCA_FLOWER_KEY_ENC_IPV6_DST, 912 &mask->enc_ipv6.dst, 913 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 914 sizeof(key->enc_ipv6.dst)))) 915 goto nla_put_failure; 916 917 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 918 &mask->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 919 sizeof(key->enc_key_id))) 920 goto nla_put_failure; 921 922 nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags); 923 924 if (tcf_exts_dump(skb, &f->exts)) 925 goto nla_put_failure; 926 927 nla_nest_end(skb, nest); 928 929 if (tcf_exts_dump_stats(skb, &f->exts) < 0) 930 goto nla_put_failure; 931 932 return skb->len; 933 934 nla_put_failure: 935 nla_nest_cancel(skb, nest); 936 return -1; 937 } 938 939 static struct tcf_proto_ops cls_fl_ops __read_mostly = { 940 .kind = "flower", 941 .classify = fl_classify, 942 .init = fl_init, 943 .destroy = fl_destroy, 944 .get = fl_get, 945 .change = fl_change, 946 .delete = fl_delete, 947 .walk = fl_walk, 948 .dump = fl_dump, 949 .owner = THIS_MODULE, 950 }; 951 952 static int __init cls_fl_init(void) 953 { 954 return register_tcf_proto_ops(&cls_fl_ops); 955 } 956 957 static void __exit cls_fl_exit(void) 958 { 959 unregister_tcf_proto_ops(&cls_fl_ops); 960 } 961 962 module_init(cls_fl_init); 963 module_exit(cls_fl_exit); 964 965 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 966 MODULE_DESCRIPTION("Flower classifier"); 967 MODULE_LICENSE("GPL v2"); 968