1 /* 2 * net/sched/cls_flower.c Flower classifier 3 * 4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation; either version 2 of the License, or 9 * (at your option) any later version. 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/init.h> 14 #include <linux/module.h> 15 #include <linux/rhashtable.h> 16 17 #include <linux/if_ether.h> 18 #include <linux/in6.h> 19 #include <linux/ip.h> 20 21 #include <net/sch_generic.h> 22 #include <net/pkt_cls.h> 23 #include <net/ip.h> 24 #include <net/flow_dissector.h> 25 26 #include <net/dst.h> 27 #include <net/dst_metadata.h> 28 29 struct fl_flow_key { 30 int indev_ifindex; 31 struct flow_dissector_key_control control; 32 struct flow_dissector_key_control enc_control; 33 struct flow_dissector_key_basic basic; 34 struct flow_dissector_key_eth_addrs eth; 35 struct flow_dissector_key_vlan vlan; 36 union { 37 struct flow_dissector_key_ipv4_addrs ipv4; 38 struct flow_dissector_key_ipv6_addrs ipv6; 39 }; 40 struct flow_dissector_key_ports tp; 41 struct flow_dissector_key_keyid enc_key_id; 42 union { 43 struct flow_dissector_key_ipv4_addrs enc_ipv4; 44 struct flow_dissector_key_ipv6_addrs enc_ipv6; 45 }; 46 struct flow_dissector_key_ports enc_tp; 47 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 48 49 struct fl_flow_mask_range { 50 unsigned short int start; 51 unsigned short int end; 52 }; 53 54 struct fl_flow_mask { 55 struct fl_flow_key key; 56 struct fl_flow_mask_range range; 57 struct rcu_head rcu; 58 }; 59 60 struct cls_fl_head { 61 struct rhashtable ht; 62 struct fl_flow_mask mask; 63 struct flow_dissector dissector; 64 u32 hgen; 65 bool mask_assigned; 66 struct list_head filters; 67 struct rhashtable_params ht_params; 68 struct rcu_head rcu; 69 }; 70 71 struct cls_fl_filter { 72 struct rhash_head ht_node; 73 struct fl_flow_key mkey; 74 struct tcf_exts exts; 75 struct tcf_result res; 76 struct fl_flow_key key; 77 struct list_head list; 78 u32 handle; 79 u32 flags; 80 struct rcu_head rcu; 81 }; 82 83 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 84 { 85 return mask->range.end - mask->range.start; 86 } 87 88 static void fl_mask_update_range(struct fl_flow_mask *mask) 89 { 90 const u8 *bytes = (const u8 *) &mask->key; 91 size_t size = sizeof(mask->key); 92 size_t i, first = 0, last = size - 1; 93 94 for (i = 0; i < sizeof(mask->key); i++) { 95 if (bytes[i]) { 96 if (!first && i) 97 first = i; 98 last = i; 99 } 100 } 101 mask->range.start = rounddown(first, sizeof(long)); 102 mask->range.end = roundup(last + 1, sizeof(long)); 103 } 104 105 static void *fl_key_get_start(struct fl_flow_key *key, 106 const struct fl_flow_mask *mask) 107 { 108 return (u8 *) key + mask->range.start; 109 } 110 111 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 112 struct fl_flow_mask *mask) 113 { 114 const long *lkey = fl_key_get_start(key, mask); 115 const long *lmask = fl_key_get_start(&mask->key, mask); 116 long *lmkey = fl_key_get_start(mkey, mask); 117 int i; 118 119 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 120 *lmkey++ = *lkey++ & *lmask++; 121 } 122 123 static void fl_clear_masked_range(struct fl_flow_key *key, 124 struct fl_flow_mask *mask) 125 { 126 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 127 } 128 129 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 130 struct tcf_result *res) 131 { 132 struct cls_fl_head *head = rcu_dereference_bh(tp->root); 133 struct cls_fl_filter *f; 134 struct fl_flow_key skb_key; 135 struct fl_flow_key skb_mkey; 136 struct ip_tunnel_info *info; 137 138 if (!atomic_read(&head->ht.nelems)) 139 return -1; 140 141 fl_clear_masked_range(&skb_key, &head->mask); 142 143 info = skb_tunnel_info(skb); 144 if (info) { 145 struct ip_tunnel_key *key = &info->key; 146 147 switch (ip_tunnel_info_af(info)) { 148 case AF_INET: 149 skb_key.enc_ipv4.src = key->u.ipv4.src; 150 skb_key.enc_ipv4.dst = key->u.ipv4.dst; 151 break; 152 case AF_INET6: 153 skb_key.enc_ipv6.src = key->u.ipv6.src; 154 skb_key.enc_ipv6.dst = key->u.ipv6.dst; 155 break; 156 } 157 158 skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id); 159 skb_key.enc_tp.src = key->tp_src; 160 skb_key.enc_tp.dst = key->tp_dst; 161 } 162 163 skb_key.indev_ifindex = skb->skb_iif; 164 /* skb_flow_dissect() does not set n_proto in case an unknown protocol, 165 * so do it rather here. 166 */ 167 skb_key.basic.n_proto = skb->protocol; 168 skb_flow_dissect(skb, &head->dissector, &skb_key, 0); 169 170 fl_set_masked_key(&skb_mkey, &skb_key, &head->mask); 171 172 f = rhashtable_lookup_fast(&head->ht, 173 fl_key_get_start(&skb_mkey, &head->mask), 174 head->ht_params); 175 if (f && !tc_skip_sw(f->flags)) { 176 *res = f->res; 177 return tcf_exts_exec(skb, &f->exts, res); 178 } 179 return -1; 180 } 181 182 static int fl_init(struct tcf_proto *tp) 183 { 184 struct cls_fl_head *head; 185 186 head = kzalloc(sizeof(*head), GFP_KERNEL); 187 if (!head) 188 return -ENOBUFS; 189 190 INIT_LIST_HEAD_RCU(&head->filters); 191 rcu_assign_pointer(tp->root, head); 192 193 return 0; 194 } 195 196 static void fl_destroy_filter(struct rcu_head *head) 197 { 198 struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu); 199 200 tcf_exts_destroy(&f->exts); 201 kfree(f); 202 } 203 204 static void fl_hw_destroy_filter(struct tcf_proto *tp, unsigned long cookie) 205 { 206 struct net_device *dev = tp->q->dev_queue->dev; 207 struct tc_cls_flower_offload offload = {0}; 208 struct tc_to_netdev tc; 209 210 if (!tc_should_offload(dev, tp, 0)) 211 return; 212 213 offload.command = TC_CLSFLOWER_DESTROY; 214 offload.cookie = cookie; 215 216 tc.type = TC_SETUP_CLSFLOWER; 217 tc.cls_flower = &offload; 218 219 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc); 220 } 221 222 static int fl_hw_replace_filter(struct tcf_proto *tp, 223 struct flow_dissector *dissector, 224 struct fl_flow_key *mask, 225 struct fl_flow_key *key, 226 struct tcf_exts *actions, 227 unsigned long cookie, u32 flags) 228 { 229 struct net_device *dev = tp->q->dev_queue->dev; 230 struct tc_cls_flower_offload offload = {0}; 231 struct tc_to_netdev tc; 232 int err; 233 234 if (!tc_should_offload(dev, tp, flags)) 235 return tc_skip_sw(flags) ? -EINVAL : 0; 236 237 offload.command = TC_CLSFLOWER_REPLACE; 238 offload.cookie = cookie; 239 offload.dissector = dissector; 240 offload.mask = mask; 241 offload.key = key; 242 offload.exts = actions; 243 244 tc.type = TC_SETUP_CLSFLOWER; 245 tc.cls_flower = &offload; 246 247 err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, 248 &tc); 249 250 if (tc_skip_sw(flags)) 251 return err; 252 253 return 0; 254 } 255 256 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f) 257 { 258 struct net_device *dev = tp->q->dev_queue->dev; 259 struct tc_cls_flower_offload offload = {0}; 260 struct tc_to_netdev tc; 261 262 if (!tc_should_offload(dev, tp, 0)) 263 return; 264 265 offload.command = TC_CLSFLOWER_STATS; 266 offload.cookie = (unsigned long)f; 267 offload.exts = &f->exts; 268 269 tc.type = TC_SETUP_CLSFLOWER; 270 tc.cls_flower = &offload; 271 272 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, &tc); 273 } 274 275 static void __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f) 276 { 277 list_del_rcu(&f->list); 278 fl_hw_destroy_filter(tp, (unsigned long)f); 279 tcf_unbind_filter(tp, &f->res); 280 call_rcu(&f->rcu, fl_destroy_filter); 281 } 282 283 static bool fl_destroy(struct tcf_proto *tp, bool force) 284 { 285 struct cls_fl_head *head = rtnl_dereference(tp->root); 286 struct cls_fl_filter *f, *next; 287 288 if (!force && !list_empty(&head->filters)) 289 return false; 290 291 list_for_each_entry_safe(f, next, &head->filters, list) 292 __fl_delete(tp, f); 293 RCU_INIT_POINTER(tp->root, NULL); 294 if (head->mask_assigned) 295 rhashtable_destroy(&head->ht); 296 kfree_rcu(head, rcu); 297 return true; 298 } 299 300 static unsigned long fl_get(struct tcf_proto *tp, u32 handle) 301 { 302 struct cls_fl_head *head = rtnl_dereference(tp->root); 303 struct cls_fl_filter *f; 304 305 list_for_each_entry(f, &head->filters, list) 306 if (f->handle == handle) 307 return (unsigned long) f; 308 return 0; 309 } 310 311 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 312 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 313 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 314 [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 315 .len = IFNAMSIZ }, 316 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 317 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 318 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 319 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 320 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 321 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 322 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 323 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 324 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 325 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 326 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 327 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 328 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 329 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 330 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 331 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 332 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 333 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 334 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 335 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 336 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 337 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 338 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 339 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 340 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 341 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 342 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 343 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 344 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 345 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 346 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, 347 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, 348 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, 349 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, 350 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, 351 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, 352 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, 353 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, 354 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, 355 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, 356 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, 357 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, 358 }; 359 360 static void fl_set_key_val(struct nlattr **tb, 361 void *val, int val_type, 362 void *mask, int mask_type, int len) 363 { 364 if (!tb[val_type]) 365 return; 366 memcpy(val, nla_data(tb[val_type]), len); 367 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 368 memset(mask, 0xff, len); 369 else 370 memcpy(mask, nla_data(tb[mask_type]), len); 371 } 372 373 static void fl_set_key_vlan(struct nlattr **tb, 374 struct flow_dissector_key_vlan *key_val, 375 struct flow_dissector_key_vlan *key_mask) 376 { 377 #define VLAN_PRIORITY_MASK 0x7 378 379 if (tb[TCA_FLOWER_KEY_VLAN_ID]) { 380 key_val->vlan_id = 381 nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK; 382 key_mask->vlan_id = VLAN_VID_MASK; 383 } 384 if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) { 385 key_val->vlan_priority = 386 nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) & 387 VLAN_PRIORITY_MASK; 388 key_mask->vlan_priority = VLAN_PRIORITY_MASK; 389 } 390 } 391 392 static int fl_set_key(struct net *net, struct nlattr **tb, 393 struct fl_flow_key *key, struct fl_flow_key *mask) 394 { 395 __be16 ethertype; 396 #ifdef CONFIG_NET_CLS_IND 397 if (tb[TCA_FLOWER_INDEV]) { 398 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV]); 399 if (err < 0) 400 return err; 401 key->indev_ifindex = err; 402 mask->indev_ifindex = 0xffffffff; 403 } 404 #endif 405 406 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 407 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 408 sizeof(key->eth.dst)); 409 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 410 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 411 sizeof(key->eth.src)); 412 413 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 414 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 415 416 if (ethertype == htons(ETH_P_8021Q)) { 417 fl_set_key_vlan(tb, &key->vlan, &mask->vlan); 418 fl_set_key_val(tb, &key->basic.n_proto, 419 TCA_FLOWER_KEY_VLAN_ETH_TYPE, 420 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 421 sizeof(key->basic.n_proto)); 422 } else { 423 key->basic.n_proto = ethertype; 424 mask->basic.n_proto = cpu_to_be16(~0); 425 } 426 } 427 428 if (key->basic.n_proto == htons(ETH_P_IP) || 429 key->basic.n_proto == htons(ETH_P_IPV6)) { 430 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 431 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 432 sizeof(key->basic.ip_proto)); 433 } 434 435 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 436 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 437 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 438 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 439 sizeof(key->ipv4.src)); 440 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 441 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 442 sizeof(key->ipv4.dst)); 443 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 444 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 445 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 446 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 447 sizeof(key->ipv6.src)); 448 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 449 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 450 sizeof(key->ipv6.dst)); 451 } 452 453 if (key->basic.ip_proto == IPPROTO_TCP) { 454 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 455 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 456 sizeof(key->tp.src)); 457 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 458 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 459 sizeof(key->tp.dst)); 460 } else if (key->basic.ip_proto == IPPROTO_UDP) { 461 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 462 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 463 sizeof(key->tp.src)); 464 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 465 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 466 sizeof(key->tp.dst)); 467 } else if (key->basic.ip_proto == IPPROTO_SCTP) { 468 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 469 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 470 sizeof(key->tp.src)); 471 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 472 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 473 sizeof(key->tp.dst)); 474 } 475 476 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 477 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 478 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 479 fl_set_key_val(tb, &key->enc_ipv4.src, 480 TCA_FLOWER_KEY_ENC_IPV4_SRC, 481 &mask->enc_ipv4.src, 482 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 483 sizeof(key->enc_ipv4.src)); 484 fl_set_key_val(tb, &key->enc_ipv4.dst, 485 TCA_FLOWER_KEY_ENC_IPV4_DST, 486 &mask->enc_ipv4.dst, 487 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 488 sizeof(key->enc_ipv4.dst)); 489 } 490 491 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 492 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 493 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 494 fl_set_key_val(tb, &key->enc_ipv6.src, 495 TCA_FLOWER_KEY_ENC_IPV6_SRC, 496 &mask->enc_ipv6.src, 497 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 498 sizeof(key->enc_ipv6.src)); 499 fl_set_key_val(tb, &key->enc_ipv6.dst, 500 TCA_FLOWER_KEY_ENC_IPV6_DST, 501 &mask->enc_ipv6.dst, 502 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 503 sizeof(key->enc_ipv6.dst)); 504 } 505 506 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 507 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC, 508 sizeof(key->enc_key_id.keyid)); 509 510 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 511 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 512 sizeof(key->enc_tp.src)); 513 514 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 515 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 516 sizeof(key->enc_tp.dst)); 517 518 return 0; 519 } 520 521 static bool fl_mask_eq(struct fl_flow_mask *mask1, 522 struct fl_flow_mask *mask2) 523 { 524 const long *lmask1 = fl_key_get_start(&mask1->key, mask1); 525 const long *lmask2 = fl_key_get_start(&mask2->key, mask2); 526 527 return !memcmp(&mask1->range, &mask2->range, sizeof(mask1->range)) && 528 !memcmp(lmask1, lmask2, fl_mask_range(mask1)); 529 } 530 531 static const struct rhashtable_params fl_ht_params = { 532 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 533 .head_offset = offsetof(struct cls_fl_filter, ht_node), 534 .automatic_shrinking = true, 535 }; 536 537 static int fl_init_hashtable(struct cls_fl_head *head, 538 struct fl_flow_mask *mask) 539 { 540 head->ht_params = fl_ht_params; 541 head->ht_params.key_len = fl_mask_range(mask); 542 head->ht_params.key_offset += mask->range.start; 543 544 return rhashtable_init(&head->ht, &head->ht_params); 545 } 546 547 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 548 #define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member)) 549 550 #define FL_KEY_IS_MASKED(mask, member) \ 551 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 552 0, FL_KEY_MEMBER_SIZE(member)) \ 553 554 #define FL_KEY_SET(keys, cnt, id, member) \ 555 do { \ 556 keys[cnt].key_id = id; \ 557 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 558 cnt++; \ 559 } while(0); 560 561 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 562 do { \ 563 if (FL_KEY_IS_MASKED(mask, member)) \ 564 FL_KEY_SET(keys, cnt, id, member); \ 565 } while(0); 566 567 static void fl_init_dissector(struct cls_fl_head *head, 568 struct fl_flow_mask *mask) 569 { 570 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 571 size_t cnt = 0; 572 573 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 574 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 575 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 576 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 577 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 578 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 579 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 580 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 581 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 582 FLOW_DISSECTOR_KEY_PORTS, tp); 583 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 584 FLOW_DISSECTOR_KEY_VLAN, vlan); 585 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 586 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 587 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 588 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4); 589 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 590 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6); 591 if (FL_KEY_IS_MASKED(&mask->key, enc_ipv4) || 592 FL_KEY_IS_MASKED(&mask->key, enc_ipv6)) 593 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL, 594 enc_control); 595 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt, 596 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp); 597 598 skb_flow_dissector_init(&head->dissector, keys, cnt); 599 } 600 601 static int fl_check_assign_mask(struct cls_fl_head *head, 602 struct fl_flow_mask *mask) 603 { 604 int err; 605 606 if (head->mask_assigned) { 607 if (!fl_mask_eq(&head->mask, mask)) 608 return -EINVAL; 609 else 610 return 0; 611 } 612 613 /* Mask is not assigned yet. So assign it and init hashtable 614 * according to that. 615 */ 616 err = fl_init_hashtable(head, mask); 617 if (err) 618 return err; 619 memcpy(&head->mask, mask, sizeof(head->mask)); 620 head->mask_assigned = true; 621 622 fl_init_dissector(head, mask); 623 624 return 0; 625 } 626 627 static int fl_set_parms(struct net *net, struct tcf_proto *tp, 628 struct cls_fl_filter *f, struct fl_flow_mask *mask, 629 unsigned long base, struct nlattr **tb, 630 struct nlattr *est, bool ovr) 631 { 632 struct tcf_exts e; 633 int err; 634 635 err = tcf_exts_init(&e, TCA_FLOWER_ACT, 0); 636 if (err < 0) 637 return err; 638 err = tcf_exts_validate(net, tp, tb, est, &e, ovr); 639 if (err < 0) 640 goto errout; 641 642 if (tb[TCA_FLOWER_CLASSID]) { 643 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 644 tcf_bind_filter(tp, &f->res, base); 645 } 646 647 err = fl_set_key(net, tb, &f->key, &mask->key); 648 if (err) 649 goto errout; 650 651 fl_mask_update_range(mask); 652 fl_set_masked_key(&f->mkey, &f->key, mask); 653 654 tcf_exts_change(tp, &f->exts, &e); 655 656 return 0; 657 errout: 658 tcf_exts_destroy(&e); 659 return err; 660 } 661 662 static u32 fl_grab_new_handle(struct tcf_proto *tp, 663 struct cls_fl_head *head) 664 { 665 unsigned int i = 0x80000000; 666 u32 handle; 667 668 do { 669 if (++head->hgen == 0x7FFFFFFF) 670 head->hgen = 1; 671 } while (--i > 0 && fl_get(tp, head->hgen)); 672 673 if (unlikely(i == 0)) { 674 pr_err("Insufficient number of handles\n"); 675 handle = 0; 676 } else { 677 handle = head->hgen; 678 } 679 680 return handle; 681 } 682 683 static int fl_change(struct net *net, struct sk_buff *in_skb, 684 struct tcf_proto *tp, unsigned long base, 685 u32 handle, struct nlattr **tca, 686 unsigned long *arg, bool ovr) 687 { 688 struct cls_fl_head *head = rtnl_dereference(tp->root); 689 struct cls_fl_filter *fold = (struct cls_fl_filter *) *arg; 690 struct cls_fl_filter *fnew; 691 struct nlattr *tb[TCA_FLOWER_MAX + 1]; 692 struct fl_flow_mask mask = {}; 693 int err; 694 695 if (!tca[TCA_OPTIONS]) 696 return -EINVAL; 697 698 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], fl_policy); 699 if (err < 0) 700 return err; 701 702 if (fold && handle && fold->handle != handle) 703 return -EINVAL; 704 705 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 706 if (!fnew) 707 return -ENOBUFS; 708 709 err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0); 710 if (err < 0) 711 goto errout; 712 713 if (!handle) { 714 handle = fl_grab_new_handle(tp, head); 715 if (!handle) { 716 err = -EINVAL; 717 goto errout; 718 } 719 } 720 fnew->handle = handle; 721 722 if (tb[TCA_FLOWER_FLAGS]) { 723 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 724 725 if (!tc_flags_valid(fnew->flags)) { 726 err = -EINVAL; 727 goto errout; 728 } 729 } 730 731 err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr); 732 if (err) 733 goto errout; 734 735 err = fl_check_assign_mask(head, &mask); 736 if (err) 737 goto errout; 738 739 if (!tc_skip_sw(fnew->flags)) { 740 err = rhashtable_insert_fast(&head->ht, &fnew->ht_node, 741 head->ht_params); 742 if (err) 743 goto errout; 744 } 745 746 err = fl_hw_replace_filter(tp, 747 &head->dissector, 748 &mask.key, 749 &fnew->key, 750 &fnew->exts, 751 (unsigned long)fnew, 752 fnew->flags); 753 if (err) 754 goto errout; 755 756 if (fold) { 757 rhashtable_remove_fast(&head->ht, &fold->ht_node, 758 head->ht_params); 759 fl_hw_destroy_filter(tp, (unsigned long)fold); 760 } 761 762 *arg = (unsigned long) fnew; 763 764 if (fold) { 765 list_replace_rcu(&fold->list, &fnew->list); 766 tcf_unbind_filter(tp, &fold->res); 767 call_rcu(&fold->rcu, fl_destroy_filter); 768 } else { 769 list_add_tail_rcu(&fnew->list, &head->filters); 770 } 771 772 return 0; 773 774 errout: 775 tcf_exts_destroy(&fnew->exts); 776 kfree(fnew); 777 return err; 778 } 779 780 static int fl_delete(struct tcf_proto *tp, unsigned long arg) 781 { 782 struct cls_fl_head *head = rtnl_dereference(tp->root); 783 struct cls_fl_filter *f = (struct cls_fl_filter *) arg; 784 785 rhashtable_remove_fast(&head->ht, &f->ht_node, 786 head->ht_params); 787 __fl_delete(tp, f); 788 return 0; 789 } 790 791 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg) 792 { 793 struct cls_fl_head *head = rtnl_dereference(tp->root); 794 struct cls_fl_filter *f; 795 796 list_for_each_entry_rcu(f, &head->filters, list) { 797 if (arg->count < arg->skip) 798 goto skip; 799 if (arg->fn(tp, (unsigned long) f, arg) < 0) { 800 arg->stop = 1; 801 break; 802 } 803 skip: 804 arg->count++; 805 } 806 } 807 808 static int fl_dump_key_val(struct sk_buff *skb, 809 void *val, int val_type, 810 void *mask, int mask_type, int len) 811 { 812 int err; 813 814 if (!memchr_inv(mask, 0, len)) 815 return 0; 816 err = nla_put(skb, val_type, len, val); 817 if (err) 818 return err; 819 if (mask_type != TCA_FLOWER_UNSPEC) { 820 err = nla_put(skb, mask_type, len, mask); 821 if (err) 822 return err; 823 } 824 return 0; 825 } 826 827 static int fl_dump_key_vlan(struct sk_buff *skb, 828 struct flow_dissector_key_vlan *vlan_key, 829 struct flow_dissector_key_vlan *vlan_mask) 830 { 831 int err; 832 833 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 834 return 0; 835 if (vlan_mask->vlan_id) { 836 err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID, 837 vlan_key->vlan_id); 838 if (err) 839 return err; 840 } 841 if (vlan_mask->vlan_priority) { 842 err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO, 843 vlan_key->vlan_priority); 844 if (err) 845 return err; 846 } 847 return 0; 848 } 849 850 static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh, 851 struct sk_buff *skb, struct tcmsg *t) 852 { 853 struct cls_fl_head *head = rtnl_dereference(tp->root); 854 struct cls_fl_filter *f = (struct cls_fl_filter *) fh; 855 struct nlattr *nest; 856 struct fl_flow_key *key, *mask; 857 858 if (!f) 859 return skb->len; 860 861 t->tcm_handle = f->handle; 862 863 nest = nla_nest_start(skb, TCA_OPTIONS); 864 if (!nest) 865 goto nla_put_failure; 866 867 if (f->res.classid && 868 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 869 goto nla_put_failure; 870 871 key = &f->key; 872 mask = &head->mask.key; 873 874 if (mask->indev_ifindex) { 875 struct net_device *dev; 876 877 dev = __dev_get_by_index(net, key->indev_ifindex); 878 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 879 goto nla_put_failure; 880 } 881 882 fl_hw_update_stats(tp, f); 883 884 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 885 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 886 sizeof(key->eth.dst)) || 887 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 888 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 889 sizeof(key->eth.src)) || 890 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 891 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 892 sizeof(key->basic.n_proto))) 893 goto nla_put_failure; 894 895 if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan)) 896 goto nla_put_failure; 897 898 if ((key->basic.n_proto == htons(ETH_P_IP) || 899 key->basic.n_proto == htons(ETH_P_IPV6)) && 900 fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 901 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 902 sizeof(key->basic.ip_proto))) 903 goto nla_put_failure; 904 905 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 906 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 907 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 908 sizeof(key->ipv4.src)) || 909 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 910 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 911 sizeof(key->ipv4.dst)))) 912 goto nla_put_failure; 913 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 914 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 915 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 916 sizeof(key->ipv6.src)) || 917 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 918 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 919 sizeof(key->ipv6.dst)))) 920 goto nla_put_failure; 921 922 if (key->basic.ip_proto == IPPROTO_TCP && 923 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 924 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 925 sizeof(key->tp.src)) || 926 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 927 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 928 sizeof(key->tp.dst)))) 929 goto nla_put_failure; 930 else if (key->basic.ip_proto == IPPROTO_UDP && 931 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 932 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 933 sizeof(key->tp.src)) || 934 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 935 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 936 sizeof(key->tp.dst)))) 937 goto nla_put_failure; 938 else if (key->basic.ip_proto == IPPROTO_SCTP && 939 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 940 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 941 sizeof(key->tp.src)) || 942 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 943 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 944 sizeof(key->tp.dst)))) 945 goto nla_put_failure; 946 947 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 948 (fl_dump_key_val(skb, &key->enc_ipv4.src, 949 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 950 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 951 sizeof(key->enc_ipv4.src)) || 952 fl_dump_key_val(skb, &key->enc_ipv4.dst, 953 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 954 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 955 sizeof(key->enc_ipv4.dst)))) 956 goto nla_put_failure; 957 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 958 (fl_dump_key_val(skb, &key->enc_ipv6.src, 959 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 960 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 961 sizeof(key->enc_ipv6.src)) || 962 fl_dump_key_val(skb, &key->enc_ipv6.dst, 963 TCA_FLOWER_KEY_ENC_IPV6_DST, 964 &mask->enc_ipv6.dst, 965 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 966 sizeof(key->enc_ipv6.dst)))) 967 goto nla_put_failure; 968 969 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 970 &mask->enc_key_id, TCA_FLOWER_UNSPEC, 971 sizeof(key->enc_key_id)) || 972 fl_dump_key_val(skb, &key->enc_tp.src, 973 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 974 &mask->enc_tp.src, 975 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 976 sizeof(key->enc_tp.src)) || 977 fl_dump_key_val(skb, &key->enc_tp.dst, 978 TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 979 &mask->enc_tp.dst, 980 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 981 sizeof(key->enc_tp.dst))) 982 goto nla_put_failure; 983 984 nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags); 985 986 if (tcf_exts_dump(skb, &f->exts)) 987 goto nla_put_failure; 988 989 nla_nest_end(skb, nest); 990 991 if (tcf_exts_dump_stats(skb, &f->exts) < 0) 992 goto nla_put_failure; 993 994 return skb->len; 995 996 nla_put_failure: 997 nla_nest_cancel(skb, nest); 998 return -1; 999 } 1000 1001 static struct tcf_proto_ops cls_fl_ops __read_mostly = { 1002 .kind = "flower", 1003 .classify = fl_classify, 1004 .init = fl_init, 1005 .destroy = fl_destroy, 1006 .get = fl_get, 1007 .change = fl_change, 1008 .delete = fl_delete, 1009 .walk = fl_walk, 1010 .dump = fl_dump, 1011 .owner = THIS_MODULE, 1012 }; 1013 1014 static int __init cls_fl_init(void) 1015 { 1016 return register_tcf_proto_ops(&cls_fl_ops); 1017 } 1018 1019 static void __exit cls_fl_exit(void) 1020 { 1021 unregister_tcf_proto_ops(&cls_fl_ops); 1022 } 1023 1024 module_init(cls_fl_init); 1025 module_exit(cls_fl_exit); 1026 1027 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 1028 MODULE_DESCRIPTION("Flower classifier"); 1029 MODULE_LICENSE("GPL v2"); 1030