1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 9 struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13 }; 14 15 struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17 }; 18 19 struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21 }; 22 23 struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25 }; 26 27 struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29 }; 30 31 struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33 }; 34 35 struct flow_match_ipv4_addrs { 36 struct flow_dissector_key_ipv4_addrs *key, *mask; 37 }; 38 39 struct flow_match_ipv6_addrs { 40 struct flow_dissector_key_ipv6_addrs *key, *mask; 41 }; 42 43 struct flow_match_ip { 44 struct flow_dissector_key_ip *key, *mask; 45 }; 46 47 struct flow_match_ports { 48 struct flow_dissector_key_ports *key, *mask; 49 }; 50 51 struct flow_match_ports_range { 52 struct flow_dissector_key_ports_range *key, *mask; 53 }; 54 55 struct flow_match_icmp { 56 struct flow_dissector_key_icmp *key, *mask; 57 }; 58 59 struct flow_match_tcp { 60 struct flow_dissector_key_tcp *key, *mask; 61 }; 62 63 struct flow_match_mpls { 64 struct flow_dissector_key_mpls *key, *mask; 65 }; 66 67 struct flow_match_enc_keyid { 68 struct flow_dissector_key_keyid *key, *mask; 69 }; 70 71 struct flow_match_enc_opts { 72 struct flow_dissector_key_enc_opts *key, *mask; 73 }; 74 75 struct flow_match_ct { 76 struct flow_dissector_key_ct *key, *mask; 77 }; 78 79 struct flow_rule; 80 81 void flow_rule_match_meta(const struct flow_rule *rule, 82 struct flow_match_meta *out); 83 void flow_rule_match_basic(const struct flow_rule *rule, 84 struct flow_match_basic *out); 85 void flow_rule_match_control(const struct flow_rule *rule, 86 struct flow_match_control *out); 87 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 88 struct flow_match_eth_addrs *out); 89 void flow_rule_match_vlan(const struct flow_rule *rule, 90 struct flow_match_vlan *out); 91 void flow_rule_match_cvlan(const struct flow_rule *rule, 92 struct flow_match_vlan *out); 93 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 94 struct flow_match_ipv4_addrs *out); 95 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 96 struct flow_match_ipv6_addrs *out); 97 void flow_rule_match_ip(const struct flow_rule *rule, 98 struct flow_match_ip *out); 99 void flow_rule_match_ports(const struct flow_rule *rule, 100 struct flow_match_ports *out); 101 void flow_rule_match_ports_range(const struct flow_rule *rule, 102 struct flow_match_ports_range *out); 103 void flow_rule_match_tcp(const struct flow_rule *rule, 104 struct flow_match_tcp *out); 105 void flow_rule_match_icmp(const struct flow_rule *rule, 106 struct flow_match_icmp *out); 107 void flow_rule_match_mpls(const struct flow_rule *rule, 108 struct flow_match_mpls *out); 109 void flow_rule_match_enc_control(const struct flow_rule *rule, 110 struct flow_match_control *out); 111 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 112 struct flow_match_ipv4_addrs *out); 113 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 114 struct flow_match_ipv6_addrs *out); 115 void flow_rule_match_enc_ip(const struct flow_rule *rule, 116 struct flow_match_ip *out); 117 void flow_rule_match_enc_ports(const struct flow_rule *rule, 118 struct flow_match_ports *out); 119 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 120 struct flow_match_enc_keyid *out); 121 void flow_rule_match_enc_opts(const struct flow_rule *rule, 122 struct flow_match_enc_opts *out); 123 void flow_rule_match_ct(const struct flow_rule *rule, 124 struct flow_match_ct *out); 125 126 enum flow_action_id { 127 FLOW_ACTION_ACCEPT = 0, 128 FLOW_ACTION_DROP, 129 FLOW_ACTION_TRAP, 130 FLOW_ACTION_GOTO, 131 FLOW_ACTION_REDIRECT, 132 FLOW_ACTION_MIRRED, 133 FLOW_ACTION_REDIRECT_INGRESS, 134 FLOW_ACTION_MIRRED_INGRESS, 135 FLOW_ACTION_VLAN_PUSH, 136 FLOW_ACTION_VLAN_POP, 137 FLOW_ACTION_VLAN_MANGLE, 138 FLOW_ACTION_TUNNEL_ENCAP, 139 FLOW_ACTION_TUNNEL_DECAP, 140 FLOW_ACTION_MANGLE, 141 FLOW_ACTION_ADD, 142 FLOW_ACTION_CSUM, 143 FLOW_ACTION_MARK, 144 FLOW_ACTION_PTYPE, 145 FLOW_ACTION_PRIORITY, 146 FLOW_ACTION_WAKE, 147 FLOW_ACTION_QUEUE, 148 FLOW_ACTION_SAMPLE, 149 FLOW_ACTION_POLICE, 150 FLOW_ACTION_CT, 151 FLOW_ACTION_CT_METADATA, 152 FLOW_ACTION_MPLS_PUSH, 153 FLOW_ACTION_MPLS_POP, 154 FLOW_ACTION_MPLS_MANGLE, 155 FLOW_ACTION_GATE, 156 FLOW_ACTION_PPPOE_PUSH, 157 FLOW_ACTION_JUMP, 158 FLOW_ACTION_PIPE, 159 FLOW_ACTION_VLAN_PUSH_ETH, 160 FLOW_ACTION_VLAN_POP_ETH, 161 FLOW_ACTION_CONTINUE, 162 NUM_FLOW_ACTIONS, 163 }; 164 165 /* This is mirroring enum pedit_header_type definition for easy mapping between 166 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 167 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 168 */ 169 enum flow_action_mangle_base { 170 FLOW_ACT_MANGLE_UNSPEC = 0, 171 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 172 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 173 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 174 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 175 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 176 }; 177 178 enum flow_action_hw_stats_bit { 179 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 180 FLOW_ACTION_HW_STATS_DELAYED_BIT, 181 FLOW_ACTION_HW_STATS_DISABLED_BIT, 182 183 FLOW_ACTION_HW_STATS_NUM_BITS 184 }; 185 186 enum flow_action_hw_stats { 187 FLOW_ACTION_HW_STATS_IMMEDIATE = 188 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 189 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 190 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 191 FLOW_ACTION_HW_STATS_DELAYED, 192 FLOW_ACTION_HW_STATS_DISABLED = 193 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 194 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 195 }; 196 197 typedef void (*action_destr)(void *priv); 198 199 struct flow_action_cookie { 200 u32 cookie_len; 201 u8 cookie[]; 202 }; 203 204 struct flow_action_cookie *flow_action_cookie_create(void *data, 205 unsigned int len, 206 gfp_t gfp); 207 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 208 209 struct flow_action_entry { 210 enum flow_action_id id; 211 u32 hw_index; 212 enum flow_action_hw_stats hw_stats; 213 action_destr destructor; 214 void *destructor_priv; 215 union { 216 u32 chain_index; /* FLOW_ACTION_GOTO */ 217 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 218 struct { /* FLOW_ACTION_VLAN */ 219 u16 vid; 220 __be16 proto; 221 u8 prio; 222 } vlan; 223 struct { /* FLOW_ACTION_VLAN_PUSH_ETH */ 224 unsigned char dst[ETH_ALEN]; 225 unsigned char src[ETH_ALEN]; 226 } vlan_push_eth; 227 struct { /* FLOW_ACTION_MANGLE */ 228 /* FLOW_ACTION_ADD */ 229 enum flow_action_mangle_base htype; 230 u32 offset; 231 u32 mask; 232 u32 val; 233 } mangle; 234 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 235 u32 csum_flags; /* FLOW_ACTION_CSUM */ 236 u32 mark; /* FLOW_ACTION_MARK */ 237 u16 ptype; /* FLOW_ACTION_PTYPE */ 238 u32 priority; /* FLOW_ACTION_PRIORITY */ 239 struct { /* FLOW_ACTION_QUEUE */ 240 u32 ctx; 241 u32 index; 242 u8 vf; 243 } queue; 244 struct { /* FLOW_ACTION_SAMPLE */ 245 struct psample_group *psample_group; 246 u32 rate; 247 u32 trunc_size; 248 bool truncate; 249 } sample; 250 struct { /* FLOW_ACTION_POLICE */ 251 u32 burst; 252 u64 rate_bytes_ps; 253 u64 peakrate_bytes_ps; 254 u32 avrate; 255 u16 overhead; 256 u64 burst_pkt; 257 u64 rate_pkt_ps; 258 u32 mtu; 259 struct { 260 enum flow_action_id act_id; 261 u32 extval; 262 } exceed, notexceed; 263 } police; 264 struct { /* FLOW_ACTION_CT */ 265 int action; 266 u16 zone; 267 struct nf_flowtable *flow_table; 268 } ct; 269 struct { 270 unsigned long cookie; 271 u32 mark; 272 u32 labels[4]; 273 bool orig_dir; 274 } ct_metadata; 275 struct { /* FLOW_ACTION_MPLS_PUSH */ 276 u32 label; 277 __be16 proto; 278 u8 tc; 279 u8 bos; 280 u8 ttl; 281 } mpls_push; 282 struct { /* FLOW_ACTION_MPLS_POP */ 283 __be16 proto; 284 } mpls_pop; 285 struct { /* FLOW_ACTION_MPLS_MANGLE */ 286 u32 label; 287 u8 tc; 288 u8 bos; 289 u8 ttl; 290 } mpls_mangle; 291 struct { 292 s32 prio; 293 u64 basetime; 294 u64 cycletime; 295 u64 cycletimeext; 296 u32 num_entries; 297 struct action_gate_entry *entries; 298 } gate; 299 struct { /* FLOW_ACTION_PPPOE_PUSH */ 300 u16 sid; 301 } pppoe; 302 }; 303 struct flow_action_cookie *cookie; /* user defined action cookie */ 304 }; 305 306 struct flow_action { 307 unsigned int num_entries; 308 struct flow_action_entry entries[]; 309 }; 310 311 static inline bool flow_action_has_entries(const struct flow_action *action) 312 { 313 return action->num_entries; 314 } 315 316 /** 317 * flow_offload_has_one_action() - check if exactly one action is present 318 * @action: tc filter flow offload action 319 * 320 * Returns true if exactly one action is present. 321 */ 322 static inline bool flow_offload_has_one_action(const struct flow_action *action) 323 { 324 return action->num_entries == 1; 325 } 326 327 static inline bool flow_action_is_last_entry(const struct flow_action *action, 328 const struct flow_action_entry *entry) 329 { 330 return entry == &action->entries[action->num_entries - 1]; 331 } 332 333 #define flow_action_for_each(__i, __act, __actions) \ 334 for (__i = 0, __act = &(__actions)->entries[0]; \ 335 __i < (__actions)->num_entries; \ 336 __act = &(__actions)->entries[++__i]) 337 338 static inline bool 339 flow_action_mixed_hw_stats_check(const struct flow_action *action, 340 struct netlink_ext_ack *extack) 341 { 342 const struct flow_action_entry *action_entry; 343 u8 last_hw_stats; 344 int i; 345 346 if (flow_offload_has_one_action(action)) 347 return true; 348 349 flow_action_for_each(i, action_entry, action) { 350 if (i && action_entry->hw_stats != last_hw_stats) { 351 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 352 return false; 353 } 354 last_hw_stats = action_entry->hw_stats; 355 } 356 return true; 357 } 358 359 static inline const struct flow_action_entry * 360 flow_action_first_entry_get(const struct flow_action *action) 361 { 362 WARN_ON(!flow_action_has_entries(action)); 363 return &action->entries[0]; 364 } 365 366 static inline bool 367 __flow_action_hw_stats_check(const struct flow_action *action, 368 struct netlink_ext_ack *extack, 369 bool check_allow_bit, 370 enum flow_action_hw_stats_bit allow_bit) 371 { 372 const struct flow_action_entry *action_entry; 373 374 if (!flow_action_has_entries(action)) 375 return true; 376 if (!flow_action_mixed_hw_stats_check(action, extack)) 377 return false; 378 379 action_entry = flow_action_first_entry_get(action); 380 381 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 382 WARN_ON_ONCE(!action_entry->hw_stats); 383 384 if (!check_allow_bit && 385 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 386 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 387 return false; 388 } else if (check_allow_bit && 389 !(action_entry->hw_stats & BIT(allow_bit))) { 390 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 391 return false; 392 } 393 return true; 394 } 395 396 static inline bool 397 flow_action_hw_stats_check(const struct flow_action *action, 398 struct netlink_ext_ack *extack, 399 enum flow_action_hw_stats_bit allow_bit) 400 { 401 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 402 } 403 404 static inline bool 405 flow_action_basic_hw_stats_check(const struct flow_action *action, 406 struct netlink_ext_ack *extack) 407 { 408 return __flow_action_hw_stats_check(action, extack, false, 0); 409 } 410 411 struct flow_rule { 412 struct flow_match match; 413 struct flow_action action; 414 }; 415 416 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 417 418 static inline bool flow_rule_match_key(const struct flow_rule *rule, 419 enum flow_dissector_key_id key) 420 { 421 return dissector_uses_key(rule->match.dissector, key); 422 } 423 424 struct flow_stats { 425 u64 pkts; 426 u64 bytes; 427 u64 drops; 428 u64 lastused; 429 enum flow_action_hw_stats used_hw_stats; 430 bool used_hw_stats_valid; 431 }; 432 433 static inline void flow_stats_update(struct flow_stats *flow_stats, 434 u64 bytes, u64 pkts, 435 u64 drops, u64 lastused, 436 enum flow_action_hw_stats used_hw_stats) 437 { 438 flow_stats->pkts += pkts; 439 flow_stats->bytes += bytes; 440 flow_stats->drops += drops; 441 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 442 443 /* The driver should pass value with a maximum of one bit set. 444 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 445 */ 446 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 447 flow_stats->used_hw_stats |= used_hw_stats; 448 flow_stats->used_hw_stats_valid = true; 449 } 450 451 enum flow_block_command { 452 FLOW_BLOCK_BIND, 453 FLOW_BLOCK_UNBIND, 454 }; 455 456 enum flow_block_binder_type { 457 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 458 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 459 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 460 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 461 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 462 }; 463 464 struct flow_block { 465 struct list_head cb_list; 466 }; 467 468 struct netlink_ext_ack; 469 470 struct flow_block_offload { 471 enum flow_block_command command; 472 enum flow_block_binder_type binder_type; 473 bool block_shared; 474 bool unlocked_driver_cb; 475 struct net *net; 476 struct flow_block *block; 477 struct list_head cb_list; 478 struct list_head *driver_block_list; 479 struct netlink_ext_ack *extack; 480 struct Qdisc *sch; 481 struct list_head *cb_list_head; 482 }; 483 484 enum tc_setup_type; 485 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 486 void *cb_priv); 487 488 struct flow_block_cb; 489 490 struct flow_block_indr { 491 struct list_head list; 492 struct net_device *dev; 493 struct Qdisc *sch; 494 enum flow_block_binder_type binder_type; 495 void *data; 496 void *cb_priv; 497 void (*cleanup)(struct flow_block_cb *block_cb); 498 }; 499 500 struct flow_block_cb { 501 struct list_head driver_list; 502 struct list_head list; 503 flow_setup_cb_t *cb; 504 void *cb_ident; 505 void *cb_priv; 506 void (*release)(void *cb_priv); 507 struct flow_block_indr indr; 508 unsigned int refcnt; 509 }; 510 511 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 512 void *cb_ident, void *cb_priv, 513 void (*release)(void *cb_priv)); 514 struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 515 void *cb_ident, void *cb_priv, 516 void (*release)(void *cb_priv), 517 struct flow_block_offload *bo, 518 struct net_device *dev, 519 struct Qdisc *sch, void *data, 520 void *indr_cb_priv, 521 void (*cleanup)(struct flow_block_cb *block_cb)); 522 void flow_block_cb_free(struct flow_block_cb *block_cb); 523 524 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 525 flow_setup_cb_t *cb, void *cb_ident); 526 527 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 528 void flow_block_cb_incref(struct flow_block_cb *block_cb); 529 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 530 531 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 532 struct flow_block_offload *offload) 533 { 534 list_add_tail(&block_cb->list, &offload->cb_list); 535 } 536 537 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 538 struct flow_block_offload *offload) 539 { 540 list_move(&block_cb->list, &offload->cb_list); 541 } 542 543 static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 544 struct flow_block_offload *offload) 545 { 546 list_del(&block_cb->indr.list); 547 list_move(&block_cb->list, &offload->cb_list); 548 } 549 550 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 551 struct list_head *driver_block_list); 552 553 int flow_block_cb_setup_simple(struct flow_block_offload *f, 554 struct list_head *driver_list, 555 flow_setup_cb_t *cb, 556 void *cb_ident, void *cb_priv, bool ingress_only); 557 558 enum flow_cls_command { 559 FLOW_CLS_REPLACE, 560 FLOW_CLS_DESTROY, 561 FLOW_CLS_STATS, 562 FLOW_CLS_TMPLT_CREATE, 563 FLOW_CLS_TMPLT_DESTROY, 564 }; 565 566 struct flow_cls_common_offload { 567 u32 chain_index; 568 __be16 protocol; 569 u32 prio; 570 struct netlink_ext_ack *extack; 571 }; 572 573 struct flow_cls_offload { 574 struct flow_cls_common_offload common; 575 enum flow_cls_command command; 576 unsigned long cookie; 577 struct flow_rule *rule; 578 struct flow_stats stats; 579 u32 classid; 580 }; 581 582 enum offload_act_command { 583 FLOW_ACT_REPLACE, 584 FLOW_ACT_DESTROY, 585 FLOW_ACT_STATS, 586 }; 587 588 struct flow_offload_action { 589 struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ 590 enum offload_act_command command; 591 enum flow_action_id id; 592 u32 index; 593 struct flow_stats stats; 594 struct flow_action action; 595 }; 596 597 struct flow_offload_action *offload_action_alloc(unsigned int num_actions); 598 599 static inline struct flow_rule * 600 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 601 { 602 return flow_cmd->rule; 603 } 604 605 static inline void flow_block_init(struct flow_block *flow_block) 606 { 607 INIT_LIST_HEAD(&flow_block->cb_list); 608 } 609 610 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 611 enum tc_setup_type type, void *type_data, 612 void *data, 613 void (*cleanup)(struct flow_block_cb *block_cb)); 614 615 int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 616 void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 617 void (*release)(void *cb_priv)); 618 int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 619 enum tc_setup_type type, void *data, 620 struct flow_block_offload *bo, 621 void (*cleanup)(struct flow_block_cb *block_cb)); 622 bool flow_indr_dev_exists(void); 623 624 #endif /* _NET_FLOW_OFFLOAD_H */ 625