1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 #include <linux/rhashtable.h> 9 10 struct flow_match { 11 struct flow_dissector *dissector; 12 void *mask; 13 void *key; 14 }; 15 16 struct flow_match_meta { 17 struct flow_dissector_key_meta *key, *mask; 18 }; 19 20 struct flow_match_basic { 21 struct flow_dissector_key_basic *key, *mask; 22 }; 23 24 struct flow_match_control { 25 struct flow_dissector_key_control *key, *mask; 26 }; 27 28 struct flow_match_eth_addrs { 29 struct flow_dissector_key_eth_addrs *key, *mask; 30 }; 31 32 struct flow_match_vlan { 33 struct flow_dissector_key_vlan *key, *mask; 34 }; 35 36 struct flow_match_ipv4_addrs { 37 struct flow_dissector_key_ipv4_addrs *key, *mask; 38 }; 39 40 struct flow_match_ipv6_addrs { 41 struct flow_dissector_key_ipv6_addrs *key, *mask; 42 }; 43 44 struct flow_match_ip { 45 struct flow_dissector_key_ip *key, *mask; 46 }; 47 48 struct flow_match_ports { 49 struct flow_dissector_key_ports *key, *mask; 50 }; 51 52 struct flow_match_icmp { 53 struct flow_dissector_key_icmp *key, *mask; 54 }; 55 56 struct flow_match_tcp { 57 struct flow_dissector_key_tcp *key, *mask; 58 }; 59 60 struct flow_match_mpls { 61 struct flow_dissector_key_mpls *key, *mask; 62 }; 63 64 struct flow_match_enc_keyid { 65 struct flow_dissector_key_keyid *key, *mask; 66 }; 67 68 struct flow_match_enc_opts { 69 struct flow_dissector_key_enc_opts *key, *mask; 70 }; 71 72 struct flow_match_ct { 73 struct flow_dissector_key_ct *key, *mask; 74 }; 75 76 struct flow_rule; 77 78 void flow_rule_match_meta(const struct flow_rule *rule, 79 struct flow_match_meta *out); 80 void flow_rule_match_basic(const struct flow_rule *rule, 81 struct flow_match_basic *out); 82 void flow_rule_match_control(const struct flow_rule *rule, 83 struct flow_match_control *out); 84 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 85 struct flow_match_eth_addrs *out); 86 void flow_rule_match_vlan(const struct flow_rule *rule, 87 struct flow_match_vlan *out); 88 void flow_rule_match_cvlan(const struct flow_rule *rule, 89 struct flow_match_vlan *out); 90 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 91 struct flow_match_ipv4_addrs *out); 92 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 93 struct flow_match_ipv6_addrs *out); 94 void flow_rule_match_ip(const struct flow_rule *rule, 95 struct flow_match_ip *out); 96 void flow_rule_match_ports(const struct flow_rule *rule, 97 struct flow_match_ports *out); 98 void flow_rule_match_tcp(const struct flow_rule *rule, 99 struct flow_match_tcp *out); 100 void flow_rule_match_icmp(const struct flow_rule *rule, 101 struct flow_match_icmp *out); 102 void flow_rule_match_mpls(const struct flow_rule *rule, 103 struct flow_match_mpls *out); 104 void flow_rule_match_enc_control(const struct flow_rule *rule, 105 struct flow_match_control *out); 106 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 107 struct flow_match_ipv4_addrs *out); 108 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 109 struct flow_match_ipv6_addrs *out); 110 void flow_rule_match_enc_ip(const struct flow_rule *rule, 111 struct flow_match_ip *out); 112 void flow_rule_match_enc_ports(const struct flow_rule *rule, 113 struct flow_match_ports *out); 114 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 115 struct flow_match_enc_keyid *out); 116 void flow_rule_match_enc_opts(const struct flow_rule *rule, 117 struct flow_match_enc_opts *out); 118 void flow_rule_match_ct(const struct flow_rule *rule, 119 struct flow_match_ct *out); 120 121 enum flow_action_id { 122 FLOW_ACTION_ACCEPT = 0, 123 FLOW_ACTION_DROP, 124 FLOW_ACTION_TRAP, 125 FLOW_ACTION_GOTO, 126 FLOW_ACTION_REDIRECT, 127 FLOW_ACTION_MIRRED, 128 FLOW_ACTION_REDIRECT_INGRESS, 129 FLOW_ACTION_MIRRED_INGRESS, 130 FLOW_ACTION_VLAN_PUSH, 131 FLOW_ACTION_VLAN_POP, 132 FLOW_ACTION_VLAN_MANGLE, 133 FLOW_ACTION_TUNNEL_ENCAP, 134 FLOW_ACTION_TUNNEL_DECAP, 135 FLOW_ACTION_MANGLE, 136 FLOW_ACTION_ADD, 137 FLOW_ACTION_CSUM, 138 FLOW_ACTION_MARK, 139 FLOW_ACTION_PTYPE, 140 FLOW_ACTION_PRIORITY, 141 FLOW_ACTION_WAKE, 142 FLOW_ACTION_QUEUE, 143 FLOW_ACTION_SAMPLE, 144 FLOW_ACTION_POLICE, 145 FLOW_ACTION_CT, 146 FLOW_ACTION_CT_METADATA, 147 FLOW_ACTION_MPLS_PUSH, 148 FLOW_ACTION_MPLS_POP, 149 FLOW_ACTION_MPLS_MANGLE, 150 NUM_FLOW_ACTIONS, 151 }; 152 153 /* This is mirroring enum pedit_header_type definition for easy mapping between 154 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 155 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 156 */ 157 enum flow_action_mangle_base { 158 FLOW_ACT_MANGLE_UNSPEC = 0, 159 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 160 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 161 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 162 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 163 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 164 }; 165 166 enum flow_action_hw_stats_bit { 167 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 168 FLOW_ACTION_HW_STATS_DELAYED_BIT, 169 FLOW_ACTION_HW_STATS_DISABLED_BIT, 170 }; 171 172 enum flow_action_hw_stats { 173 FLOW_ACTION_HW_STATS_DONT_CARE = 0, 174 FLOW_ACTION_HW_STATS_IMMEDIATE = 175 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 176 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 177 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 178 FLOW_ACTION_HW_STATS_DELAYED, 179 FLOW_ACTION_HW_STATS_DISABLED = 180 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 181 }; 182 183 typedef void (*action_destr)(void *priv); 184 185 struct flow_action_cookie { 186 u32 cookie_len; 187 u8 cookie[]; 188 }; 189 190 struct flow_action_cookie *flow_action_cookie_create(void *data, 191 unsigned int len, 192 gfp_t gfp); 193 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 194 195 struct flow_action_entry { 196 enum flow_action_id id; 197 enum flow_action_hw_stats hw_stats; 198 action_destr destructor; 199 void *destructor_priv; 200 union { 201 u32 chain_index; /* FLOW_ACTION_GOTO */ 202 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 203 struct { /* FLOW_ACTION_VLAN */ 204 u16 vid; 205 __be16 proto; 206 u8 prio; 207 } vlan; 208 struct { /* FLOW_ACTION_MANGLE */ 209 /* FLOW_ACTION_ADD */ 210 enum flow_action_mangle_base htype; 211 u32 offset; 212 u32 mask; 213 u32 val; 214 } mangle; 215 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 216 u32 csum_flags; /* FLOW_ACTION_CSUM */ 217 u32 mark; /* FLOW_ACTION_MARK */ 218 u16 ptype; /* FLOW_ACTION_PTYPE */ 219 u32 priority; /* FLOW_ACTION_PRIORITY */ 220 struct { /* FLOW_ACTION_QUEUE */ 221 u32 ctx; 222 u32 index; 223 u8 vf; 224 } queue; 225 struct { /* FLOW_ACTION_SAMPLE */ 226 struct psample_group *psample_group; 227 u32 rate; 228 u32 trunc_size; 229 bool truncate; 230 } sample; 231 struct { /* FLOW_ACTION_POLICE */ 232 s64 burst; 233 u64 rate_bytes_ps; 234 } police; 235 struct { /* FLOW_ACTION_CT */ 236 int action; 237 u16 zone; 238 struct nf_flowtable *flow_table; 239 } ct; 240 struct { 241 unsigned long cookie; 242 u32 mark; 243 u32 labels[4]; 244 } ct_metadata; 245 struct { /* FLOW_ACTION_MPLS_PUSH */ 246 u32 label; 247 __be16 proto; 248 u8 tc; 249 u8 bos; 250 u8 ttl; 251 } mpls_push; 252 struct { /* FLOW_ACTION_MPLS_POP */ 253 __be16 proto; 254 } mpls_pop; 255 struct { /* FLOW_ACTION_MPLS_MANGLE */ 256 u32 label; 257 u8 tc; 258 u8 bos; 259 u8 ttl; 260 } mpls_mangle; 261 }; 262 struct flow_action_cookie *cookie; /* user defined action cookie */ 263 }; 264 265 struct flow_action { 266 unsigned int num_entries; 267 struct flow_action_entry entries[]; 268 }; 269 270 static inline bool flow_action_has_entries(const struct flow_action *action) 271 { 272 return action->num_entries; 273 } 274 275 /** 276 * flow_action_has_one_action() - check if exactly one action is present 277 * @action: tc filter flow offload action 278 * 279 * Returns true if exactly one action is present. 280 */ 281 static inline bool flow_offload_has_one_action(const struct flow_action *action) 282 { 283 return action->num_entries == 1; 284 } 285 286 #define flow_action_for_each(__i, __act, __actions) \ 287 for (__i = 0, __act = &(__actions)->entries[0]; \ 288 __i < (__actions)->num_entries; \ 289 __act = &(__actions)->entries[++__i]) 290 291 static inline bool 292 flow_action_mixed_hw_stats_check(const struct flow_action *action, 293 struct netlink_ext_ack *extack) 294 { 295 const struct flow_action_entry *action_entry; 296 u8 uninitialized_var(last_hw_stats); 297 int i; 298 299 if (flow_offload_has_one_action(action)) 300 return true; 301 302 flow_action_for_each(i, action_entry, action) { 303 if (i && action_entry->hw_stats != last_hw_stats) { 304 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 305 return false; 306 } 307 last_hw_stats = action_entry->hw_stats; 308 } 309 return true; 310 } 311 312 static inline const struct flow_action_entry * 313 flow_action_first_entry_get(const struct flow_action *action) 314 { 315 WARN_ON(!flow_action_has_entries(action)); 316 return &action->entries[0]; 317 } 318 319 static inline bool 320 __flow_action_hw_stats_check(const struct flow_action *action, 321 struct netlink_ext_ack *extack, 322 bool check_allow_bit, 323 enum flow_action_hw_stats_bit allow_bit) 324 { 325 const struct flow_action_entry *action_entry; 326 327 if (!flow_action_has_entries(action)) 328 return true; 329 if (!flow_action_mixed_hw_stats_check(action, extack)) 330 return false; 331 332 action_entry = flow_action_first_entry_get(action); 333 if (action_entry->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE) 334 return true; 335 336 if (!check_allow_bit && 337 action_entry->hw_stats != FLOW_ACTION_HW_STATS_ANY) { 338 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 339 return false; 340 } else if (check_allow_bit && 341 !(action_entry->hw_stats & BIT(allow_bit))) { 342 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 343 return false; 344 } 345 return true; 346 } 347 348 static inline bool 349 flow_action_hw_stats_check(const struct flow_action *action, 350 struct netlink_ext_ack *extack, 351 enum flow_action_hw_stats_bit allow_bit) 352 { 353 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 354 } 355 356 static inline bool 357 flow_action_basic_hw_stats_check(const struct flow_action *action, 358 struct netlink_ext_ack *extack) 359 { 360 return __flow_action_hw_stats_check(action, extack, false, 0); 361 } 362 363 struct flow_rule { 364 struct flow_match match; 365 struct flow_action action; 366 }; 367 368 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 369 370 static inline bool flow_rule_match_key(const struct flow_rule *rule, 371 enum flow_dissector_key_id key) 372 { 373 return dissector_uses_key(rule->match.dissector, key); 374 } 375 376 struct flow_stats { 377 u64 pkts; 378 u64 bytes; 379 u64 lastused; 380 enum flow_action_hw_stats used_hw_stats; 381 bool used_hw_stats_valid; 382 }; 383 384 static inline void flow_stats_update(struct flow_stats *flow_stats, 385 u64 bytes, u64 pkts, u64 lastused, 386 enum flow_action_hw_stats used_hw_stats) 387 { 388 flow_stats->pkts += pkts; 389 flow_stats->bytes += bytes; 390 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 391 392 /* The driver should pass value with a maximum of one bit set. 393 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 394 */ 395 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 396 flow_stats->used_hw_stats |= used_hw_stats; 397 flow_stats->used_hw_stats_valid = true; 398 } 399 400 enum flow_block_command { 401 FLOW_BLOCK_BIND, 402 FLOW_BLOCK_UNBIND, 403 }; 404 405 enum flow_block_binder_type { 406 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 407 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 408 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 409 }; 410 411 struct flow_block { 412 struct list_head cb_list; 413 }; 414 415 struct netlink_ext_ack; 416 417 struct flow_block_offload { 418 enum flow_block_command command; 419 enum flow_block_binder_type binder_type; 420 bool block_shared; 421 bool unlocked_driver_cb; 422 struct net *net; 423 struct flow_block *block; 424 struct list_head cb_list; 425 struct list_head *driver_block_list; 426 struct netlink_ext_ack *extack; 427 }; 428 429 enum tc_setup_type; 430 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 431 void *cb_priv); 432 433 struct flow_block_cb { 434 struct list_head driver_list; 435 struct list_head list; 436 flow_setup_cb_t *cb; 437 void *cb_ident; 438 void *cb_priv; 439 void (*release)(void *cb_priv); 440 unsigned int refcnt; 441 }; 442 443 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 444 void *cb_ident, void *cb_priv, 445 void (*release)(void *cb_priv)); 446 void flow_block_cb_free(struct flow_block_cb *block_cb); 447 448 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 449 flow_setup_cb_t *cb, void *cb_ident); 450 451 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 452 void flow_block_cb_incref(struct flow_block_cb *block_cb); 453 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 454 455 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 456 struct flow_block_offload *offload) 457 { 458 list_add_tail(&block_cb->list, &offload->cb_list); 459 } 460 461 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 462 struct flow_block_offload *offload) 463 { 464 list_move(&block_cb->list, &offload->cb_list); 465 } 466 467 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 468 struct list_head *driver_block_list); 469 470 int flow_block_cb_setup_simple(struct flow_block_offload *f, 471 struct list_head *driver_list, 472 flow_setup_cb_t *cb, 473 void *cb_ident, void *cb_priv, bool ingress_only); 474 475 enum flow_cls_command { 476 FLOW_CLS_REPLACE, 477 FLOW_CLS_DESTROY, 478 FLOW_CLS_STATS, 479 FLOW_CLS_TMPLT_CREATE, 480 FLOW_CLS_TMPLT_DESTROY, 481 }; 482 483 struct flow_cls_common_offload { 484 u32 chain_index; 485 __be16 protocol; 486 u32 prio; 487 struct netlink_ext_ack *extack; 488 }; 489 490 struct flow_cls_offload { 491 struct flow_cls_common_offload common; 492 enum flow_cls_command command; 493 unsigned long cookie; 494 struct flow_rule *rule; 495 struct flow_stats stats; 496 u32 classid; 497 }; 498 499 static inline struct flow_rule * 500 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 501 { 502 return flow_cmd->rule; 503 } 504 505 static inline void flow_block_init(struct flow_block *flow_block) 506 { 507 INIT_LIST_HEAD(&flow_block->cb_list); 508 } 509 510 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, void *cb_priv, 511 enum tc_setup_type type, void *type_data); 512 513 typedef void flow_indr_block_cmd_t(struct net_device *dev, 514 flow_indr_block_bind_cb_t *cb, void *cb_priv, 515 enum flow_block_command command); 516 517 struct flow_indr_block_entry { 518 flow_indr_block_cmd_t *cb; 519 struct list_head list; 520 }; 521 522 void flow_indr_add_block_cb(struct flow_indr_block_entry *entry); 523 524 void flow_indr_del_block_cb(struct flow_indr_block_entry *entry); 525 526 int __flow_indr_block_cb_register(struct net_device *dev, void *cb_priv, 527 flow_indr_block_bind_cb_t *cb, 528 void *cb_ident); 529 530 void __flow_indr_block_cb_unregister(struct net_device *dev, 531 flow_indr_block_bind_cb_t *cb, 532 void *cb_ident); 533 534 int flow_indr_block_cb_register(struct net_device *dev, void *cb_priv, 535 flow_indr_block_bind_cb_t *cb, void *cb_ident); 536 537 void flow_indr_block_cb_unregister(struct net_device *dev, 538 flow_indr_block_bind_cb_t *cb, 539 void *cb_ident); 540 541 void flow_indr_block_call(struct net_device *dev, 542 struct flow_block_offload *bo, 543 enum flow_block_command command, 544 enum tc_setup_type type); 545 546 #endif /* _NET_FLOW_OFFLOAD_H */ 547