1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 #include <linux/rhashtable.h> 9 10 struct flow_match { 11 struct flow_dissector *dissector; 12 void *mask; 13 void *key; 14 }; 15 16 struct flow_match_meta { 17 struct flow_dissector_key_meta *key, *mask; 18 }; 19 20 struct flow_match_basic { 21 struct flow_dissector_key_basic *key, *mask; 22 }; 23 24 struct flow_match_control { 25 struct flow_dissector_key_control *key, *mask; 26 }; 27 28 struct flow_match_eth_addrs { 29 struct flow_dissector_key_eth_addrs *key, *mask; 30 }; 31 32 struct flow_match_vlan { 33 struct flow_dissector_key_vlan *key, *mask; 34 }; 35 36 struct flow_match_ipv4_addrs { 37 struct flow_dissector_key_ipv4_addrs *key, *mask; 38 }; 39 40 struct flow_match_ipv6_addrs { 41 struct flow_dissector_key_ipv6_addrs *key, *mask; 42 }; 43 44 struct flow_match_ip { 45 struct flow_dissector_key_ip *key, *mask; 46 }; 47 48 struct flow_match_ports { 49 struct flow_dissector_key_ports *key, *mask; 50 }; 51 52 struct flow_match_icmp { 53 struct flow_dissector_key_icmp *key, *mask; 54 }; 55 56 struct flow_match_tcp { 57 struct flow_dissector_key_tcp *key, *mask; 58 }; 59 60 struct flow_match_mpls { 61 struct flow_dissector_key_mpls *key, *mask; 62 }; 63 64 struct flow_match_enc_keyid { 65 struct flow_dissector_key_keyid *key, *mask; 66 }; 67 68 struct flow_match_enc_opts { 69 struct flow_dissector_key_enc_opts *key, *mask; 70 }; 71 72 struct flow_match_ct { 73 struct flow_dissector_key_ct *key, *mask; 74 }; 75 76 struct flow_rule; 77 78 void flow_rule_match_meta(const struct flow_rule *rule, 79 struct flow_match_meta *out); 80 void flow_rule_match_basic(const struct flow_rule *rule, 81 struct flow_match_basic *out); 82 void flow_rule_match_control(const struct flow_rule *rule, 83 struct flow_match_control *out); 84 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 85 struct flow_match_eth_addrs *out); 86 void flow_rule_match_vlan(const struct flow_rule *rule, 87 struct flow_match_vlan *out); 88 void flow_rule_match_cvlan(const struct flow_rule *rule, 89 struct flow_match_vlan *out); 90 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 91 struct flow_match_ipv4_addrs *out); 92 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 93 struct flow_match_ipv6_addrs *out); 94 void flow_rule_match_ip(const struct flow_rule *rule, 95 struct flow_match_ip *out); 96 void flow_rule_match_ports(const struct flow_rule *rule, 97 struct flow_match_ports *out); 98 void flow_rule_match_tcp(const struct flow_rule *rule, 99 struct flow_match_tcp *out); 100 void flow_rule_match_icmp(const struct flow_rule *rule, 101 struct flow_match_icmp *out); 102 void flow_rule_match_mpls(const struct flow_rule *rule, 103 struct flow_match_mpls *out); 104 void flow_rule_match_enc_control(const struct flow_rule *rule, 105 struct flow_match_control *out); 106 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 107 struct flow_match_ipv4_addrs *out); 108 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 109 struct flow_match_ipv6_addrs *out); 110 void flow_rule_match_enc_ip(const struct flow_rule *rule, 111 struct flow_match_ip *out); 112 void flow_rule_match_enc_ports(const struct flow_rule *rule, 113 struct flow_match_ports *out); 114 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 115 struct flow_match_enc_keyid *out); 116 void flow_rule_match_enc_opts(const struct flow_rule *rule, 117 struct flow_match_enc_opts *out); 118 void flow_rule_match_ct(const struct flow_rule *rule, 119 struct flow_match_ct *out); 120 121 enum flow_action_id { 122 FLOW_ACTION_ACCEPT = 0, 123 FLOW_ACTION_DROP, 124 FLOW_ACTION_TRAP, 125 FLOW_ACTION_GOTO, 126 FLOW_ACTION_REDIRECT, 127 FLOW_ACTION_MIRRED, 128 FLOW_ACTION_REDIRECT_INGRESS, 129 FLOW_ACTION_MIRRED_INGRESS, 130 FLOW_ACTION_VLAN_PUSH, 131 FLOW_ACTION_VLAN_POP, 132 FLOW_ACTION_VLAN_MANGLE, 133 FLOW_ACTION_TUNNEL_ENCAP, 134 FLOW_ACTION_TUNNEL_DECAP, 135 FLOW_ACTION_MANGLE, 136 FLOW_ACTION_ADD, 137 FLOW_ACTION_CSUM, 138 FLOW_ACTION_MARK, 139 FLOW_ACTION_PTYPE, 140 FLOW_ACTION_WAKE, 141 FLOW_ACTION_QUEUE, 142 FLOW_ACTION_SAMPLE, 143 FLOW_ACTION_POLICE, 144 FLOW_ACTION_CT, 145 FLOW_ACTION_CT_METADATA, 146 FLOW_ACTION_MPLS_PUSH, 147 FLOW_ACTION_MPLS_POP, 148 FLOW_ACTION_MPLS_MANGLE, 149 NUM_FLOW_ACTIONS, 150 }; 151 152 /* This is mirroring enum pedit_header_type definition for easy mapping between 153 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 154 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 155 */ 156 enum flow_action_mangle_base { 157 FLOW_ACT_MANGLE_UNSPEC = 0, 158 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 159 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 160 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 161 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 162 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 163 }; 164 165 enum flow_action_hw_stats_type_bit { 166 FLOW_ACTION_HW_STATS_TYPE_IMMEDIATE_BIT, 167 FLOW_ACTION_HW_STATS_TYPE_DELAYED_BIT, 168 }; 169 170 enum flow_action_hw_stats_type { 171 FLOW_ACTION_HW_STATS_TYPE_DISABLED = 0, 172 FLOW_ACTION_HW_STATS_TYPE_IMMEDIATE = 173 BIT(FLOW_ACTION_HW_STATS_TYPE_IMMEDIATE_BIT), 174 FLOW_ACTION_HW_STATS_TYPE_DELAYED = 175 BIT(FLOW_ACTION_HW_STATS_TYPE_DELAYED_BIT), 176 FLOW_ACTION_HW_STATS_TYPE_ANY = 177 FLOW_ACTION_HW_STATS_TYPE_IMMEDIATE | 178 FLOW_ACTION_HW_STATS_TYPE_DELAYED, 179 }; 180 181 typedef void (*action_destr)(void *priv); 182 183 struct flow_action_cookie { 184 u32 cookie_len; 185 u8 cookie[]; 186 }; 187 188 struct flow_action_cookie *flow_action_cookie_create(void *data, 189 unsigned int len, 190 gfp_t gfp); 191 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 192 193 struct flow_action_entry { 194 enum flow_action_id id; 195 enum flow_action_hw_stats_type hw_stats_type; 196 action_destr destructor; 197 void *destructor_priv; 198 union { 199 u32 chain_index; /* FLOW_ACTION_GOTO */ 200 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 201 struct { /* FLOW_ACTION_VLAN */ 202 u16 vid; 203 __be16 proto; 204 u8 prio; 205 } vlan; 206 struct { /* FLOW_ACTION_PACKET_EDIT */ 207 enum flow_action_mangle_base htype; 208 u32 offset; 209 u32 mask; 210 u32 val; 211 } mangle; 212 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 213 u32 csum_flags; /* FLOW_ACTION_CSUM */ 214 u32 mark; /* FLOW_ACTION_MARK */ 215 u16 ptype; /* FLOW_ACTION_PTYPE */ 216 struct { /* FLOW_ACTION_QUEUE */ 217 u32 ctx; 218 u32 index; 219 u8 vf; 220 } queue; 221 struct { /* FLOW_ACTION_SAMPLE */ 222 struct psample_group *psample_group; 223 u32 rate; 224 u32 trunc_size; 225 bool truncate; 226 } sample; 227 struct { /* FLOW_ACTION_POLICE */ 228 s64 burst; 229 u64 rate_bytes_ps; 230 } police; 231 struct { /* FLOW_ACTION_CT */ 232 int action; 233 u16 zone; 234 struct nf_flowtable *flow_table; 235 } ct; 236 struct { 237 unsigned long cookie; 238 u32 mark; 239 u32 labels[4]; 240 } ct_metadata; 241 struct { /* FLOW_ACTION_MPLS_PUSH */ 242 u32 label; 243 __be16 proto; 244 u8 tc; 245 u8 bos; 246 u8 ttl; 247 } mpls_push; 248 struct { /* FLOW_ACTION_MPLS_POP */ 249 __be16 proto; 250 } mpls_pop; 251 struct { /* FLOW_ACTION_MPLS_MANGLE */ 252 u32 label; 253 u8 tc; 254 u8 bos; 255 u8 ttl; 256 } mpls_mangle; 257 }; 258 struct flow_action_cookie *cookie; /* user defined action cookie */ 259 }; 260 261 struct flow_action { 262 unsigned int num_entries; 263 struct flow_action_entry entries[]; 264 }; 265 266 static inline bool flow_action_has_entries(const struct flow_action *action) 267 { 268 return action->num_entries; 269 } 270 271 /** 272 * flow_action_has_one_action() - check if exactly one action is present 273 * @action: tc filter flow offload action 274 * 275 * Returns true if exactly one action is present. 276 */ 277 static inline bool flow_offload_has_one_action(const struct flow_action *action) 278 { 279 return action->num_entries == 1; 280 } 281 282 #define flow_action_for_each(__i, __act, __actions) \ 283 for (__i = 0, __act = &(__actions)->entries[0]; \ 284 __i < (__actions)->num_entries; \ 285 __act = &(__actions)->entries[++__i]) 286 287 static inline bool 288 flow_action_mixed_hw_stats_types_check(const struct flow_action *action, 289 struct netlink_ext_ack *extack) 290 { 291 const struct flow_action_entry *action_entry; 292 u8 uninitialized_var(last_hw_stats_type); 293 int i; 294 295 if (flow_offload_has_one_action(action)) 296 return true; 297 298 flow_action_for_each(i, action_entry, action) { 299 if (i && action_entry->hw_stats_type != last_hw_stats_type) { 300 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 301 return false; 302 } 303 last_hw_stats_type = action_entry->hw_stats_type; 304 } 305 return true; 306 } 307 308 static inline const struct flow_action_entry * 309 flow_action_first_entry_get(const struct flow_action *action) 310 { 311 WARN_ON(!flow_action_has_entries(action)); 312 return &action->entries[0]; 313 } 314 315 static inline bool 316 __flow_action_hw_stats_types_check(const struct flow_action *action, 317 struct netlink_ext_ack *extack, 318 bool check_allow_bit, 319 enum flow_action_hw_stats_type_bit allow_bit) 320 { 321 const struct flow_action_entry *action_entry; 322 323 if (!flow_action_has_entries(action)) 324 return true; 325 if (!flow_action_mixed_hw_stats_types_check(action, extack)) 326 return false; 327 action_entry = flow_action_first_entry_get(action); 328 if (!check_allow_bit && 329 action_entry->hw_stats_type != FLOW_ACTION_HW_STATS_TYPE_ANY) { 330 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 331 return false; 332 } else if (check_allow_bit && 333 !(action_entry->hw_stats_type & BIT(allow_bit))) { 334 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 335 return false; 336 } 337 return true; 338 } 339 340 static inline bool 341 flow_action_hw_stats_types_check(const struct flow_action *action, 342 struct netlink_ext_ack *extack, 343 enum flow_action_hw_stats_type_bit allow_bit) 344 { 345 return __flow_action_hw_stats_types_check(action, extack, 346 true, allow_bit); 347 } 348 349 static inline bool 350 flow_action_basic_hw_stats_types_check(const struct flow_action *action, 351 struct netlink_ext_ack *extack) 352 { 353 return __flow_action_hw_stats_types_check(action, extack, false, 0); 354 } 355 356 struct flow_rule { 357 struct flow_match match; 358 struct flow_action action; 359 }; 360 361 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 362 363 static inline bool flow_rule_match_key(const struct flow_rule *rule, 364 enum flow_dissector_key_id key) 365 { 366 return dissector_uses_key(rule->match.dissector, key); 367 } 368 369 struct flow_stats { 370 u64 pkts; 371 u64 bytes; 372 u64 lastused; 373 }; 374 375 static inline void flow_stats_update(struct flow_stats *flow_stats, 376 u64 bytes, u64 pkts, u64 lastused) 377 { 378 flow_stats->pkts += pkts; 379 flow_stats->bytes += bytes; 380 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 381 } 382 383 enum flow_block_command { 384 FLOW_BLOCK_BIND, 385 FLOW_BLOCK_UNBIND, 386 }; 387 388 enum flow_block_binder_type { 389 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 390 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 391 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 392 }; 393 394 struct flow_block { 395 struct list_head cb_list; 396 }; 397 398 struct netlink_ext_ack; 399 400 struct flow_block_offload { 401 enum flow_block_command command; 402 enum flow_block_binder_type binder_type; 403 bool block_shared; 404 bool unlocked_driver_cb; 405 struct net *net; 406 struct flow_block *block; 407 struct list_head cb_list; 408 struct list_head *driver_block_list; 409 struct netlink_ext_ack *extack; 410 }; 411 412 enum tc_setup_type; 413 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 414 void *cb_priv); 415 416 struct flow_block_cb { 417 struct list_head driver_list; 418 struct list_head list; 419 flow_setup_cb_t *cb; 420 void *cb_ident; 421 void *cb_priv; 422 void (*release)(void *cb_priv); 423 unsigned int refcnt; 424 }; 425 426 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 427 void *cb_ident, void *cb_priv, 428 void (*release)(void *cb_priv)); 429 void flow_block_cb_free(struct flow_block_cb *block_cb); 430 431 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 432 flow_setup_cb_t *cb, void *cb_ident); 433 434 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 435 void flow_block_cb_incref(struct flow_block_cb *block_cb); 436 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 437 438 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 439 struct flow_block_offload *offload) 440 { 441 list_add_tail(&block_cb->list, &offload->cb_list); 442 } 443 444 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 445 struct flow_block_offload *offload) 446 { 447 list_move(&block_cb->list, &offload->cb_list); 448 } 449 450 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 451 struct list_head *driver_block_list); 452 453 int flow_block_cb_setup_simple(struct flow_block_offload *f, 454 struct list_head *driver_list, 455 flow_setup_cb_t *cb, 456 void *cb_ident, void *cb_priv, bool ingress_only); 457 458 enum flow_cls_command { 459 FLOW_CLS_REPLACE, 460 FLOW_CLS_DESTROY, 461 FLOW_CLS_STATS, 462 FLOW_CLS_TMPLT_CREATE, 463 FLOW_CLS_TMPLT_DESTROY, 464 }; 465 466 struct flow_cls_common_offload { 467 u32 chain_index; 468 __be16 protocol; 469 u32 prio; 470 struct netlink_ext_ack *extack; 471 }; 472 473 struct flow_cls_offload { 474 struct flow_cls_common_offload common; 475 enum flow_cls_command command; 476 unsigned long cookie; 477 struct flow_rule *rule; 478 struct flow_stats stats; 479 u32 classid; 480 }; 481 482 static inline struct flow_rule * 483 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 484 { 485 return flow_cmd->rule; 486 } 487 488 static inline void flow_block_init(struct flow_block *flow_block) 489 { 490 INIT_LIST_HEAD(&flow_block->cb_list); 491 } 492 493 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, void *cb_priv, 494 enum tc_setup_type type, void *type_data); 495 496 typedef void flow_indr_block_cmd_t(struct net_device *dev, 497 flow_indr_block_bind_cb_t *cb, void *cb_priv, 498 enum flow_block_command command); 499 500 struct flow_indr_block_entry { 501 flow_indr_block_cmd_t *cb; 502 struct list_head list; 503 }; 504 505 void flow_indr_add_block_cb(struct flow_indr_block_entry *entry); 506 507 void flow_indr_del_block_cb(struct flow_indr_block_entry *entry); 508 509 int __flow_indr_block_cb_register(struct net_device *dev, void *cb_priv, 510 flow_indr_block_bind_cb_t *cb, 511 void *cb_ident); 512 513 void __flow_indr_block_cb_unregister(struct net_device *dev, 514 flow_indr_block_bind_cb_t *cb, 515 void *cb_ident); 516 517 int flow_indr_block_cb_register(struct net_device *dev, void *cb_priv, 518 flow_indr_block_bind_cb_t *cb, void *cb_ident); 519 520 void flow_indr_block_cb_unregister(struct net_device *dev, 521 flow_indr_block_bind_cb_t *cb, 522 void *cb_ident); 523 524 void flow_indr_block_call(struct net_device *dev, 525 struct flow_block_offload *bo, 526 enum flow_block_command command); 527 528 #endif /* _NET_FLOW_OFFLOAD_H */ 529