1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008, Intel Corporation. 4 * 5 * Author: Alexander Duyck <alexander.h.duyck@intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/skbuff.h> 12 #include <linux/rtnetlink.h> 13 #include <net/netlink.h> 14 #include <net/pkt_sched.h> 15 #include <net/ip.h> 16 #include <net/ipv6.h> 17 #include <net/dsfield.h> 18 #include <net/pkt_cls.h> 19 #include <net/tc_wrapper.h> 20 21 #include <linux/tc_act/tc_skbedit.h> 22 #include <net/tc_act/tc_skbedit.h> 23 24 static struct tc_action_ops act_skbedit_ops; 25 26 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params, 27 struct sk_buff *skb) 28 { 29 u16 queue_mapping = params->queue_mapping; 30 31 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) { 32 u32 hash = skb_get_hash(skb); 33 34 queue_mapping += hash % params->mapping_mod; 35 } 36 37 return netdev_cap_txqueue(skb->dev, queue_mapping); 38 } 39 40 TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, 41 const struct tc_action *a, 42 struct tcf_result *res) 43 { 44 struct tcf_skbedit *d = to_skbedit(a); 45 struct tcf_skbedit_params *params; 46 47 tcf_lastuse_update(&d->tcf_tm); 48 bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb); 49 50 params = rcu_dereference_bh(d->params); 51 52 if (params->flags & SKBEDIT_F_PRIORITY) 53 skb->priority = params->priority; 54 if (params->flags & SKBEDIT_F_INHERITDSFIELD) { 55 int wlen = skb_network_offset(skb); 56 57 switch (skb_protocol(skb, true)) { 58 case htons(ETH_P_IP): 59 wlen += sizeof(struct iphdr); 60 if (!pskb_may_pull(skb, wlen)) 61 goto err; 62 skb->priority = ipv4_get_dsfield(ip_hdr(skb)) >> 2; 63 break; 64 65 case htons(ETH_P_IPV6): 66 wlen += sizeof(struct ipv6hdr); 67 if (!pskb_may_pull(skb, wlen)) 68 goto err; 69 skb->priority = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2; 70 break; 71 } 72 } 73 if (params->flags & SKBEDIT_F_QUEUE_MAPPING && 74 skb->dev->real_num_tx_queues > params->queue_mapping) { 75 #ifdef CONFIG_NET_EGRESS 76 netdev_xmit_skip_txqueue(true); 77 #endif 78 skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb)); 79 } 80 if (params->flags & SKBEDIT_F_MARK) { 81 skb->mark &= ~params->mask; 82 skb->mark |= params->mark & params->mask; 83 } 84 if (params->flags & SKBEDIT_F_PTYPE) 85 skb->pkt_type = params->ptype; 86 return params->action; 87 88 err: 89 qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats)); 90 return TC_ACT_SHOT; 91 } 92 93 static void tcf_skbedit_stats_update(struct tc_action *a, u64 bytes, 94 u64 packets, u64 drops, 95 u64 lastuse, bool hw) 96 { 97 struct tcf_skbedit *d = to_skbedit(a); 98 struct tcf_t *tm = &d->tcf_tm; 99 100 tcf_action_update_stats(a, bytes, packets, drops, hw); 101 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 102 } 103 104 static const struct nla_policy skbedit_policy[TCA_SKBEDIT_MAX + 1] = { 105 [TCA_SKBEDIT_PARMS] = { .len = sizeof(struct tc_skbedit) }, 106 [TCA_SKBEDIT_PRIORITY] = { .len = sizeof(u32) }, 107 [TCA_SKBEDIT_QUEUE_MAPPING] = { .len = sizeof(u16) }, 108 [TCA_SKBEDIT_MARK] = { .len = sizeof(u32) }, 109 [TCA_SKBEDIT_PTYPE] = { .len = sizeof(u16) }, 110 [TCA_SKBEDIT_MASK] = { .len = sizeof(u32) }, 111 [TCA_SKBEDIT_FLAGS] = { .len = sizeof(u64) }, 112 [TCA_SKBEDIT_QUEUE_MAPPING_MAX] = { .len = sizeof(u16) }, 113 }; 114 115 static int tcf_skbedit_init(struct net *net, struct nlattr *nla, 116 struct nlattr *est, struct tc_action **a, 117 struct tcf_proto *tp, u32 act_flags, 118 struct netlink_ext_ack *extack) 119 { 120 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id); 121 bool bind = act_flags & TCA_ACT_FLAGS_BIND; 122 struct tcf_skbedit_params *params_new; 123 struct nlattr *tb[TCA_SKBEDIT_MAX + 1]; 124 struct tcf_chain *goto_ch = NULL; 125 struct tc_skbedit *parm; 126 struct tcf_skbedit *d; 127 u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL; 128 u16 *queue_mapping = NULL, *ptype = NULL; 129 u32 mapping_mod = 1; 130 bool exists = false; 131 int ret = 0, err; 132 u32 index; 133 134 if (nla == NULL) 135 return -EINVAL; 136 137 err = nla_parse_nested_deprecated(tb, TCA_SKBEDIT_MAX, nla, 138 skbedit_policy, NULL); 139 if (err < 0) 140 return err; 141 142 if (tb[TCA_SKBEDIT_PARMS] == NULL) 143 return -EINVAL; 144 145 if (tb[TCA_SKBEDIT_PRIORITY] != NULL) { 146 flags |= SKBEDIT_F_PRIORITY; 147 priority = nla_data(tb[TCA_SKBEDIT_PRIORITY]); 148 } 149 150 if (tb[TCA_SKBEDIT_QUEUE_MAPPING] != NULL) { 151 if (is_tcf_skbedit_ingress(act_flags) && 152 !(act_flags & TCA_ACT_FLAGS_SKIP_SW)) { 153 NL_SET_ERR_MSG_MOD(extack, "\"queue_mapping\" option on receive side is hardware only, use skip_sw"); 154 return -EOPNOTSUPP; 155 } 156 flags |= SKBEDIT_F_QUEUE_MAPPING; 157 queue_mapping = nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING]); 158 } 159 160 if (tb[TCA_SKBEDIT_PTYPE] != NULL) { 161 ptype = nla_data(tb[TCA_SKBEDIT_PTYPE]); 162 if (!skb_pkt_type_ok(*ptype)) 163 return -EINVAL; 164 flags |= SKBEDIT_F_PTYPE; 165 } 166 167 if (tb[TCA_SKBEDIT_MARK] != NULL) { 168 flags |= SKBEDIT_F_MARK; 169 mark = nla_data(tb[TCA_SKBEDIT_MARK]); 170 } 171 172 if (tb[TCA_SKBEDIT_MASK] != NULL) { 173 flags |= SKBEDIT_F_MASK; 174 mask = nla_data(tb[TCA_SKBEDIT_MASK]); 175 } 176 177 if (tb[TCA_SKBEDIT_FLAGS] != NULL) { 178 u64 *pure_flags = nla_data(tb[TCA_SKBEDIT_FLAGS]); 179 180 if (*pure_flags & SKBEDIT_F_TXQ_SKBHASH) { 181 u16 *queue_mapping_max; 182 183 if (!tb[TCA_SKBEDIT_QUEUE_MAPPING] || 184 !tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]) { 185 NL_SET_ERR_MSG_MOD(extack, "Missing required range of queue_mapping."); 186 return -EINVAL; 187 } 188 189 queue_mapping_max = 190 nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]); 191 if (*queue_mapping_max < *queue_mapping) { 192 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid, max < min."); 193 return -EINVAL; 194 } 195 196 mapping_mod = *queue_mapping_max - *queue_mapping + 1; 197 if (mapping_mod > U16_MAX) { 198 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid."); 199 return -EINVAL; 200 } 201 flags |= SKBEDIT_F_TXQ_SKBHASH; 202 } 203 if (*pure_flags & SKBEDIT_F_INHERITDSFIELD) 204 flags |= SKBEDIT_F_INHERITDSFIELD; 205 } 206 207 parm = nla_data(tb[TCA_SKBEDIT_PARMS]); 208 index = parm->index; 209 err = tcf_idr_check_alloc(tn, &index, a, bind); 210 if (err < 0) 211 return err; 212 exists = err; 213 if (exists && bind) 214 return ACT_P_BOUND; 215 216 if (!flags) { 217 if (exists) 218 tcf_idr_release(*a, bind); 219 else 220 tcf_idr_cleanup(tn, index); 221 return -EINVAL; 222 } 223 224 if (!exists) { 225 ret = tcf_idr_create(tn, index, est, a, 226 &act_skbedit_ops, bind, true, act_flags); 227 if (ret) { 228 tcf_idr_cleanup(tn, index); 229 return ret; 230 } 231 232 d = to_skbedit(*a); 233 ret = ACT_P_CREATED; 234 } else { 235 d = to_skbedit(*a); 236 if (!(act_flags & TCA_ACT_FLAGS_REPLACE)) { 237 tcf_idr_release(*a, bind); 238 return -EEXIST; 239 } 240 } 241 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 242 if (err < 0) 243 goto release_idr; 244 245 params_new = kzalloc(sizeof(*params_new), GFP_KERNEL); 246 if (unlikely(!params_new)) { 247 err = -ENOMEM; 248 goto put_chain; 249 } 250 251 params_new->flags = flags; 252 if (flags & SKBEDIT_F_PRIORITY) 253 params_new->priority = *priority; 254 if (flags & SKBEDIT_F_QUEUE_MAPPING) { 255 params_new->queue_mapping = *queue_mapping; 256 params_new->mapping_mod = mapping_mod; 257 } 258 if (flags & SKBEDIT_F_MARK) 259 params_new->mark = *mark; 260 if (flags & SKBEDIT_F_PTYPE) 261 params_new->ptype = *ptype; 262 /* default behaviour is to use all the bits */ 263 params_new->mask = 0xffffffff; 264 if (flags & SKBEDIT_F_MASK) 265 params_new->mask = *mask; 266 267 params_new->action = parm->action; 268 spin_lock_bh(&d->tcf_lock); 269 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 270 params_new = rcu_replace_pointer(d->params, params_new, 271 lockdep_is_held(&d->tcf_lock)); 272 spin_unlock_bh(&d->tcf_lock); 273 if (params_new) 274 kfree_rcu(params_new, rcu); 275 if (goto_ch) 276 tcf_chain_put_by_act(goto_ch); 277 278 return ret; 279 put_chain: 280 if (goto_ch) 281 tcf_chain_put_by_act(goto_ch); 282 release_idr: 283 tcf_idr_release(*a, bind); 284 return err; 285 } 286 287 static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a, 288 int bind, int ref) 289 { 290 const struct tcf_skbedit *d = to_skbedit(a); 291 unsigned char *b = skb_tail_pointer(skb); 292 const struct tcf_skbedit_params *params; 293 struct tc_skbedit opt = { 294 .index = d->tcf_index, 295 .refcnt = refcount_read(&d->tcf_refcnt) - ref, 296 .bindcnt = atomic_read(&d->tcf_bindcnt) - bind, 297 }; 298 u64 pure_flags = 0; 299 struct tcf_t t; 300 301 rcu_read_lock(); 302 params = rcu_dereference(d->params); 303 opt.action = params->action; 304 305 if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt)) 306 goto nla_put_failure; 307 if ((params->flags & SKBEDIT_F_PRIORITY) && 308 nla_put_u32(skb, TCA_SKBEDIT_PRIORITY, params->priority)) 309 goto nla_put_failure; 310 if ((params->flags & SKBEDIT_F_QUEUE_MAPPING) && 311 nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING, params->queue_mapping)) 312 goto nla_put_failure; 313 if ((params->flags & SKBEDIT_F_MARK) && 314 nla_put_u32(skb, TCA_SKBEDIT_MARK, params->mark)) 315 goto nla_put_failure; 316 if ((params->flags & SKBEDIT_F_PTYPE) && 317 nla_put_u16(skb, TCA_SKBEDIT_PTYPE, params->ptype)) 318 goto nla_put_failure; 319 if ((params->flags & SKBEDIT_F_MASK) && 320 nla_put_u32(skb, TCA_SKBEDIT_MASK, params->mask)) 321 goto nla_put_failure; 322 if (params->flags & SKBEDIT_F_INHERITDSFIELD) 323 pure_flags |= SKBEDIT_F_INHERITDSFIELD; 324 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) { 325 if (nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING_MAX, 326 params->queue_mapping + params->mapping_mod - 1)) 327 goto nla_put_failure; 328 329 pure_flags |= SKBEDIT_F_TXQ_SKBHASH; 330 } 331 if (pure_flags != 0 && 332 nla_put(skb, TCA_SKBEDIT_FLAGS, sizeof(pure_flags), &pure_flags)) 333 goto nla_put_failure; 334 335 tcf_tm_dump(&t, &d->tcf_tm); 336 if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD)) 337 goto nla_put_failure; 338 rcu_read_unlock(); 339 340 return skb->len; 341 342 nla_put_failure: 343 rcu_read_unlock(); 344 nlmsg_trim(skb, b); 345 return -1; 346 } 347 348 static void tcf_skbedit_cleanup(struct tc_action *a) 349 { 350 struct tcf_skbedit *d = to_skbedit(a); 351 struct tcf_skbedit_params *params; 352 353 params = rcu_dereference_protected(d->params, 1); 354 if (params) 355 kfree_rcu(params, rcu); 356 } 357 358 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act) 359 { 360 return nla_total_size(sizeof(struct tc_skbedit)) 361 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_PRIORITY */ 362 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING */ 363 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */ 364 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MARK */ 365 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_PTYPE */ 366 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MASK */ 367 + nla_total_size_64bit(sizeof(u64)); /* TCA_SKBEDIT_FLAGS */ 368 } 369 370 static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data, 371 u32 *index_inc, bool bind, 372 struct netlink_ext_ack *extack) 373 { 374 if (bind) { 375 struct flow_action_entry *entry = entry_data; 376 377 if (is_tcf_skbedit_mark(act)) { 378 entry->id = FLOW_ACTION_MARK; 379 entry->mark = tcf_skbedit_mark(act); 380 } else if (is_tcf_skbedit_ptype(act)) { 381 entry->id = FLOW_ACTION_PTYPE; 382 entry->ptype = tcf_skbedit_ptype(act); 383 } else if (is_tcf_skbedit_priority(act)) { 384 entry->id = FLOW_ACTION_PRIORITY; 385 entry->priority = tcf_skbedit_priority(act); 386 } else if (is_tcf_skbedit_tx_queue_mapping(act)) { 387 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used on transmit side"); 388 return -EOPNOTSUPP; 389 } else if (is_tcf_skbedit_rx_queue_mapping(act)) { 390 entry->id = FLOW_ACTION_RX_QUEUE_MAPPING; 391 entry->rx_queue = tcf_skbedit_rx_queue_mapping(act); 392 } else if (is_tcf_skbedit_inheritdsfield(act)) { 393 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used"); 394 return -EOPNOTSUPP; 395 } else { 396 NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload"); 397 return -EOPNOTSUPP; 398 } 399 *index_inc = 1; 400 } else { 401 struct flow_offload_action *fl_action = entry_data; 402 403 if (is_tcf_skbedit_mark(act)) 404 fl_action->id = FLOW_ACTION_MARK; 405 else if (is_tcf_skbedit_ptype(act)) 406 fl_action->id = FLOW_ACTION_PTYPE; 407 else if (is_tcf_skbedit_priority(act)) 408 fl_action->id = FLOW_ACTION_PRIORITY; 409 else if (is_tcf_skbedit_rx_queue_mapping(act)) 410 fl_action->id = FLOW_ACTION_RX_QUEUE_MAPPING; 411 else 412 return -EOPNOTSUPP; 413 } 414 415 return 0; 416 } 417 418 static struct tc_action_ops act_skbedit_ops = { 419 .kind = "skbedit", 420 .id = TCA_ID_SKBEDIT, 421 .owner = THIS_MODULE, 422 .act = tcf_skbedit_act, 423 .stats_update = tcf_skbedit_stats_update, 424 .dump = tcf_skbedit_dump, 425 .init = tcf_skbedit_init, 426 .cleanup = tcf_skbedit_cleanup, 427 .get_fill_size = tcf_skbedit_get_fill_size, 428 .offload_act_setup = tcf_skbedit_offload_act_setup, 429 .size = sizeof(struct tcf_skbedit), 430 }; 431 MODULE_ALIAS_NET_ACT("skbedit"); 432 433 static __net_init int skbedit_init_net(struct net *net) 434 { 435 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id); 436 437 return tc_action_net_init(net, tn, &act_skbedit_ops); 438 } 439 440 static void __net_exit skbedit_exit_net(struct list_head *net_list) 441 { 442 tc_action_net_exit(net_list, act_skbedit_ops.net_id); 443 } 444 445 static struct pernet_operations skbedit_net_ops = { 446 .init = skbedit_init_net, 447 .exit_batch = skbedit_exit_net, 448 .id = &act_skbedit_ops.net_id, 449 .size = sizeof(struct tc_action_net), 450 }; 451 452 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>"); 453 MODULE_DESCRIPTION("SKB Editing"); 454 MODULE_LICENSE("GPL"); 455 456 static int __init skbedit_init_module(void) 457 { 458 return tcf_register_action(&act_skbedit_ops, &skbedit_net_ops); 459 } 460 461 static void __exit skbedit_cleanup_module(void) 462 { 463 tcf_unregister_action(&act_skbedit_ops, &skbedit_net_ops); 464 } 465 466 module_init(skbedit_init_module); 467 module_exit(skbedit_cleanup_module); 468