1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2008, Intel Corporation.
4 *
5 * Author: Alexander Duyck <alexander.h.duyck@intel.com>
6 */
7
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/skbuff.h>
12 #include <linux/rtnetlink.h>
13 #include <net/netlink.h>
14 #include <net/pkt_sched.h>
15 #include <net/ip.h>
16 #include <net/ipv6.h>
17 #include <net/dsfield.h>
18 #include <net/pkt_cls.h>
19 #include <net/tc_wrapper.h>
20
21 #include <linux/tc_act/tc_skbedit.h>
22 #include <net/tc_act/tc_skbedit.h>
23
24 static struct tc_action_ops act_skbedit_ops;
25
tcf_skbedit_hash(struct tcf_skbedit_params * params,struct sk_buff * skb)26 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params,
27 struct sk_buff *skb)
28 {
29 u16 queue_mapping = params->queue_mapping;
30
31 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
32 u32 hash = skb_get_hash(skb);
33
34 queue_mapping += hash % params->mapping_mod;
35 }
36
37 return netdev_cap_txqueue(skb->dev, queue_mapping);
38 }
39
tcf_skbedit_act(struct sk_buff * skb,const struct tc_action * a,struct tcf_result * res)40 TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb,
41 const struct tc_action *a,
42 struct tcf_result *res)
43 {
44 struct tcf_skbedit *d = to_skbedit(a);
45 struct tcf_skbedit_params *params;
46
47 tcf_lastuse_update(&d->tcf_tm);
48 bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb);
49
50 params = rcu_dereference_bh(d->params);
51
52 if (params->flags & SKBEDIT_F_PRIORITY)
53 skb->priority = params->priority;
54 if (params->flags & SKBEDIT_F_INHERITDSFIELD) {
55 int wlen = skb_network_offset(skb);
56
57 switch (skb_protocol(skb, true)) {
58 case htons(ETH_P_IP):
59 wlen += sizeof(struct iphdr);
60 if (!pskb_may_pull(skb, wlen))
61 goto err;
62 skb->priority = ipv4_get_dsfield(ip_hdr(skb)) >> 2;
63 break;
64
65 case htons(ETH_P_IPV6):
66 wlen += sizeof(struct ipv6hdr);
67 if (!pskb_may_pull(skb, wlen))
68 goto err;
69 skb->priority = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;
70 break;
71 }
72 }
73 if (params->flags & SKBEDIT_F_QUEUE_MAPPING &&
74 skb->dev->real_num_tx_queues > params->queue_mapping) {
75 if (skb_at_tc_egress(skb))
76 netdev_xmit_skip_txqueue(true);
77 skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb));
78 }
79 if (params->flags & SKBEDIT_F_MARK) {
80 skb->mark &= ~params->mask;
81 skb->mark |= params->mark & params->mask;
82 }
83 if (params->flags & SKBEDIT_F_PTYPE)
84 skb->pkt_type = params->ptype;
85 return params->action;
86
87 err:
88 qstats_cpu_drop_inc(d->common.cpu_qstats);
89 return TC_ACT_SHOT;
90 }
91
tcf_skbedit_stats_update(struct tc_action * a,u64 bytes,u64 packets,u64 drops,u64 lastuse,bool hw)92 static void tcf_skbedit_stats_update(struct tc_action *a, u64 bytes,
93 u64 packets, u64 drops,
94 u64 lastuse, bool hw)
95 {
96 struct tcf_skbedit *d = to_skbedit(a);
97 struct tcf_t *tm = &d->tcf_tm;
98
99 tcf_action_update_stats(a, bytes, packets, drops, hw);
100 tm->lastuse = max_t(u64, tm->lastuse, lastuse);
101 }
102
103 static const struct nla_policy skbedit_policy[TCA_SKBEDIT_MAX + 1] = {
104 [TCA_SKBEDIT_PARMS] = { .len = sizeof(struct tc_skbedit) },
105 [TCA_SKBEDIT_PRIORITY] = { .len = sizeof(u32) },
106 [TCA_SKBEDIT_QUEUE_MAPPING] = { .len = sizeof(u16) },
107 [TCA_SKBEDIT_MARK] = { .len = sizeof(u32) },
108 [TCA_SKBEDIT_PTYPE] = { .len = sizeof(u16) },
109 [TCA_SKBEDIT_MASK] = { .len = sizeof(u32) },
110 [TCA_SKBEDIT_FLAGS] = { .len = sizeof(u64) },
111 [TCA_SKBEDIT_QUEUE_MAPPING_MAX] = { .len = sizeof(u16) },
112 };
113
tcf_skbedit_init(struct net * net,struct nlattr * nla,struct nlattr * est,struct tc_action ** a,struct tcf_proto * tp,u32 act_flags,struct netlink_ext_ack * extack)114 static int tcf_skbedit_init(struct net *net, struct nlattr *nla,
115 struct nlattr *est, struct tc_action **a,
116 struct tcf_proto *tp, u32 act_flags,
117 struct netlink_ext_ack *extack)
118 {
119 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
120 bool bind = act_flags & TCA_ACT_FLAGS_BIND;
121 struct tcf_skbedit_params *params_new;
122 struct nlattr *tb[TCA_SKBEDIT_MAX + 1];
123 struct tcf_chain *goto_ch = NULL;
124 struct tc_skbedit *parm;
125 struct tcf_skbedit *d;
126 u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL;
127 u16 *queue_mapping = NULL, *ptype = NULL;
128 u32 mapping_mod = 1;
129 bool exists = false;
130 int ret = 0, err;
131 u32 index;
132
133 if (nla == NULL)
134 return -EINVAL;
135
136 err = nla_parse_nested_deprecated(tb, TCA_SKBEDIT_MAX, nla,
137 skbedit_policy, NULL);
138 if (err < 0)
139 return err;
140
141 if (tb[TCA_SKBEDIT_PARMS] == NULL)
142 return -EINVAL;
143
144 if (tb[TCA_SKBEDIT_PRIORITY] != NULL) {
145 flags |= SKBEDIT_F_PRIORITY;
146 priority = nla_data(tb[TCA_SKBEDIT_PRIORITY]);
147 }
148
149 if (tb[TCA_SKBEDIT_QUEUE_MAPPING] != NULL) {
150 if (is_tcf_skbedit_ingress(act_flags) &&
151 !(act_flags & TCA_ACT_FLAGS_SKIP_SW)) {
152 NL_SET_ERR_MSG_MOD(extack, "\"queue_mapping\" option on receive side is hardware only, use skip_sw");
153 return -EOPNOTSUPP;
154 }
155 flags |= SKBEDIT_F_QUEUE_MAPPING;
156 queue_mapping = nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING]);
157 }
158
159 if (tb[TCA_SKBEDIT_PTYPE] != NULL) {
160 ptype = nla_data(tb[TCA_SKBEDIT_PTYPE]);
161 if (!skb_pkt_type_ok(*ptype))
162 return -EINVAL;
163 flags |= SKBEDIT_F_PTYPE;
164 }
165
166 if (tb[TCA_SKBEDIT_MARK] != NULL) {
167 flags |= SKBEDIT_F_MARK;
168 mark = nla_data(tb[TCA_SKBEDIT_MARK]);
169 }
170
171 if (tb[TCA_SKBEDIT_MASK] != NULL) {
172 flags |= SKBEDIT_F_MASK;
173 mask = nla_data(tb[TCA_SKBEDIT_MASK]);
174 }
175
176 if (tb[TCA_SKBEDIT_FLAGS] != NULL) {
177 u64 *pure_flags = nla_data(tb[TCA_SKBEDIT_FLAGS]);
178
179 if (*pure_flags & SKBEDIT_F_TXQ_SKBHASH) {
180 u16 *queue_mapping_max;
181
182 if (!tb[TCA_SKBEDIT_QUEUE_MAPPING] ||
183 !tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]) {
184 NL_SET_ERR_MSG_MOD(extack, "Missing required range of queue_mapping.");
185 return -EINVAL;
186 }
187
188 queue_mapping_max =
189 nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]);
190 if (*queue_mapping_max < *queue_mapping) {
191 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid, max < min.");
192 return -EINVAL;
193 }
194
195 mapping_mod = *queue_mapping_max - *queue_mapping + 1;
196 if (mapping_mod > U16_MAX) {
197 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid.");
198 return -EINVAL;
199 }
200 flags |= SKBEDIT_F_TXQ_SKBHASH;
201 }
202 if (*pure_flags & SKBEDIT_F_INHERITDSFIELD)
203 flags |= SKBEDIT_F_INHERITDSFIELD;
204 }
205
206 parm = nla_data(tb[TCA_SKBEDIT_PARMS]);
207 index = parm->index;
208 err = tcf_idr_check_alloc(tn, &index, a, bind);
209 if (err < 0)
210 return err;
211 exists = err;
212 if (exists && bind)
213 return ACT_P_BOUND;
214
215 if (!flags) {
216 if (exists)
217 tcf_idr_release(*a, bind);
218 else
219 tcf_idr_cleanup(tn, index);
220 return -EINVAL;
221 }
222
223 if (!exists) {
224 ret = tcf_idr_create(tn, index, est, a,
225 &act_skbedit_ops, bind, true, act_flags);
226 if (ret) {
227 tcf_idr_cleanup(tn, index);
228 return ret;
229 }
230
231 d = to_skbedit(*a);
232 ret = ACT_P_CREATED;
233 } else {
234 d = to_skbedit(*a);
235 if (!(act_flags & TCA_ACT_FLAGS_REPLACE)) {
236 tcf_idr_release(*a, bind);
237 return -EEXIST;
238 }
239 }
240 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
241 if (err < 0)
242 goto release_idr;
243
244 params_new = kzalloc_obj(*params_new);
245 if (unlikely(!params_new)) {
246 err = -ENOMEM;
247 goto put_chain;
248 }
249
250 params_new->flags = flags;
251 if (flags & SKBEDIT_F_PRIORITY)
252 params_new->priority = *priority;
253 if (flags & SKBEDIT_F_QUEUE_MAPPING) {
254 params_new->queue_mapping = *queue_mapping;
255 params_new->mapping_mod = mapping_mod;
256 }
257 if (flags & SKBEDIT_F_MARK)
258 params_new->mark = *mark;
259 if (flags & SKBEDIT_F_PTYPE)
260 params_new->ptype = *ptype;
261 /* default behaviour is to use all the bits */
262 params_new->mask = 0xffffffff;
263 if (flags & SKBEDIT_F_MASK)
264 params_new->mask = *mask;
265
266 params_new->action = parm->action;
267 spin_lock_bh(&d->tcf_lock);
268 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
269 params_new = rcu_replace_pointer(d->params, params_new,
270 lockdep_is_held(&d->tcf_lock));
271 spin_unlock_bh(&d->tcf_lock);
272 if (params_new)
273 kfree_rcu(params_new, rcu);
274 if (goto_ch)
275 tcf_chain_put_by_act(goto_ch);
276
277 return ret;
278 put_chain:
279 if (goto_ch)
280 tcf_chain_put_by_act(goto_ch);
281 release_idr:
282 tcf_idr_release(*a, bind);
283 return err;
284 }
285
tcf_skbedit_dump(struct sk_buff * skb,struct tc_action * a,int bind,int ref)286 static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a,
287 int bind, int ref)
288 {
289 const struct tcf_skbedit *d = to_skbedit(a);
290 unsigned char *b = skb_tail_pointer(skb);
291 const struct tcf_skbedit_params *params;
292 struct tc_skbedit opt = {
293 .index = d->tcf_index,
294 .refcnt = refcount_read(&d->tcf_refcnt) - ref,
295 .bindcnt = atomic_read(&d->tcf_bindcnt) - bind,
296 };
297 u64 pure_flags = 0;
298 struct tcf_t t;
299
300 rcu_read_lock();
301 params = rcu_dereference(d->params);
302 opt.action = params->action;
303
304 if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt))
305 goto nla_put_failure;
306 if ((params->flags & SKBEDIT_F_PRIORITY) &&
307 nla_put_u32(skb, TCA_SKBEDIT_PRIORITY, params->priority))
308 goto nla_put_failure;
309 if ((params->flags & SKBEDIT_F_QUEUE_MAPPING) &&
310 nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING, params->queue_mapping))
311 goto nla_put_failure;
312 if ((params->flags & SKBEDIT_F_MARK) &&
313 nla_put_u32(skb, TCA_SKBEDIT_MARK, params->mark))
314 goto nla_put_failure;
315 if ((params->flags & SKBEDIT_F_PTYPE) &&
316 nla_put_u16(skb, TCA_SKBEDIT_PTYPE, params->ptype))
317 goto nla_put_failure;
318 if ((params->flags & SKBEDIT_F_MASK) &&
319 nla_put_u32(skb, TCA_SKBEDIT_MASK, params->mask))
320 goto nla_put_failure;
321 if (params->flags & SKBEDIT_F_INHERITDSFIELD)
322 pure_flags |= SKBEDIT_F_INHERITDSFIELD;
323 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
324 if (nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING_MAX,
325 params->queue_mapping + params->mapping_mod - 1))
326 goto nla_put_failure;
327
328 pure_flags |= SKBEDIT_F_TXQ_SKBHASH;
329 }
330 if (pure_flags != 0 &&
331 nla_put(skb, TCA_SKBEDIT_FLAGS, sizeof(pure_flags), &pure_flags))
332 goto nla_put_failure;
333
334 tcf_tm_dump(&t, &d->tcf_tm);
335 if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD))
336 goto nla_put_failure;
337 rcu_read_unlock();
338
339 return skb->len;
340
341 nla_put_failure:
342 rcu_read_unlock();
343 nlmsg_trim(skb, b);
344 return -1;
345 }
346
tcf_skbedit_cleanup(struct tc_action * a)347 static void tcf_skbedit_cleanup(struct tc_action *a)
348 {
349 struct tcf_skbedit *d = to_skbedit(a);
350 struct tcf_skbedit_params *params;
351
352 params = rcu_dereference_protected(d->params, 1);
353 if (params)
354 kfree_rcu(params, rcu);
355 }
356
tcf_skbedit_get_fill_size(const struct tc_action * act)357 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act)
358 {
359 return nla_total_size(sizeof(struct tc_skbedit))
360 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_PRIORITY */
361 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING */
362 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */
363 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MARK */
364 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_PTYPE */
365 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MASK */
366 + nla_total_size_64bit(sizeof(u64)); /* TCA_SKBEDIT_FLAGS */
367 }
368
tcf_skbedit_offload_act_setup(struct tc_action * act,void * entry_data,u32 * index_inc,bool bind,struct netlink_ext_ack * extack)369 static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data,
370 u32 *index_inc, bool bind,
371 struct netlink_ext_ack *extack)
372 {
373 if (bind) {
374 struct flow_action_entry *entry = entry_data;
375
376 if (is_tcf_skbedit_mark(act)) {
377 entry->id = FLOW_ACTION_MARK;
378 entry->mark = tcf_skbedit_mark(act);
379 } else if (is_tcf_skbedit_ptype(act)) {
380 entry->id = FLOW_ACTION_PTYPE;
381 entry->ptype = tcf_skbedit_ptype(act);
382 } else if (is_tcf_skbedit_priority(act)) {
383 entry->id = FLOW_ACTION_PRIORITY;
384 entry->priority = tcf_skbedit_priority(act);
385 } else if (is_tcf_skbedit_tx_queue_mapping(act)) {
386 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used on transmit side");
387 return -EOPNOTSUPP;
388 } else if (is_tcf_skbedit_rx_queue_mapping(act)) {
389 entry->id = FLOW_ACTION_RX_QUEUE_MAPPING;
390 entry->rx_queue = tcf_skbedit_rx_queue_mapping(act);
391 } else if (is_tcf_skbedit_inheritdsfield(act)) {
392 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used");
393 return -EOPNOTSUPP;
394 } else {
395 NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload");
396 return -EOPNOTSUPP;
397 }
398 *index_inc = 1;
399 } else {
400 struct flow_offload_action *fl_action = entry_data;
401
402 if (is_tcf_skbedit_mark(act))
403 fl_action->id = FLOW_ACTION_MARK;
404 else if (is_tcf_skbedit_ptype(act))
405 fl_action->id = FLOW_ACTION_PTYPE;
406 else if (is_tcf_skbedit_priority(act))
407 fl_action->id = FLOW_ACTION_PRIORITY;
408 else if (is_tcf_skbedit_rx_queue_mapping(act))
409 fl_action->id = FLOW_ACTION_RX_QUEUE_MAPPING;
410 else
411 return -EOPNOTSUPP;
412 }
413
414 return 0;
415 }
416
417 static struct tc_action_ops act_skbedit_ops = {
418 .kind = "skbedit",
419 .id = TCA_ID_SKBEDIT,
420 .owner = THIS_MODULE,
421 .act = tcf_skbedit_act,
422 .stats_update = tcf_skbedit_stats_update,
423 .dump = tcf_skbedit_dump,
424 .init = tcf_skbedit_init,
425 .cleanup = tcf_skbedit_cleanup,
426 .get_fill_size = tcf_skbedit_get_fill_size,
427 .offload_act_setup = tcf_skbedit_offload_act_setup,
428 .size = sizeof(struct tcf_skbedit),
429 };
430 MODULE_ALIAS_NET_ACT("skbedit");
431
skbedit_init_net(struct net * net)432 static __net_init int skbedit_init_net(struct net *net)
433 {
434 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
435
436 return tc_action_net_init(net, tn, &act_skbedit_ops);
437 }
438
skbedit_exit_net(struct list_head * net_list)439 static void __net_exit skbedit_exit_net(struct list_head *net_list)
440 {
441 tc_action_net_exit(net_list, act_skbedit_ops.net_id);
442 }
443
444 static struct pernet_operations skbedit_net_ops = {
445 .init = skbedit_init_net,
446 .exit_batch = skbedit_exit_net,
447 .id = &act_skbedit_ops.net_id,
448 .size = sizeof(struct tc_action_net),
449 };
450
451 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>");
452 MODULE_DESCRIPTION("SKB Editing");
453 MODULE_LICENSE("GPL");
454
skbedit_init_module(void)455 static int __init skbedit_init_module(void)
456 {
457 return tcf_register_action(&act_skbedit_ops, &skbedit_net_ops);
458 }
459
skbedit_cleanup_module(void)460 static void __exit skbedit_cleanup_module(void)
461 {
462 tcf_unregister_action(&act_skbedit_ops, &skbedit_net_ops);
463 }
464
465 module_init(skbedit_init_module);
466 module_exit(skbedit_cleanup_module);
467