1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2008, Intel Corporation.
4 *
5 * Author: Alexander Duyck <alexander.h.duyck@intel.com>
6 */
7
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/skbuff.h>
12 #include <linux/rtnetlink.h>
13 #include <net/netlink.h>
14 #include <net/pkt_sched.h>
15 #include <net/ip.h>
16 #include <net/ipv6.h>
17 #include <net/dsfield.h>
18 #include <net/pkt_cls.h>
19 #include <net/tc_wrapper.h>
20
21 #include <linux/tc_act/tc_skbedit.h>
22 #include <net/tc_act/tc_skbedit.h>
23
24 static struct tc_action_ops act_skbedit_ops;
25
tcf_skbedit_hash(struct tcf_skbedit_params * params,struct sk_buff * skb)26 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params,
27 struct sk_buff *skb)
28 {
29 u16 queue_mapping = params->queue_mapping;
30
31 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
32 u32 hash = skb_get_hash(skb);
33
34 queue_mapping += hash % params->mapping_mod;
35 }
36
37 return netdev_cap_txqueue(skb->dev, queue_mapping);
38 }
39
tcf_skbedit_act(struct sk_buff * skb,const struct tc_action * a,struct tcf_result * res)40 TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb,
41 const struct tc_action *a,
42 struct tcf_result *res)
43 {
44 struct tcf_skbedit *d = to_skbedit(a);
45 struct tcf_skbedit_params *params;
46
47 tcf_lastuse_update(&d->tcf_tm);
48 bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb);
49
50 params = rcu_dereference_bh(d->params);
51
52 if (params->flags & SKBEDIT_F_PRIORITY)
53 skb->priority = params->priority;
54 if (params->flags & SKBEDIT_F_INHERITDSFIELD) {
55 int wlen = skb_network_offset(skb);
56
57 switch (skb_protocol(skb, true)) {
58 case htons(ETH_P_IP):
59 wlen += sizeof(struct iphdr);
60 if (!pskb_may_pull(skb, wlen))
61 goto err;
62 skb->priority = ipv4_get_dsfield(ip_hdr(skb)) >> 2;
63 break;
64
65 case htons(ETH_P_IPV6):
66 wlen += sizeof(struct ipv6hdr);
67 if (!pskb_may_pull(skb, wlen))
68 goto err;
69 skb->priority = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;
70 break;
71 }
72 }
73 if (params->flags & SKBEDIT_F_QUEUE_MAPPING &&
74 skb->dev->real_num_tx_queues > params->queue_mapping) {
75 #ifdef CONFIG_NET_EGRESS
76 netdev_xmit_skip_txqueue(true);
77 #endif
78 skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb));
79 }
80 if (params->flags & SKBEDIT_F_MARK) {
81 skb->mark &= ~params->mask;
82 skb->mark |= params->mark & params->mask;
83 }
84 if (params->flags & SKBEDIT_F_PTYPE)
85 skb->pkt_type = params->ptype;
86 return params->action;
87
88 err:
89 qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats));
90 return TC_ACT_SHOT;
91 }
92
tcf_skbedit_stats_update(struct tc_action * a,u64 bytes,u64 packets,u64 drops,u64 lastuse,bool hw)93 static void tcf_skbedit_stats_update(struct tc_action *a, u64 bytes,
94 u64 packets, u64 drops,
95 u64 lastuse, bool hw)
96 {
97 struct tcf_skbedit *d = to_skbedit(a);
98 struct tcf_t *tm = &d->tcf_tm;
99
100 tcf_action_update_stats(a, bytes, packets, drops, hw);
101 tm->lastuse = max_t(u64, tm->lastuse, lastuse);
102 }
103
104 static const struct nla_policy skbedit_policy[TCA_SKBEDIT_MAX + 1] = {
105 [TCA_SKBEDIT_PARMS] = { .len = sizeof(struct tc_skbedit) },
106 [TCA_SKBEDIT_PRIORITY] = { .len = sizeof(u32) },
107 [TCA_SKBEDIT_QUEUE_MAPPING] = { .len = sizeof(u16) },
108 [TCA_SKBEDIT_MARK] = { .len = sizeof(u32) },
109 [TCA_SKBEDIT_PTYPE] = { .len = sizeof(u16) },
110 [TCA_SKBEDIT_MASK] = { .len = sizeof(u32) },
111 [TCA_SKBEDIT_FLAGS] = { .len = sizeof(u64) },
112 [TCA_SKBEDIT_QUEUE_MAPPING_MAX] = { .len = sizeof(u16) },
113 };
114
tcf_skbedit_init(struct net * net,struct nlattr * nla,struct nlattr * est,struct tc_action ** a,struct tcf_proto * tp,u32 act_flags,struct netlink_ext_ack * extack)115 static int tcf_skbedit_init(struct net *net, struct nlattr *nla,
116 struct nlattr *est, struct tc_action **a,
117 struct tcf_proto *tp, u32 act_flags,
118 struct netlink_ext_ack *extack)
119 {
120 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
121 bool bind = act_flags & TCA_ACT_FLAGS_BIND;
122 struct tcf_skbedit_params *params_new;
123 struct nlattr *tb[TCA_SKBEDIT_MAX + 1];
124 struct tcf_chain *goto_ch = NULL;
125 struct tc_skbedit *parm;
126 struct tcf_skbedit *d;
127 u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL;
128 u16 *queue_mapping = NULL, *ptype = NULL;
129 u32 mapping_mod = 1;
130 bool exists = false;
131 int ret = 0, err;
132 u32 index;
133
134 if (nla == NULL)
135 return -EINVAL;
136
137 err = nla_parse_nested_deprecated(tb, TCA_SKBEDIT_MAX, nla,
138 skbedit_policy, NULL);
139 if (err < 0)
140 return err;
141
142 if (tb[TCA_SKBEDIT_PARMS] == NULL)
143 return -EINVAL;
144
145 if (tb[TCA_SKBEDIT_PRIORITY] != NULL) {
146 flags |= SKBEDIT_F_PRIORITY;
147 priority = nla_data(tb[TCA_SKBEDIT_PRIORITY]);
148 }
149
150 if (tb[TCA_SKBEDIT_QUEUE_MAPPING] != NULL) {
151 if (is_tcf_skbedit_ingress(act_flags) &&
152 !(act_flags & TCA_ACT_FLAGS_SKIP_SW)) {
153 NL_SET_ERR_MSG_MOD(extack, "\"queue_mapping\" option on receive side is hardware only, use skip_sw");
154 return -EOPNOTSUPP;
155 }
156 flags |= SKBEDIT_F_QUEUE_MAPPING;
157 queue_mapping = nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING]);
158 }
159
160 if (tb[TCA_SKBEDIT_PTYPE] != NULL) {
161 ptype = nla_data(tb[TCA_SKBEDIT_PTYPE]);
162 if (!skb_pkt_type_ok(*ptype))
163 return -EINVAL;
164 flags |= SKBEDIT_F_PTYPE;
165 }
166
167 if (tb[TCA_SKBEDIT_MARK] != NULL) {
168 flags |= SKBEDIT_F_MARK;
169 mark = nla_data(tb[TCA_SKBEDIT_MARK]);
170 }
171
172 if (tb[TCA_SKBEDIT_MASK] != NULL) {
173 flags |= SKBEDIT_F_MASK;
174 mask = nla_data(tb[TCA_SKBEDIT_MASK]);
175 }
176
177 if (tb[TCA_SKBEDIT_FLAGS] != NULL) {
178 u64 *pure_flags = nla_data(tb[TCA_SKBEDIT_FLAGS]);
179
180 if (*pure_flags & SKBEDIT_F_TXQ_SKBHASH) {
181 u16 *queue_mapping_max;
182
183 if (!tb[TCA_SKBEDIT_QUEUE_MAPPING] ||
184 !tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]) {
185 NL_SET_ERR_MSG_MOD(extack, "Missing required range of queue_mapping.");
186 return -EINVAL;
187 }
188
189 queue_mapping_max =
190 nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]);
191 if (*queue_mapping_max < *queue_mapping) {
192 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid, max < min.");
193 return -EINVAL;
194 }
195
196 mapping_mod = *queue_mapping_max - *queue_mapping + 1;
197 if (mapping_mod > U16_MAX) {
198 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid.");
199 return -EINVAL;
200 }
201 flags |= SKBEDIT_F_TXQ_SKBHASH;
202 }
203 if (*pure_flags & SKBEDIT_F_INHERITDSFIELD)
204 flags |= SKBEDIT_F_INHERITDSFIELD;
205 }
206
207 parm = nla_data(tb[TCA_SKBEDIT_PARMS]);
208 index = parm->index;
209 err = tcf_idr_check_alloc(tn, &index, a, bind);
210 if (err < 0)
211 return err;
212 exists = err;
213 if (exists && bind)
214 return ACT_P_BOUND;
215
216 if (!flags) {
217 if (exists)
218 tcf_idr_release(*a, bind);
219 else
220 tcf_idr_cleanup(tn, index);
221 return -EINVAL;
222 }
223
224 if (!exists) {
225 ret = tcf_idr_create(tn, index, est, a,
226 &act_skbedit_ops, bind, true, act_flags);
227 if (ret) {
228 tcf_idr_cleanup(tn, index);
229 return ret;
230 }
231
232 d = to_skbedit(*a);
233 ret = ACT_P_CREATED;
234 } else {
235 d = to_skbedit(*a);
236 if (!(act_flags & TCA_ACT_FLAGS_REPLACE)) {
237 tcf_idr_release(*a, bind);
238 return -EEXIST;
239 }
240 }
241 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
242 if (err < 0)
243 goto release_idr;
244
245 params_new = kzalloc_obj(*params_new);
246 if (unlikely(!params_new)) {
247 err = -ENOMEM;
248 goto put_chain;
249 }
250
251 params_new->flags = flags;
252 if (flags & SKBEDIT_F_PRIORITY)
253 params_new->priority = *priority;
254 if (flags & SKBEDIT_F_QUEUE_MAPPING) {
255 params_new->queue_mapping = *queue_mapping;
256 params_new->mapping_mod = mapping_mod;
257 }
258 if (flags & SKBEDIT_F_MARK)
259 params_new->mark = *mark;
260 if (flags & SKBEDIT_F_PTYPE)
261 params_new->ptype = *ptype;
262 /* default behaviour is to use all the bits */
263 params_new->mask = 0xffffffff;
264 if (flags & SKBEDIT_F_MASK)
265 params_new->mask = *mask;
266
267 params_new->action = parm->action;
268 spin_lock_bh(&d->tcf_lock);
269 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
270 params_new = rcu_replace_pointer(d->params, params_new,
271 lockdep_is_held(&d->tcf_lock));
272 spin_unlock_bh(&d->tcf_lock);
273 if (params_new)
274 kfree_rcu(params_new, rcu);
275 if (goto_ch)
276 tcf_chain_put_by_act(goto_ch);
277
278 return ret;
279 put_chain:
280 if (goto_ch)
281 tcf_chain_put_by_act(goto_ch);
282 release_idr:
283 tcf_idr_release(*a, bind);
284 return err;
285 }
286
tcf_skbedit_dump(struct sk_buff * skb,struct tc_action * a,int bind,int ref)287 static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a,
288 int bind, int ref)
289 {
290 const struct tcf_skbedit *d = to_skbedit(a);
291 unsigned char *b = skb_tail_pointer(skb);
292 const struct tcf_skbedit_params *params;
293 struct tc_skbedit opt = {
294 .index = d->tcf_index,
295 .refcnt = refcount_read(&d->tcf_refcnt) - ref,
296 .bindcnt = atomic_read(&d->tcf_bindcnt) - bind,
297 };
298 u64 pure_flags = 0;
299 struct tcf_t t;
300
301 rcu_read_lock();
302 params = rcu_dereference(d->params);
303 opt.action = params->action;
304
305 if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt))
306 goto nla_put_failure;
307 if ((params->flags & SKBEDIT_F_PRIORITY) &&
308 nla_put_u32(skb, TCA_SKBEDIT_PRIORITY, params->priority))
309 goto nla_put_failure;
310 if ((params->flags & SKBEDIT_F_QUEUE_MAPPING) &&
311 nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING, params->queue_mapping))
312 goto nla_put_failure;
313 if ((params->flags & SKBEDIT_F_MARK) &&
314 nla_put_u32(skb, TCA_SKBEDIT_MARK, params->mark))
315 goto nla_put_failure;
316 if ((params->flags & SKBEDIT_F_PTYPE) &&
317 nla_put_u16(skb, TCA_SKBEDIT_PTYPE, params->ptype))
318 goto nla_put_failure;
319 if ((params->flags & SKBEDIT_F_MASK) &&
320 nla_put_u32(skb, TCA_SKBEDIT_MASK, params->mask))
321 goto nla_put_failure;
322 if (params->flags & SKBEDIT_F_INHERITDSFIELD)
323 pure_flags |= SKBEDIT_F_INHERITDSFIELD;
324 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) {
325 if (nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING_MAX,
326 params->queue_mapping + params->mapping_mod - 1))
327 goto nla_put_failure;
328
329 pure_flags |= SKBEDIT_F_TXQ_SKBHASH;
330 }
331 if (pure_flags != 0 &&
332 nla_put(skb, TCA_SKBEDIT_FLAGS, sizeof(pure_flags), &pure_flags))
333 goto nla_put_failure;
334
335 tcf_tm_dump(&t, &d->tcf_tm);
336 if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD))
337 goto nla_put_failure;
338 rcu_read_unlock();
339
340 return skb->len;
341
342 nla_put_failure:
343 rcu_read_unlock();
344 nlmsg_trim(skb, b);
345 return -1;
346 }
347
tcf_skbedit_cleanup(struct tc_action * a)348 static void tcf_skbedit_cleanup(struct tc_action *a)
349 {
350 struct tcf_skbedit *d = to_skbedit(a);
351 struct tcf_skbedit_params *params;
352
353 params = rcu_dereference_protected(d->params, 1);
354 if (params)
355 kfree_rcu(params, rcu);
356 }
357
tcf_skbedit_get_fill_size(const struct tc_action * act)358 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act)
359 {
360 return nla_total_size(sizeof(struct tc_skbedit))
361 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_PRIORITY */
362 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING */
363 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */
364 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MARK */
365 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_PTYPE */
366 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MASK */
367 + nla_total_size_64bit(sizeof(u64)); /* TCA_SKBEDIT_FLAGS */
368 }
369
tcf_skbedit_offload_act_setup(struct tc_action * act,void * entry_data,u32 * index_inc,bool bind,struct netlink_ext_ack * extack)370 static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data,
371 u32 *index_inc, bool bind,
372 struct netlink_ext_ack *extack)
373 {
374 if (bind) {
375 struct flow_action_entry *entry = entry_data;
376
377 if (is_tcf_skbedit_mark(act)) {
378 entry->id = FLOW_ACTION_MARK;
379 entry->mark = tcf_skbedit_mark(act);
380 } else if (is_tcf_skbedit_ptype(act)) {
381 entry->id = FLOW_ACTION_PTYPE;
382 entry->ptype = tcf_skbedit_ptype(act);
383 } else if (is_tcf_skbedit_priority(act)) {
384 entry->id = FLOW_ACTION_PRIORITY;
385 entry->priority = tcf_skbedit_priority(act);
386 } else if (is_tcf_skbedit_tx_queue_mapping(act)) {
387 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used on transmit side");
388 return -EOPNOTSUPP;
389 } else if (is_tcf_skbedit_rx_queue_mapping(act)) {
390 entry->id = FLOW_ACTION_RX_QUEUE_MAPPING;
391 entry->rx_queue = tcf_skbedit_rx_queue_mapping(act);
392 } else if (is_tcf_skbedit_inheritdsfield(act)) {
393 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used");
394 return -EOPNOTSUPP;
395 } else {
396 NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload");
397 return -EOPNOTSUPP;
398 }
399 *index_inc = 1;
400 } else {
401 struct flow_offload_action *fl_action = entry_data;
402
403 if (is_tcf_skbedit_mark(act))
404 fl_action->id = FLOW_ACTION_MARK;
405 else if (is_tcf_skbedit_ptype(act))
406 fl_action->id = FLOW_ACTION_PTYPE;
407 else if (is_tcf_skbedit_priority(act))
408 fl_action->id = FLOW_ACTION_PRIORITY;
409 else if (is_tcf_skbedit_rx_queue_mapping(act))
410 fl_action->id = FLOW_ACTION_RX_QUEUE_MAPPING;
411 else
412 return -EOPNOTSUPP;
413 }
414
415 return 0;
416 }
417
418 static struct tc_action_ops act_skbedit_ops = {
419 .kind = "skbedit",
420 .id = TCA_ID_SKBEDIT,
421 .owner = THIS_MODULE,
422 .act = tcf_skbedit_act,
423 .stats_update = tcf_skbedit_stats_update,
424 .dump = tcf_skbedit_dump,
425 .init = tcf_skbedit_init,
426 .cleanup = tcf_skbedit_cleanup,
427 .get_fill_size = tcf_skbedit_get_fill_size,
428 .offload_act_setup = tcf_skbedit_offload_act_setup,
429 .size = sizeof(struct tcf_skbedit),
430 };
431 MODULE_ALIAS_NET_ACT("skbedit");
432
skbedit_init_net(struct net * net)433 static __net_init int skbedit_init_net(struct net *net)
434 {
435 struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
436
437 return tc_action_net_init(net, tn, &act_skbedit_ops);
438 }
439
skbedit_exit_net(struct list_head * net_list)440 static void __net_exit skbedit_exit_net(struct list_head *net_list)
441 {
442 tc_action_net_exit(net_list, act_skbedit_ops.net_id);
443 }
444
445 static struct pernet_operations skbedit_net_ops = {
446 .init = skbedit_init_net,
447 .exit_batch = skbedit_exit_net,
448 .id = &act_skbedit_ops.net_id,
449 .size = sizeof(struct tc_action_net),
450 };
451
452 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>");
453 MODULE_DESCRIPTION("SKB Editing");
454 MODULE_LICENSE("GPL");
455
skbedit_init_module(void)456 static int __init skbedit_init_module(void)
457 {
458 return tcf_register_action(&act_skbedit_ops, &skbedit_net_ops);
459 }
460
skbedit_cleanup_module(void)461 static void __exit skbedit_cleanup_module(void)
462 {
463 tcf_unregister_action(&act_skbedit_ops, &skbedit_net_ops);
464 }
465
466 module_init(skbedit_init_module);
467 module_exit(skbedit_cleanup_module);
468