1 /*-
2 * Copyright (c) 2023 NVIDIA corporation & affiliates.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
26
27 #include "opt_ipsec.h"
28
29 #include <sys/types.h>
30 #include <netinet/in.h>
31 #include <sys/socket.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <net/if.h>
35 #include <net/if_var.h>
36 #include <net/pfkeyv2.h>
37 #include <netipsec/key.h>
38 #include <netipsec/key_var.h>
39 #include <netipsec/keydb.h>
40 #include <netipsec/ipsec.h>
41 #include <netipsec/xform.h>
42 #include <netipsec/ipsec_offload.h>
43 #include <dev/mlx5/fs.h>
44 #include <dev/mlx5/mlx5_en/en.h>
45 #include <dev/mlx5/mlx5_accel/ipsec.h>
46
47 #define MLX5_IPSEC_RESCHED msecs_to_jiffies(1000)
48
49 static void mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi,
50 void *priv);
51 static int mlx5e_if_sa_deinstall(struct ifnet *ifp, u_int dev_spi, void *priv);
52
to_ipsec_sa_entry(void * x)53 static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(void *x)
54 {
55 return (struct mlx5e_ipsec_sa_entry *)x;
56 }
57
to_ipsec_pol_entry(void * x)58 static struct mlx5e_ipsec_pol_entry *to_ipsec_pol_entry(void *x)
59 {
60 return (struct mlx5e_ipsec_pol_entry *)x;
61 }
62
63 static void
mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry * sa_entry,u64 * packets,u64 * bytes)64 mlx5e_ipsec_handle_counters_onedir(struct mlx5e_ipsec_sa_entry *sa_entry,
65 u64 *packets, u64 *bytes)
66 {
67 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
68 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
69
70 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes);
71 }
72
73 static struct mlx5e_ipsec_sa_entry *
mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir * pb,struct mlx5e_ipsec_sa_entry * sa_entry)74 mlx5e_ipsec_other_sa_entry(struct mlx5e_ipsec_priv_bothdir *pb,
75 struct mlx5e_ipsec_sa_entry *sa_entry)
76 {
77 return (pb->priv_in == sa_entry ? pb->priv_out : pb->priv_in);
78 }
79
80 static void
mlx5e_ipsec_handle_counters(struct work_struct * _work)81 mlx5e_ipsec_handle_counters(struct work_struct *_work)
82 {
83 struct mlx5e_ipsec_dwork *dwork =
84 container_of(_work, struct mlx5e_ipsec_dwork, dwork.work);
85 struct mlx5e_ipsec_sa_entry *sa_entry = dwork->sa_entry;
86 struct mlx5e_ipsec_sa_entry *other_sa_entry;
87 u64 bytes, bytes1, packets1, packets;
88
89 if (sa_entry->attrs.drop)
90 return;
91 other_sa_entry = mlx5e_ipsec_other_sa_entry(dwork->pb, sa_entry);
92 if (other_sa_entry == NULL || other_sa_entry->attrs.drop)
93 return;
94
95 mlx5e_ipsec_handle_counters_onedir(sa_entry, &packets, &bytes);
96 mlx5e_ipsec_handle_counters_onedir(other_sa_entry, &packets1, &bytes1);
97 packets += packets1;
98 bytes += bytes1;
99
100 #ifdef IPSEC_OFFLOAD
101 ipsec_accel_drv_sa_lifetime_update(
102 sa_entry->savp, sa_entry->ifpo, sa_entry->kspi, bytes, packets);
103 #endif
104
105 queue_delayed_work(sa_entry->ipsec->wq, &dwork->dwork,
106 MLX5_IPSEC_RESCHED);
107 }
108
109 static int
mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry * sa_entry,struct mlx5e_ipsec_priv_bothdir * pb)110 mlx5e_ipsec_create_dwork(struct mlx5e_ipsec_sa_entry *sa_entry,
111 struct mlx5e_ipsec_priv_bothdir *pb)
112 {
113 struct mlx5e_ipsec_dwork *dwork;
114
115 dwork = kzalloc(sizeof(*dwork), GFP_KERNEL);
116 if (!dwork)
117 return (ENOMEM);
118
119 dwork->sa_entry = sa_entry;
120 dwork->pb = pb;
121 INIT_DELAYED_WORK(&dwork->dwork, mlx5e_ipsec_handle_counters);
122 sa_entry->dwork = dwork;
123 return 0;
124 }
125
mlx5_xform_ah_authsize(const struct auth_hash * esph)126 static int mlx5_xform_ah_authsize(const struct auth_hash *esph)
127 {
128 int alen;
129
130 if (esph == NULL)
131 return 0;
132
133 switch (esph->type) {
134 case CRYPTO_SHA2_256_HMAC:
135 case CRYPTO_SHA2_384_HMAC:
136 case CRYPTO_SHA2_512_HMAC:
137 alen = esph->hashsize / 2; /* RFC4868 2.3 */
138 break;
139
140 case CRYPTO_POLY1305:
141 case CRYPTO_AES_NIST_GMAC:
142 alen = esph->hashsize;
143 break;
144
145 default:
146 alen = AH_HMAC_HASHLEN;
147 break;
148 }
149
150 return alen;
151 }
152
mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry * sa_entry,struct mlx5_accel_esp_xfrm_attrs * attrs,u8 dir)153 void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
154 struct mlx5_accel_esp_xfrm_attrs *attrs,
155 u8 dir)
156 {
157 struct secasvar *savp = sa_entry->savp;
158 const struct auth_hash *esph = savp->tdb_authalgxform;
159 struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
160 struct secasindex *saidx = &savp->sah->saidx;
161 struct seckey *key_encap = savp->key_enc;
162 int key_len;
163
164 memset(attrs, 0, sizeof(*attrs));
165
166 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
167 key_len = _KEYLEN(key_encap) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
168
169 memcpy(aes_gcm->aes_key, key_encap->key_data, key_len);
170 aes_gcm->key_len = key_len;
171
172 /* salt and seq_iv */
173 aes_gcm->seq_iv = 0;
174 memcpy(&aes_gcm->salt, key_encap->key_data + key_len,
175 sizeof(aes_gcm->salt));
176
177 switch (savp->alg_enc) {
178 case SADB_X_EALG_AESGCM8:
179 attrs->authsize = 8 / 4; /* in dwords */
180 break;
181 case SADB_X_EALG_AESGCM12:
182 attrs->authsize = 12 / 4; /* in dwords */
183 break;
184 case SADB_X_EALG_AESGCM16:
185 attrs->authsize = 16 / 4; /* in dwords */
186 break;
187 default: break;
188 }
189
190 /* iv len */
191 aes_gcm->icv_len = mlx5_xform_ah_authsize(esph); //TBD: check if value make sense
192
193 attrs->dir = dir;
194 /* spi - host order */
195 attrs->spi = ntohl(savp->spi);
196 attrs->family = saidx->dst.sa.sa_family;
197 attrs->reqid = saidx->reqid;
198
199 if (saidx->src.sa.sa_family == AF_INET) {
200 attrs->saddr.a4 = saidx->src.sin.sin_addr.s_addr;
201 attrs->daddr.a4 = saidx->dst.sin.sin_addr.s_addr;
202 } else {
203 memcpy(&attrs->saddr.a6, &saidx->src.sin6.sin6_addr, 16);
204 memcpy(&attrs->daddr.a6, &saidx->dst.sin6.sin6_addr, 16);
205 }
206
207 if (savp->natt) {
208 attrs->encap = true;
209 attrs->sport = savp->natt->sport;
210 attrs->dport = savp->natt->dport;
211 }
212
213 if (savp->flags & SADB_X_SAFLAGS_ESN) {
214 /* We support replay window with ESN only */
215 attrs->replay_esn.trigger = true;
216 if (sa_entry->esn_state.esn_msb)
217 attrs->replay_esn.esn = sa_entry->esn_state.esn;
218 else
219 /* According to RFC4303, section "3.3.3. Sequence Number Generation",
220 * the first packet sent using a given SA will contain a sequence
221 * number of 1.
222 */
223 attrs->replay_esn.esn = max_t(u32, sa_entry->esn_state.esn, 1);
224 attrs->replay_esn.esn_msb = sa_entry->esn_state.esn_msb;
225 attrs->replay_esn.overlap = sa_entry->esn_state.overlap;
226
227 if (savp->replay) {
228 switch (savp->replay->wsize) {
229 case 4:
230 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_32BIT;
231 break;
232 case 8:
233 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_64BIT;
234 break;
235 case 16:
236 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_128BIT;
237 break;
238 case 32:
239 attrs->replay_esn.replay_window = MLX5_IPSEC_ASO_REPLAY_WIN_256BIT;
240 break;
241 default:
242 /* Do nothing */
243 break;
244 }
245 }
246 }
247 }
248
mlx5e_xfrm_validate_state(struct mlx5_core_dev * mdev,struct secasvar * savp)249 static int mlx5e_xfrm_validate_state(struct mlx5_core_dev *mdev,
250 struct secasvar *savp)
251 {
252 struct secasindex *saidx = &savp->sah->saidx;
253 struct seckey *key_encp = savp->key_enc;
254 int keylen;
255
256 if (!(mlx5_ipsec_device_caps(mdev) &
257 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
258 mlx5_core_err(mdev, "FULL offload is not supported\n");
259 return (EINVAL);
260 }
261 if (savp->state == SADB_SASTATE_DEAD)
262 return (EINVAL);
263 if (savp->alg_enc == SADB_EALG_NONE) {
264 mlx5_core_err(mdev, "Cannot offload authenticated xfrm states\n");
265 return (EINVAL);
266 }
267 if (savp->alg_enc != SADB_X_EALG_AESGCM16) {
268 mlx5_core_err(mdev, "Only IPSec aes-gcm-16 encryption protocol may be offloaded\n");
269 return (EINVAL);
270 }
271 if (savp->tdb_compalgxform) {
272 mlx5_core_err(mdev, "Cannot offload compressed xfrm states\n");
273 return (EINVAL);
274 }
275 if (savp->alg_auth != SADB_X_AALG_AES128GMAC && savp->alg_auth != SADB_X_AALG_AES256GMAC) {
276 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bits\n");
277 return (EINVAL);
278 }
279 if ((saidx->dst.sa.sa_family != AF_INET && saidx->dst.sa.sa_family != AF_INET6) ||
280 (saidx->src.sa.sa_family != AF_INET && saidx->src.sa.sa_family != AF_INET6)) {
281 mlx5_core_err(mdev, "Only IPv4/6 xfrm states may be offloaded\n");
282 return (EINVAL);
283 }
284 if (saidx->proto != IPPROTO_ESP) {
285 mlx5_core_err(mdev, "Only ESP xfrm state may be offloaded\n");
286 return (EINVAL);
287 }
288 /* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
289 keylen = _KEYLEN(key_encp) - SAV_ISCTRORGCM(savp) * 4 - SAV_ISCHACHA(savp) * 4;
290 if (keylen != 128/8 && keylen != 256 / 8) {
291 mlx5_core_err(mdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
292 return (EINVAL);
293 }
294
295 if (saidx->mode != IPSEC_MODE_TRANSPORT) {
296 mlx5_core_err(mdev, "Only transport xfrm states may be offloaded in full offlaod mode\n");
297 return (EINVAL);
298 }
299
300 if (savp->natt) {
301 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESPINUDP)) {
302 mlx5_core_err(mdev, "Encapsulation is not supported\n");
303 return (EINVAL);
304 }
305 }
306
307 if (savp->replay && savp->replay->wsize != 0 && savp->replay->wsize != 4 &&
308 savp->replay->wsize != 8 && savp->replay->wsize != 16 && savp->replay->wsize != 32) {
309 mlx5_core_err(mdev, "Unsupported replay window size %d\n", savp->replay->wsize);
310 return (EINVAL);
311 }
312
313 if ((savp->flags & SADB_X_SAFLAGS_ESN) != 0) {
314 if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ESN) == 0) {
315 mlx5_core_err(mdev, "ESN is not supported\n");
316 return (EINVAL);
317 }
318 } else if (savp->replay != NULL && savp->replay->wsize != 0) {
319 mlx5_core_warn(mdev,
320 "non-ESN but replay-protect SA offload is not supported\n");
321 return (EINVAL);
322 }
323 return 0;
324 }
325
326 static int
mlx5e_if_sa_newkey_onedir(struct ifnet * ifp,void * sav,int dir,u_int drv_spi,struct mlx5e_ipsec_sa_entry ** privp,struct mlx5e_ipsec_priv_bothdir * pb,struct ifnet * ifpo)327 mlx5e_if_sa_newkey_onedir(struct ifnet *ifp, void *sav, int dir, u_int drv_spi,
328 struct mlx5e_ipsec_sa_entry **privp, struct mlx5e_ipsec_priv_bothdir *pb,
329 struct ifnet *ifpo)
330 {
331 #ifdef IPSEC_OFFLOAD
332 struct rm_priotracker tracker;
333 #endif
334 struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
335 struct mlx5e_priv *priv = if_getsoftc(ifp);
336 struct mlx5_core_dev *mdev = priv->mdev;
337 struct mlx5e_ipsec *ipsec = priv->ipsec;
338 u16 vid = VLAN_NONE;
339 int err;
340
341 if (priv->gone != 0 || ipsec == NULL)
342 return (EOPNOTSUPP);
343
344 if (if_gettype(ifpo) == IFT_L2VLAN)
345 VLAN_TAG(ifpo, &vid);
346
347 #ifdef IPSEC_OFFLOAD
348 ipsec_sahtree_rlock(&tracker);
349 #endif
350 err = mlx5e_xfrm_validate_state(mdev, sav);
351 #ifdef IPSEC_OFFLOAD
352 ipsec_sahtree_runlock(&tracker);
353 #endif
354 if (err)
355 return err;
356
357 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
358 if (sa_entry == NULL)
359 return (ENOMEM);
360
361 sa_entry->kspi = drv_spi;
362 sa_entry->savp = sav;
363 sa_entry->ifp = ifp;
364 sa_entry->ifpo = ifpo;
365 sa_entry->ipsec = ipsec;
366 sa_entry->vid = vid;
367
368 #ifdef IPSEC_OFFLOAD
369 ipsec_sahtree_rlock(&tracker);
370 #endif
371 err = mlx5e_xfrm_validate_state(mdev, sav);
372 if (err != 0) {
373 #ifdef IPSEC_OFFLOAD
374 ipsec_sahtree_runlock(&tracker);
375 #endif
376 goto err_xfrm;
377 }
378 mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs, dir);
379 #ifdef IPSEC_OFFLOAD
380 ipsec_sahtree_runlock(&tracker);
381 #endif
382
383 err = mlx5e_ipsec_create_dwork(sa_entry, pb);
384 if (err)
385 goto err_xfrm;
386
387 /* create hw context */
388 err = mlx5_ipsec_create_sa_ctx(sa_entry);
389 if (err)
390 goto err_sa_ctx;
391
392 err = mlx5e_accel_ipsec_fs_add_rule(sa_entry);
393 if (err)
394 goto err_fs;
395
396 *privp = sa_entry;
397 if (sa_entry->dwork)
398 queue_delayed_work(ipsec->wq, &sa_entry->dwork->dwork, MLX5_IPSEC_RESCHED);
399
400 err = xa_insert(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id, sa_entry, GFP_KERNEL);
401 if (err)
402 goto err_xa;
403
404 return 0;
405
406 err_xa:
407 if (sa_entry->dwork)
408 cancel_delayed_work_sync(&sa_entry->dwork->dwork);
409 mlx5e_accel_ipsec_fs_del_rule(sa_entry);
410 err_fs:
411 mlx5_ipsec_free_sa_ctx(sa_entry);
412 err_sa_ctx:
413 kfree(sa_entry->dwork);
414 sa_entry->dwork = NULL;
415 err_xfrm:
416 kfree(sa_entry);
417 mlx5_en_err(ifp, "Device failed to offload this state");
418 return err;
419 }
420
421 #define GET_TRUNK_IF(vifp, ifp, ept) \
422 if (if_gettype(vifp) == IFT_L2VLAN) { \
423 NET_EPOCH_ENTER(ept); \
424 ifp = VLAN_TRUNKDEV(vifp); \
425 NET_EPOCH_EXIT(ept); \
426 } else { \
427 ifp = vifp; \
428 }
429
430 static int
mlx5e_if_sa_newkey(struct ifnet * ifpo,void * sav,u_int dev_spi,void ** privp)431 mlx5e_if_sa_newkey(struct ifnet *ifpo, void *sav, u_int dev_spi, void **privp)
432 {
433 struct mlx5e_ipsec_priv_bothdir *pb;
434 struct epoch_tracker et;
435 struct ifnet *ifp;
436 int error;
437
438 GET_TRUNK_IF(ifpo, ifp, et);
439
440 pb = malloc(sizeof(struct mlx5e_ipsec_priv_bothdir), M_DEVBUF,
441 M_WAITOK | M_ZERO);
442 error = mlx5e_if_sa_newkey_onedir(
443 ifp, sav, IPSEC_DIR_INBOUND, dev_spi, &pb->priv_in, pb, ifpo);
444 if (error != 0) {
445 free(pb, M_DEVBUF);
446 return (error);
447 }
448 error = mlx5e_if_sa_newkey_onedir(
449 ifp, sav, IPSEC_DIR_OUTBOUND, dev_spi, &pb->priv_out, pb, ifpo);
450 if (error == 0) {
451 *privp = pb;
452 } else {
453 if (pb->priv_in->dwork != NULL)
454 cancel_delayed_work_sync(&pb->priv_in->dwork->dwork);
455 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb->priv_in);
456 free(pb, M_DEVBUF);
457 }
458 return (error);
459 }
460
461 static void
mlx5e_if_sa_deinstall_onekey(struct ifnet * ifp,u_int dev_spi,void * priv)462 mlx5e_if_sa_deinstall_onekey(struct ifnet *ifp, u_int dev_spi, void *priv)
463 {
464 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
465 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
466 struct mlx5e_ipsec_sa_entry *old;
467
468 old = xa_erase(&mdev->ipsec_sadb, sa_entry->ipsec_obj_id);
469 WARN_ON(old != sa_entry);
470
471 mlx5e_accel_ipsec_fs_del_rule(sa_entry);
472 mlx5_ipsec_free_sa_ctx(sa_entry);
473 kfree(sa_entry->dwork);
474 kfree(sa_entry);
475 }
476
477 static int
mlx5e_if_sa_deinstall(struct ifnet * ifpo,u_int dev_spi,void * priv)478 mlx5e_if_sa_deinstall(struct ifnet *ifpo, u_int dev_spi, void *priv)
479 {
480 struct mlx5e_ipsec_priv_bothdir pb, *pbp;
481 struct epoch_tracker et;
482 struct ifnet *ifp;
483
484 GET_TRUNK_IF(ifpo, ifp, et);
485
486 pbp = priv;
487 pb = *(struct mlx5e_ipsec_priv_bothdir *)priv;
488 pbp->priv_in = pbp->priv_out = NULL;
489
490 if (pb.priv_in->dwork != NULL)
491 cancel_delayed_work_sync(&pb.priv_in->dwork->dwork);
492 if (pb.priv_out->dwork != NULL)
493 cancel_delayed_work_sync(&pb.priv_out->dwork->dwork);
494
495 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_in);
496 mlx5e_if_sa_deinstall_onekey(ifp, dev_spi, pb.priv_out);
497 free(pbp, M_DEVBUF);
498 return (0);
499 }
500
501 static void
mlx5e_if_sa_cnt_one(struct ifnet * ifp,void * sa,uint32_t drv_spi,void * priv,u64 * bytes,u64 * packets)502 mlx5e_if_sa_cnt_one(struct ifnet *ifp, void *sa, uint32_t drv_spi,
503 void *priv, u64 *bytes, u64 *packets)
504 {
505 struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(priv);
506 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule;
507 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
508
509 mlx5_fc_query(mdev, ipsec_rule->fc, packets, bytes);
510 }
511
512 static int
mlx5e_if_sa_cnt(struct ifnet * ifpo,void * sa,uint32_t drv_spi,void * priv,struct seclifetime * lt)513 mlx5e_if_sa_cnt(struct ifnet *ifpo, void *sa, uint32_t drv_spi, void *priv,
514 struct seclifetime *lt)
515 {
516 struct mlx5e_ipsec_priv_bothdir *pb;
517 u64 packets_in, packets_out;
518 u64 bytes_in, bytes_out;
519 struct epoch_tracker et;
520 struct ifnet *ifp;
521
522 GET_TRUNK_IF(ifpo, ifp, et);
523
524 pb = priv;
525 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_in,
526 &bytes_in, &packets_in);
527 mlx5e_if_sa_cnt_one(ifp, sa, drv_spi, pb->priv_out,
528 &bytes_out, &packets_out);
529 /* TODO: remove this casting once Kostia changes allocation type to be u64 */
530 lt->bytes = bytes_in + bytes_out;
531 lt->allocations = (uint32_t)(packets_in + packets_out);
532 return (0);
533 }
534
mlx5e_xfrm_validate_policy(struct mlx5_core_dev * mdev,struct secpolicy * sp,struct inpcb * inp)535 static int mlx5e_xfrm_validate_policy(struct mlx5_core_dev *mdev,
536 struct secpolicy *sp, struct inpcb *inp)
537 {
538 struct secpolicyindex *spidx = &sp->spidx;
539
540 if (!(mlx5_ipsec_device_caps(mdev) &
541 MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
542 mlx5_core_err(mdev, "FULL offload is not supported\n");
543 return (EINVAL);
544 }
545
546 if (sp->tcount > 1) {
547 mlx5_core_err(mdev, "Can offload exactly one template, "
548 "not %d\n", sp->tcount);
549 return (EINVAL);
550 }
551
552 if (sp->policy == IPSEC_POLICY_BYPASS &&
553 !(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO)) {
554 mlx5_core_err(mdev, "Device does not support policy priority\n");
555 return (EINVAL);
556 }
557
558 if (sp->tcount > 0 && inp != NULL) {
559 mlx5_core_err(mdev, "Not valid input data\n");
560 return (EINVAL);
561 }
562
563 if (spidx->dir != IPSEC_DIR_INBOUND && spidx->dir != IPSEC_DIR_OUTBOUND) {
564 mlx5_core_err(mdev, "Wrong policy direction\n");
565 return (EINVAL);
566 }
567
568 if (sp->tcount > 0 && sp->req[0]->saidx.mode != IPSEC_MODE_TRANSPORT) {
569 mlx5_core_err(mdev, "Device supports transport mode only");
570 return (EINVAL);
571 }
572
573 if (sp->policy != IPSEC_POLICY_DISCARD &&
574 sp->policy != IPSEC_POLICY_IPSEC && sp->policy != IPSEC_POLICY_BYPASS) {
575 mlx5_core_err(mdev, "Offloaded policy must be specific on its action\n");
576 return (EINVAL);
577 }
578
579 if (sp->policy == IPSEC_POLICY_BYPASS && !inp) {
580 mlx5_core_err(mdev, "Missing port information for IKE bypass\n");
581 return (EINVAL);
582 }
583
584 if (inp != NULL) {
585 INP_RLOCK(inp);
586 if (inp->inp_socket == NULL || inp->inp_socket->so_proto->
587 pr_protocol != IPPROTO_UDP) {
588 mlx5_core_err(mdev, "Unsupported IKE bypass protocol %d\n",
589 inp->inp_socket == NULL ? -1 :
590 inp->inp_socket->so_proto->pr_protocol);
591 INP_RUNLOCK(inp);
592 return (EINVAL);
593 }
594 INP_RUNLOCK(inp);
595 }
596
597 /* TODO fill relevant bits */
598 return 0;
599 }
600
601 static void
mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry * pol_entry,struct mlx5_accel_pol_xfrm_attrs * attrs,struct inpcb * inp,u16 vid)602 mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
603 struct mlx5_accel_pol_xfrm_attrs *attrs, struct inpcb *inp, u16 vid)
604 {
605 struct secpolicy *sp = pol_entry->sp;
606 struct secpolicyindex *spidx = &sp->spidx;
607
608 memset(attrs, 0, sizeof(*attrs));
609
610 if (!inp) {
611 if (spidx->src.sa.sa_family == AF_INET) {
612 attrs->saddr.a4 = spidx->src.sin.sin_addr.s_addr;
613 attrs->daddr.a4 = spidx->dst.sin.sin_addr.s_addr;
614 } else if (spidx->src.sa.sa_family == AF_INET6) {
615 memcpy(&attrs->saddr.a6, &spidx->src.sin6.sin6_addr, 16);
616 memcpy(&attrs->daddr.a6, &spidx->dst.sin6.sin6_addr, 16);
617 } else {
618 KASSERT(0, ("unsupported family %d", spidx->src.sa.sa_family));
619 }
620 attrs->family = spidx->src.sa.sa_family;
621 attrs->prio = 0;
622 attrs->action = sp->policy;
623 attrs->reqid = sp->req[0]->saidx.reqid;
624 } else {
625 INP_RLOCK(inp);
626 if ((inp->inp_vflag & INP_IPV4) != 0) {
627 attrs->saddr.a4 = inp->inp_laddr.s_addr;
628 attrs->daddr.a4 = inp->inp_faddr.s_addr;
629 attrs->family = AF_INET;
630 } else if ((inp->inp_vflag & INP_IPV6) != 0) {
631 memcpy(&attrs->saddr.a6, &inp->in6p_laddr, 16);
632 memcpy(&attrs->daddr.a6, &inp->in6p_laddr, 16);
633 attrs->family = AF_INET6;
634 } else {
635 KASSERT(0, ("unsupported family %d", inp->inp_vflag));
636 }
637 attrs->upspec.dport = inp->inp_fport;
638 attrs->upspec.sport = inp->inp_lport;
639 attrs->upspec.proto = inp->inp_ip_p;
640 INP_RUNLOCK(inp);
641
642 /* Give highest priority for PCB policies */
643 attrs->prio = 1;
644 attrs->action = IPSEC_POLICY_IPSEC;
645 }
646 attrs->dir = spidx->dir;
647 attrs->vid = vid;
648 }
649
650 static int
mlx5e_if_spd_install(struct ifnet * ifpo,void * sp,void * inp1,void ** ifdatap)651 mlx5e_if_spd_install(struct ifnet *ifpo, void *sp, void *inp1, void **ifdatap)
652 {
653 struct mlx5e_ipsec_pol_entry *pol_entry;
654 struct mlx5e_priv *priv;
655 struct epoch_tracker et;
656 u16 vid = VLAN_NONE;
657 struct ifnet *ifp;
658 int err;
659
660 GET_TRUNK_IF(ifpo, ifp, et);
661 if (if_gettype(ifpo) == IFT_L2VLAN)
662 VLAN_TAG(ifpo, &vid);
663 priv = if_getsoftc(ifp);
664 if (priv->gone || !priv->ipsec)
665 return (EOPNOTSUPP);
666
667 err = mlx5e_xfrm_validate_policy(priv->mdev, sp, inp1);
668 if (err)
669 return err;
670
671 pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL);
672 if (!pol_entry)
673 return (ENOMEM);
674
675 pol_entry->sp = sp;
676 pol_entry->ipsec = priv->ipsec;
677
678 mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs,
679 inp1, vid);
680 err = mlx5e_accel_ipsec_fs_add_pol(pol_entry);
681 if (err)
682 goto err_pol;
683 *ifdatap = pol_entry;
684
685 return 0;
686
687 err_pol:
688 kfree(pol_entry);
689 mlx5_en_err(ifp, "Device failed to offload this policy");
690 return err;
691 }
692
693 static int
mlx5e_if_spd_deinstall(struct ifnet * ifpo,void * sp,void * ifdata)694 mlx5e_if_spd_deinstall(struct ifnet *ifpo, void *sp, void *ifdata)
695 {
696 struct mlx5e_ipsec_pol_entry *pol_entry;
697
698 pol_entry = to_ipsec_pol_entry(ifdata);
699 mlx5e_accel_ipsec_fs_del_pol(pol_entry);
700 kfree(pol_entry);
701 return 0;
702 }
703
mlx5e_ipsec_cleanup(struct mlx5e_priv * priv)704 void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
705 {
706 struct mlx5e_ipsec *pipsec = priv->ipsec;
707 if (!pipsec)
708 return;
709
710 mlx5e_accel_ipsec_fs_cleanup(pipsec);
711 destroy_workqueue(pipsec->wq);
712 mlx5e_ipsec_aso_cleanup(pipsec);
713 kfree(pipsec);
714 priv->ipsec = NULL;
715 }
716
717 static int
mlx5e_if_ipsec_hwassist(if_t ifneto,void * sav __unused,uint32_t drv_spi __unused,void * priv __unused)718 mlx5e_if_ipsec_hwassist(if_t ifneto, void *sav __unused,
719 uint32_t drv_spi __unused, void *priv __unused)
720 {
721 if_t ifnet;
722
723 if (if_gettype(ifneto) == IFT_L2VLAN) {
724 ifnet = VLAN_TRUNKDEV(ifneto);
725 } else {
726 ifnet = ifneto;
727 }
728
729 return (if_gethwassist(ifnet) & (CSUM_TSO | CSUM_TCP | CSUM_UDP |
730 CSUM_IP | CSUM_IP6_TSO | CSUM_IP6_TCP | CSUM_IP6_UDP));
731 }
732
733 static const struct if_ipsec_accel_methods mlx5e_ipsec_funcs = {
734 .if_sa_newkey = mlx5e_if_sa_newkey,
735 .if_sa_deinstall = mlx5e_if_sa_deinstall,
736 .if_spdadd = mlx5e_if_spd_install,
737 .if_spddel = mlx5e_if_spd_deinstall,
738 .if_sa_cnt = mlx5e_if_sa_cnt,
739 .if_hwassist = mlx5e_if_ipsec_hwassist,
740 };
741
mlx5e_ipsec_init(struct mlx5e_priv * priv)742 int mlx5e_ipsec_init(struct mlx5e_priv *priv)
743 {
744 struct mlx5_core_dev *mdev = priv->mdev;
745 struct mlx5e_ipsec *pipsec;
746 if_t ifp = priv->ifp;
747 int ret;
748
749 mlx5_core_info(mdev, "ipsec "
750 "offload %d log_max_dek %d gen_obj_types %d "
751 "ipsec_encrypt %d ipsec_decrypt %d "
752 "esp_aes_gcm_128_encrypt %d esp_aes_gcm_128_decrypt %d "
753 "ipsec_full_offload %d "
754 "reformat_add_esp_trasport %d reformat_del_esp_trasport %d "
755 "decap %d "
756 "ignore_flow_level_tx %d ignore_flow_level_rx %d "
757 "reformat_natt_tx %d reformat_natt_rx %d "
758 "ipsec_esn %d\n",
759 MLX5_CAP_GEN(mdev, ipsec_offload) != 0,
760 MLX5_CAP_GEN(mdev, log_max_dek) != 0,
761 (MLX5_CAP_GEN_64(mdev, general_obj_types) &
762 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC) != 0,
763 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) != 0,
764 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt) != 0,
765 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) != 0,
766 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt) != 0,
767 MLX5_CAP_IPSEC(mdev, ipsec_full_offload) != 0,
768 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) != 0,
769 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) != 0,
770 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap) != 0,
771 MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ignore_flow_level) != 0,
772 MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ignore_flow_level) != 0,
773 MLX5_CAP_FLOWTABLE_NIC_TX(mdev,
774 reformat_add_esp_transport_over_udp) != 0,
775 MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
776 reformat_del_esp_transport_over_udp) != 0,
777 MLX5_CAP_IPSEC(mdev, ipsec_esn) != 0);
778
779 if (!(mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
780 mlx5_core_dbg(mdev, "Not an IPSec offload device\n");
781 return 0;
782 }
783
784 xa_init_flags(&mdev->ipsec_sadb, XA_FLAGS_ALLOC);
785
786 pipsec = kzalloc(sizeof(*pipsec), GFP_KERNEL);
787 if (pipsec == NULL)
788 return (ENOMEM);
789
790 pipsec->mdev = mdev;
791 pipsec->pdn = priv->pdn;
792 pipsec->mkey = priv->mr.key;
793
794 ret = mlx5e_ipsec_aso_init(pipsec);
795 if (ret)
796 goto err_ipsec_aso;
797
798 pipsec->wq = alloc_workqueue("mlx5e_ipsec", WQ_UNBOUND, 0);
799 if (pipsec->wq == NULL) {
800 ret = ENOMEM;
801 goto err_ipsec_wq;
802 }
803
804 ret = mlx5e_accel_ipsec_fs_init(pipsec);
805 if (ret)
806 goto err_ipsec_alloc;
807
808 if_setipsec_accel_methods(ifp, &mlx5e_ipsec_funcs);
809 priv->ipsec = pipsec;
810 mlx5_core_dbg(mdev, "IPSec attached to netdevice\n");
811 return 0;
812
813 err_ipsec_alloc:
814 destroy_workqueue(pipsec->wq);
815 err_ipsec_wq:
816 mlx5e_ipsec_aso_cleanup(pipsec);
817 err_ipsec_aso:
818 kfree(pipsec);
819 mlx5_core_err(priv->mdev, "IPSec initialization failed, %d\n", ret);
820 return ret;
821 }
822