1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/module.h> 6 #include <linux/kernel.h> 7 #include <linux/init.h> 8 #include <linux/netdevice.h> 9 #include <asm/unaligned.h> 10 #include <linux/ktime.h> 11 #include <net/xfrm.h> 12 13 #include "../nfpcore/nfp_dev.h" 14 #include "../nfp_net_ctrl.h" 15 #include "../nfp_net.h" 16 #include "crypto.h" 17 18 #define NFP_NET_IPSEC_MAX_SA_CNT (16 * 1024) /* Firmware support a maximum of 16K SA offload */ 19 20 /* IPsec config message cmd codes */ 21 enum nfp_ipsec_cfg_mssg_cmd_codes { 22 NFP_IPSEC_CFG_MSSG_ADD_SA, /* Add a new SA */ 23 NFP_IPSEC_CFG_MSSG_INV_SA /* Invalidate an existing SA */ 24 }; 25 26 /* IPsec config message response codes */ 27 enum nfp_ipsec_cfg_mssg_rsp_codes { 28 NFP_IPSEC_CFG_MSSG_OK, 29 NFP_IPSEC_CFG_MSSG_FAILED, 30 NFP_IPSEC_CFG_MSSG_SA_VALID, 31 NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED, 32 NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED, 33 NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD 34 }; 35 36 /* Protocol */ 37 enum nfp_ipsec_sa_prot { 38 NFP_IPSEC_PROTOCOL_AH = 0, 39 NFP_IPSEC_PROTOCOL_ESP = 1 40 }; 41 42 /* Mode */ 43 enum nfp_ipsec_sa_mode { 44 NFP_IPSEC_PROTMODE_TRANSPORT = 0, 45 NFP_IPSEC_PROTMODE_TUNNEL = 1 46 }; 47 48 /* Cipher types */ 49 enum nfp_ipsec_sa_cipher { 50 NFP_IPSEC_CIPHER_NULL, 51 NFP_IPSEC_CIPHER_3DES, 52 NFP_IPSEC_CIPHER_AES128, 53 NFP_IPSEC_CIPHER_AES192, 54 NFP_IPSEC_CIPHER_AES256, 55 NFP_IPSEC_CIPHER_AES128_NULL, 56 NFP_IPSEC_CIPHER_AES192_NULL, 57 NFP_IPSEC_CIPHER_AES256_NULL, 58 NFP_IPSEC_CIPHER_CHACHA20 59 }; 60 61 /* Cipher modes */ 62 enum nfp_ipsec_sa_cipher_mode { 63 NFP_IPSEC_CIMODE_ECB, 64 NFP_IPSEC_CIMODE_CBC, 65 NFP_IPSEC_CIMODE_CFB, 66 NFP_IPSEC_CIMODE_OFB, 67 NFP_IPSEC_CIMODE_CTR 68 }; 69 70 /* Hash types */ 71 enum nfp_ipsec_sa_hash_type { 72 NFP_IPSEC_HASH_NONE, 73 NFP_IPSEC_HASH_MD5_96, 74 NFP_IPSEC_HASH_SHA1_96, 75 NFP_IPSEC_HASH_SHA256_96, 76 NFP_IPSEC_HASH_SHA384_96, 77 NFP_IPSEC_HASH_SHA512_96, 78 NFP_IPSEC_HASH_MD5_128, 79 NFP_IPSEC_HASH_SHA1_80, 80 NFP_IPSEC_HASH_SHA256_128, 81 NFP_IPSEC_HASH_SHA384_192, 82 NFP_IPSEC_HASH_SHA512_256, 83 NFP_IPSEC_HASH_GF128_128, 84 NFP_IPSEC_HASH_POLY1305_128 85 }; 86 87 /* IPSEC_CFG_MSSG_ADD_SA */ 88 struct nfp_ipsec_cfg_add_sa { 89 u32 ciph_key[8]; /* Cipher Key */ 90 union { 91 u32 auth_key[16]; /* Authentication Key */ 92 struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */ 93 u32 salt; /* Initialized with SA */ 94 u32 resv[15]; 95 } aesgcm_fields; 96 }; 97 struct sa_ctrl_word { 98 uint32_t hash :4; /* From nfp_ipsec_sa_hash_type */ 99 uint32_t cimode :4; /* From nfp_ipsec_sa_cipher_mode */ 100 uint32_t cipher :4; /* From nfp_ipsec_sa_cipher */ 101 uint32_t mode :2; /* From nfp_ipsec_sa_mode */ 102 uint32_t proto :2; /* From nfp_ipsec_sa_prot */ 103 uint32_t dir :1; /* SA direction */ 104 uint32_t resv0 :12; 105 uint32_t encap_dsbl:1; /* Encap/Decap disable */ 106 uint32_t resv1 :2; /* Must be set to 0 */ 107 } ctrl_word; 108 u32 spi; /* SPI Value */ 109 uint32_t pmtu_limit :16; /* PMTU Limit */ 110 uint32_t resv0 :5; 111 uint32_t ipv6 :1; /* Outbound IPv6 addr format */ 112 uint32_t resv1 :10; 113 u32 resv2[2]; 114 u32 src_ip[4]; /* Src IP addr */ 115 u32 dst_ip[4]; /* Dst IP addr */ 116 u32 resv3[6]; 117 }; 118 119 /* IPSEC_CFG_MSSG */ 120 struct nfp_ipsec_cfg_mssg { 121 union { 122 struct{ 123 uint32_t cmd:16; /* One of nfp_ipsec_cfg_mssg_cmd_codes */ 124 uint32_t rsp:16; /* One of nfp_ipsec_cfg_mssg_rsp_codes */ 125 uint32_t sa_idx:16; /* SA table index */ 126 uint32_t spare0:16; 127 struct nfp_ipsec_cfg_add_sa cfg_add_sa; 128 }; 129 u32 raw[64]; 130 }; 131 }; 132 133 static int nfp_net_ipsec_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry) 134 { 135 unsigned int offset = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL; 136 struct nfp_ipsec_cfg_mssg *msg = (struct nfp_ipsec_cfg_mssg *)entry->msg; 137 int i, msg_size, ret; 138 139 ret = nfp_net_mbox_lock(nn, sizeof(*msg)); 140 if (ret) 141 return ret; 142 143 msg_size = ARRAY_SIZE(msg->raw); 144 for (i = 0; i < msg_size; i++) 145 nn_writel(nn, offset + 4 * i, msg->raw[i]); 146 147 ret = nfp_net_mbox_reconfig(nn, entry->cmd); 148 if (ret < 0) { 149 nn_ctrl_bar_unlock(nn); 150 return ret; 151 } 152 153 /* For now we always read the whole message response back */ 154 for (i = 0; i < msg_size; i++) 155 msg->raw[i] = nn_readl(nn, offset + 4 * i); 156 157 nn_ctrl_bar_unlock(nn); 158 159 switch (msg->rsp) { 160 case NFP_IPSEC_CFG_MSSG_OK: 161 return 0; 162 case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD: 163 return -EINVAL; 164 case NFP_IPSEC_CFG_MSSG_SA_VALID: 165 return -EEXIST; 166 case NFP_IPSEC_CFG_MSSG_FAILED: 167 case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED: 168 case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED: 169 return -EIO; 170 default: 171 return -EINVAL; 172 } 173 } 174 175 static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen) 176 { 177 bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC); 178 179 switch (keylen) { 180 case 128: 181 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL : 182 NFP_IPSEC_CIPHER_AES128; 183 break; 184 case 192: 185 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL : 186 NFP_IPSEC_CIPHER_AES192; 187 break; 188 case 256: 189 cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL : 190 NFP_IPSEC_CIPHER_AES256; 191 break; 192 default: 193 return -EINVAL; 194 } 195 196 return 0; 197 } 198 199 static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 200 { 201 switch (*trunc_len) { 202 case 96: 203 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96; 204 break; 205 case 128: 206 cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128; 207 break; 208 default: 209 *trunc_len = 0; 210 } 211 } 212 213 static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 214 { 215 switch (*trunc_len) { 216 case 96: 217 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96; 218 break; 219 case 80: 220 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80; 221 break; 222 default: 223 *trunc_len = 0; 224 } 225 } 226 227 static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 228 { 229 switch (*trunc_len) { 230 case 96: 231 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96; 232 break; 233 case 128: 234 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128; 235 break; 236 default: 237 *trunc_len = 0; 238 } 239 } 240 241 static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 242 { 243 switch (*trunc_len) { 244 case 96: 245 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96; 246 break; 247 case 192: 248 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192; 249 break; 250 default: 251 *trunc_len = 0; 252 } 253 } 254 255 static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len) 256 { 257 switch (*trunc_len) { 258 case 96: 259 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96; 260 break; 261 case 256: 262 cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256; 263 break; 264 default: 265 *trunc_len = 0; 266 } 267 } 268 269 static int nfp_net_xfrm_add_state(struct xfrm_state *x, 270 struct netlink_ext_ack *extack) 271 { 272 struct net_device *netdev = x->xso.real_dev; 273 struct nfp_ipsec_cfg_mssg msg = {}; 274 int i, key_len, trunc_len, err = 0; 275 struct nfp_ipsec_cfg_add_sa *cfg; 276 struct nfp_net *nn; 277 unsigned int saidx; 278 279 nn = netdev_priv(netdev); 280 cfg = &msg.cfg_add_sa; 281 282 /* General */ 283 switch (x->props.mode) { 284 case XFRM_MODE_TUNNEL: 285 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL; 286 break; 287 case XFRM_MODE_TRANSPORT: 288 cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT; 289 break; 290 default: 291 NL_SET_ERR_MSG_MOD(extack, "Unsupported mode for xfrm offload"); 292 return -EINVAL; 293 } 294 295 switch (x->id.proto) { 296 case IPPROTO_ESP: 297 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP; 298 break; 299 case IPPROTO_AH: 300 cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH; 301 break; 302 default: 303 NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for xfrm offload"); 304 return -EINVAL; 305 } 306 307 if (x->props.flags & XFRM_STATE_ESN) { 308 NL_SET_ERR_MSG_MOD(extack, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload"); 309 return -EINVAL; 310 } 311 312 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) { 313 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type"); 314 return -EINVAL; 315 } 316 317 cfg->spi = ntohl(x->id.spi); 318 319 /* Hash/Authentication */ 320 if (x->aalg) 321 trunc_len = x->aalg->alg_trunc_len; 322 else 323 trunc_len = 0; 324 325 switch (x->props.aalgo) { 326 case SADB_AALG_NONE: 327 if (x->aead) { 328 trunc_len = -1; 329 } else { 330 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 331 return -EINVAL; 332 } 333 break; 334 case SADB_X_AALG_NULL: 335 cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE; 336 trunc_len = -1; 337 break; 338 case SADB_AALG_MD5HMAC: 339 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) { 340 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 341 return -EINVAL; 342 } 343 set_md5hmac(cfg, &trunc_len); 344 break; 345 case SADB_AALG_SHA1HMAC: 346 set_sha1hmac(cfg, &trunc_len); 347 break; 348 case SADB_X_AALG_SHA2_256HMAC: 349 set_sha2_256hmac(cfg, &trunc_len); 350 break; 351 case SADB_X_AALG_SHA2_384HMAC: 352 set_sha2_384hmac(cfg, &trunc_len); 353 break; 354 case SADB_X_AALG_SHA2_512HMAC: 355 set_sha2_512hmac(cfg, &trunc_len); 356 break; 357 default: 358 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm"); 359 return -EINVAL; 360 } 361 362 if (!trunc_len) { 363 NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm trunc length"); 364 return -EINVAL; 365 } 366 367 if (x->aalg) { 368 key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE); 369 if (key_len > sizeof(cfg->auth_key)) { 370 NL_SET_ERR_MSG_MOD(extack, "Insufficient space for offloaded auth key"); 371 return -EINVAL; 372 } 373 for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++) 374 cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key + 375 sizeof(cfg->auth_key[0]) * i); 376 } 377 378 /* Encryption */ 379 switch (x->props.ealgo) { 380 case SADB_EALG_NONE: 381 case SADB_EALG_NULL: 382 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 383 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL; 384 break; 385 case SADB_EALG_3DESCBC: 386 if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) { 387 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload"); 388 return -EINVAL; 389 } 390 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 391 cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES; 392 break; 393 case SADB_X_EALG_AES_GCM_ICV16: 394 case SADB_X_EALG_NULL_AES_GMAC: 395 if (!x->aead) { 396 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data"); 397 return -EINVAL; 398 } 399 400 if (x->aead->alg_icv_len != 128) { 401 NL_SET_ERR_MSG_MOD(extack, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16"); 402 return -EINVAL; 403 } 404 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR; 405 cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128; 406 407 /* Aead->alg_key_len includes 32-bit salt */ 408 if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) { 409 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length"); 410 return -EINVAL; 411 } 412 break; 413 case SADB_X_EALG_AESCBC: 414 cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC; 415 if (!x->ealg) { 416 NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data"); 417 return -EINVAL; 418 } 419 if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) { 420 NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length"); 421 return -EINVAL; 422 } 423 break; 424 default: 425 NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload"); 426 return -EINVAL; 427 } 428 429 if (x->aead) { 430 int salt_len = 4; 431 432 key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE); 433 key_len -= salt_len; 434 435 if (key_len > sizeof(cfg->ciph_key)) { 436 NL_SET_ERR_MSG_MOD(extack, "aead: Insufficient space for offloaded key"); 437 return -EINVAL; 438 } 439 440 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 441 cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key + 442 sizeof(cfg->ciph_key[0]) * i); 443 444 /* Load up the salt */ 445 cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len); 446 } 447 448 if (x->ealg) { 449 key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE); 450 451 if (key_len > sizeof(cfg->ciph_key)) { 452 NL_SET_ERR_MSG_MOD(extack, "ealg: Insufficient space for offloaded key"); 453 return -EINVAL; 454 } 455 for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++) 456 cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key + 457 sizeof(cfg->ciph_key[0]) * i); 458 } 459 460 /* IP related info */ 461 switch (x->props.family) { 462 case AF_INET: 463 cfg->ipv6 = 0; 464 cfg->src_ip[0] = ntohl(x->props.saddr.a4); 465 cfg->dst_ip[0] = ntohl(x->id.daddr.a4); 466 break; 467 case AF_INET6: 468 cfg->ipv6 = 1; 469 for (i = 0; i < 4; i++) { 470 cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]); 471 cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]); 472 } 473 break; 474 default: 475 NL_SET_ERR_MSG_MOD(extack, "Unsupported address family"); 476 return -EINVAL; 477 } 478 479 /* Maximum nic IPsec code could handle. Other limits may apply. */ 480 cfg->pmtu_limit = 0xffff; 481 cfg->ctrl_word.encap_dsbl = 1; 482 483 /* SA direction */ 484 cfg->ctrl_word.dir = x->xso.dir; 485 486 /* Find unused SA data*/ 487 err = xa_alloc(&nn->xa_ipsec, &saidx, x, 488 XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL); 489 if (err < 0) { 490 NL_SET_ERR_MSG_MOD(extack, "Unable to get sa_data number for IPsec"); 491 return err; 492 } 493 494 /* Allocate saidx and commit the SA */ 495 msg.cmd = NFP_IPSEC_CFG_MSSG_ADD_SA; 496 msg.sa_idx = saidx; 497 err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, &msg, 498 sizeof(msg), nfp_net_ipsec_cfg); 499 if (err) { 500 xa_erase(&nn->xa_ipsec, saidx); 501 NL_SET_ERR_MSG_MOD(extack, "Failed to issue IPsec command"); 502 return err; 503 } 504 505 /* 0 is invalid offload_handle for kernel */ 506 x->xso.offload_handle = saidx + 1; 507 return 0; 508 } 509 510 static void nfp_net_xfrm_del_state(struct xfrm_state *x) 511 { 512 struct nfp_ipsec_cfg_mssg msg = { 513 .cmd = NFP_IPSEC_CFG_MSSG_INV_SA, 514 .sa_idx = x->xso.offload_handle - 1, 515 }; 516 struct net_device *netdev = x->xso.real_dev; 517 struct nfp_net *nn; 518 int err; 519 520 nn = netdev_priv(netdev); 521 err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, &msg, 522 sizeof(msg), nfp_net_ipsec_cfg); 523 if (err) 524 nn_warn(nn, "Failed to invalidate SA in hardware\n"); 525 526 xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1); 527 } 528 529 static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 530 { 531 if (x->props.family == AF_INET) 532 /* Offload with IPv4 options is not supported yet */ 533 return ip_hdr(skb)->ihl == 5; 534 535 /* Offload with IPv6 extension headers is not support yet */ 536 return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr)); 537 } 538 539 static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = { 540 .xdo_dev_state_add = nfp_net_xfrm_add_state, 541 .xdo_dev_state_delete = nfp_net_xfrm_del_state, 542 .xdo_dev_offload_ok = nfp_net_ipsec_offload_ok, 543 }; 544 545 void nfp_net_ipsec_init(struct nfp_net *nn) 546 { 547 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 548 return; 549 550 xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC); 551 nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops; 552 } 553 554 void nfp_net_ipsec_clean(struct nfp_net *nn) 555 { 556 if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)) 557 return; 558 559 WARN_ON(!xa_empty(&nn->xa_ipsec)); 560 xa_destroy(&nn->xa_ipsec); 561 } 562 563 bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb, 564 struct nfp_ipsec_offload *offload_info) 565 { 566 struct xfrm_offload *xo = xfrm_offload(skb); 567 struct xfrm_state *x; 568 569 x = xfrm_input_state(skb); 570 if (!x) 571 return false; 572 573 offload_info->seq_hi = xo->seq.hi; 574 offload_info->seq_low = xo->seq.low; 575 offload_info->handle = x->xso.offload_handle; 576 577 return true; 578 } 579 580 int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb) 581 { 582 struct net_device *netdev = skb->dev; 583 struct xfrm_offload *xo; 584 struct xfrm_state *x; 585 struct sec_path *sp; 586 struct nfp_net *nn; 587 u32 saidx; 588 589 nn = netdev_priv(netdev); 590 591 saidx = meta->ipsec_saidx - 1; 592 if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT) 593 return -EINVAL; 594 595 sp = secpath_set(skb); 596 if (unlikely(!sp)) 597 return -ENOMEM; 598 599 xa_lock(&nn->xa_ipsec); 600 x = xa_load(&nn->xa_ipsec, saidx); 601 xa_unlock(&nn->xa_ipsec); 602 if (!x) 603 return -EINVAL; 604 605 xfrm_state_hold(x); 606 sp->xvec[sp->len++] = x; 607 sp->olen++; 608 xo = xfrm_offload(skb); 609 xo->flags = CRYPTO_DONE; 610 xo->status = CRYPTO_SUCCESS; 611 612 return 0; 613 } 614