1 // SPDX-License-Identifier: GPL-2.0 2 /* OpenVPN data channel offload 3 * 4 * Copyright (C) 2020-2025 OpenVPN, Inc. 5 * 6 * Author: James Yonan <james@openvpn.net> 7 * Antonio Quartulli <antonio@openvpn.net> 8 */ 9 10 #include <crypto/aead.h> 11 #include <linux/skbuff.h> 12 #include <linux/workqueue.h> 13 #include <net/ip.h> 14 #include <net/ipv6.h> 15 #include <net/udp.h> 16 17 #include "ovpnpriv.h" 18 #include "main.h" 19 #include "io.h" 20 #include "pktid.h" 21 #include "crypto_aead.h" 22 #include "crypto.h" 23 #include "peer.h" 24 #include "proto.h" 25 #include "skb.h" 26 27 #define OVPN_AUTH_TAG_SIZE 16 28 #define OVPN_AAD_SIZE (OVPN_OPCODE_SIZE + OVPN_NONCE_WIRE_SIZE) 29 30 #define ALG_NAME_AES "gcm(aes)" 31 #define ALG_NAME_CHACHAPOLY "rfc7539(chacha20,poly1305)" 32 33 static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks) 34 { 35 return OVPN_OPCODE_SIZE + /* OP header size */ 36 sizeof(u32) + /* Packet ID */ 37 crypto_aead_authsize(ks->encrypt); /* Auth Tag */ 38 } 39 40 /** 41 * ovpn_aead_crypto_tmp_size - compute the size of a temporary object containing 42 * an AEAD request structure with extra space for SG 43 * and IV. 44 * @tfm: the AEAD cipher handle 45 * @nfrags: the number of fragments in the skb 46 * 47 * This function calculates the size of a contiguous memory block that includes 48 * the initialization vector (IV), the AEAD request, and an array of scatterlist 49 * entries. For alignment considerations, the IV is placed first, followed by 50 * the request, and then the scatterlist. 51 * Additional alignment is applied according to the requirements of the 52 * underlying structures. 53 * 54 * Return: the size of the temporary memory that needs to be allocated 55 */ 56 static unsigned int ovpn_aead_crypto_tmp_size(struct crypto_aead *tfm, 57 const unsigned int nfrags) 58 { 59 unsigned int len = OVPN_NONCE_SIZE; 60 61 DEBUG_NET_WARN_ON_ONCE(crypto_aead_ivsize(tfm) != OVPN_NONCE_SIZE); 62 63 /* min size for a buffer of ivsize, aligned to alignmask */ 64 len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1); 65 /* round up to the next multiple of the crypto ctx alignment */ 66 len = ALIGN(len, crypto_tfm_ctx_alignment()); 67 68 /* reserve space for the AEAD request */ 69 len += sizeof(struct aead_request) + crypto_aead_reqsize(tfm); 70 /* round up to the next multiple of the scatterlist alignment */ 71 len = ALIGN(len, __alignof__(struct scatterlist)); 72 73 /* add enough space for nfrags + 2 scatterlist entries */ 74 len += array_size(sizeof(struct scatterlist), nfrags + 2); 75 return len; 76 } 77 78 /** 79 * ovpn_aead_crypto_tmp_iv - retrieve the pointer to the IV within a temporary 80 * buffer allocated using ovpn_aead_crypto_tmp_size 81 * @aead: the AEAD cipher handle 82 * @tmp: a pointer to the beginning of the temporary buffer 83 * 84 * This function retrieves a pointer to the initialization vector (IV) in the 85 * temporary buffer. If the AEAD cipher specifies an IV size, the pointer is 86 * adjusted using the AEAD's alignment mask to ensure proper alignment. 87 * 88 * Returns: a pointer to the IV within the temporary buffer 89 */ 90 static u8 *ovpn_aead_crypto_tmp_iv(struct crypto_aead *aead, void *tmp) 91 { 92 return likely(crypto_aead_ivsize(aead)) ? 93 PTR_ALIGN((u8 *)tmp, crypto_aead_alignmask(aead) + 1) : 94 tmp; 95 } 96 97 /** 98 * ovpn_aead_crypto_tmp_req - retrieve the pointer to the AEAD request structure 99 * within a temporary buffer allocated using 100 * ovpn_aead_crypto_tmp_size 101 * @aead: the AEAD cipher handle 102 * @iv: a pointer to the initialization vector in the temporary buffer 103 * 104 * This function computes the location of the AEAD request structure that 105 * immediately follows the IV in the temporary buffer and it ensures the request 106 * is aligned to the crypto transform context alignment. 107 * 108 * Returns: a pointer to the AEAD request structure 109 */ 110 static struct aead_request *ovpn_aead_crypto_tmp_req(struct crypto_aead *aead, 111 const u8 *iv) 112 { 113 return (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead), 114 crypto_tfm_ctx_alignment()); 115 } 116 117 /** 118 * ovpn_aead_crypto_req_sg - locate the scatterlist following the AEAD request 119 * within a temporary buffer allocated using 120 * ovpn_aead_crypto_tmp_size 121 * @aead: the AEAD cipher handle 122 * @req: a pointer to the AEAD request structure in the temporary buffer 123 * 124 * This function computes the starting address of the scatterlist that is 125 * allocated immediately after the AEAD request structure. It aligns the pointer 126 * based on the alignment requirements of the scatterlist structure. 127 * 128 * Returns: a pointer to the scatterlist 129 */ 130 static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead, 131 struct aead_request *req) 132 { 133 return (void *)ALIGN((unsigned long)(req + 1) + 134 crypto_aead_reqsize(aead), 135 __alignof__(struct scatterlist)); 136 } 137 138 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, 139 struct sk_buff *skb) 140 { 141 const unsigned int tag_size = crypto_aead_authsize(ks->encrypt); 142 struct aead_request *req; 143 struct sk_buff *trailer; 144 struct scatterlist *sg; 145 int nfrags, ret; 146 u32 pktid, op; 147 void *tmp; 148 u8 *iv; 149 150 ovpn_skb_cb(skb)->peer = peer; 151 ovpn_skb_cb(skb)->ks = ks; 152 153 /* Sample AEAD header format: 154 * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a... 155 * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ] 156 * [4-byte 157 * IV head] 158 */ 159 160 /* check that there's enough headroom in the skb for packet 161 * encapsulation 162 */ 163 if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM))) 164 return -ENOBUFS; 165 166 /* get number of skb frags and ensure that packet data is writable */ 167 nfrags = skb_cow_data(skb, 0, &trailer); 168 if (unlikely(nfrags < 0)) 169 return nfrags; 170 171 if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2))) 172 return -ENOSPC; 173 174 /* allocate temporary memory for iv, sg and req */ 175 tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->encrypt, nfrags), 176 GFP_ATOMIC); 177 if (unlikely(!tmp)) 178 return -ENOMEM; 179 180 ovpn_skb_cb(skb)->crypto_tmp = tmp; 181 182 iv = ovpn_aead_crypto_tmp_iv(ks->encrypt, tmp); 183 req = ovpn_aead_crypto_tmp_req(ks->encrypt, iv); 184 sg = ovpn_aead_crypto_req_sg(ks->encrypt, req); 185 186 /* sg table: 187 * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+OVPN_NONCE_WIRE_SIZE), 188 * 1, 2, 3, ..., n: payload, 189 * n+1: auth_tag (len=tag_size) 190 */ 191 sg_init_table(sg, nfrags + 2); 192 193 /* build scatterlist to encrypt packet payload */ 194 ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len); 195 if (unlikely(ret < 0)) { 196 netdev_err(peer->ovpn->dev, 197 "encrypt: cannot map skb to sg: %d\n", ret); 198 return ret; 199 } 200 201 /* append auth_tag onto scatterlist */ 202 __skb_push(skb, tag_size); 203 sg_set_buf(sg + ret + 1, skb->data, tag_size); 204 205 /* obtain packet ID, which is used both as a first 206 * 4 bytes of nonce and last 4 bytes of associated data. 207 */ 208 ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); 209 if (unlikely(ret < 0)) 210 return ret; 211 212 /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes 213 * nonce 214 */ 215 ovpn_pktid_aead_write(pktid, ks->nonce_tail_xmit, iv); 216 217 /* make space for packet id and push it to the front */ 218 __skb_push(skb, OVPN_NONCE_WIRE_SIZE); 219 memcpy(skb->data, iv, OVPN_NONCE_WIRE_SIZE); 220 221 /* add packet op as head of additional data */ 222 op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id); 223 __skb_push(skb, OVPN_OPCODE_SIZE); 224 BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE); 225 *((__force __be32 *)skb->data) = htonl(op); 226 227 /* AEAD Additional data */ 228 sg_set_buf(sg, skb->data, OVPN_AAD_SIZE); 229 230 /* setup async crypto operation */ 231 aead_request_set_tfm(req, ks->encrypt); 232 aead_request_set_callback(req, 0, ovpn_encrypt_post, skb); 233 aead_request_set_crypt(req, sg, sg, 234 skb->len - ovpn_aead_encap_overhead(ks), iv); 235 aead_request_set_ad(req, OVPN_AAD_SIZE); 236 237 /* encrypt it */ 238 return crypto_aead_encrypt(req); 239 } 240 241 int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, 242 struct sk_buff *skb) 243 { 244 const unsigned int tag_size = crypto_aead_authsize(ks->decrypt); 245 int ret, payload_len, nfrags; 246 unsigned int payload_offset; 247 struct aead_request *req; 248 struct sk_buff *trailer; 249 struct scatterlist *sg; 250 void *tmp; 251 u8 *iv; 252 253 payload_offset = OVPN_AAD_SIZE + tag_size; 254 payload_len = skb->len - payload_offset; 255 256 ovpn_skb_cb(skb)->payload_offset = payload_offset; 257 ovpn_skb_cb(skb)->peer = peer; 258 ovpn_skb_cb(skb)->ks = ks; 259 260 /* sanity check on packet size, payload size must be >= 0 */ 261 if (unlikely(payload_len < 0)) 262 return -EINVAL; 263 264 /* Prepare the skb data buffer to be accessed up until the auth tag. 265 * This is required because this area is directly mapped into the sg 266 * list. 267 */ 268 if (unlikely(!pskb_may_pull(skb, payload_offset))) 269 return -ENODATA; 270 271 /* get number of skb frags and ensure that packet data is writable */ 272 nfrags = skb_cow_data(skb, 0, &trailer); 273 if (unlikely(nfrags < 0)) 274 return nfrags; 275 276 if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2))) 277 return -ENOSPC; 278 279 /* allocate temporary memory for iv, sg and req */ 280 tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->decrypt, nfrags), 281 GFP_ATOMIC); 282 if (unlikely(!tmp)) 283 return -ENOMEM; 284 285 ovpn_skb_cb(skb)->crypto_tmp = tmp; 286 287 iv = ovpn_aead_crypto_tmp_iv(ks->decrypt, tmp); 288 req = ovpn_aead_crypto_tmp_req(ks->decrypt, iv); 289 sg = ovpn_aead_crypto_req_sg(ks->decrypt, req); 290 291 /* sg table: 292 * 0: op, wire nonce (AD, len=OVPN_OPCODE_SIZE+OVPN_NONCE_WIRE_SIZE), 293 * 1, 2, 3, ..., n: payload, 294 * n+1: auth_tag (len=tag_size) 295 */ 296 sg_init_table(sg, nfrags + 2); 297 298 /* packet op is head of additional data */ 299 sg_set_buf(sg, skb->data, OVPN_AAD_SIZE); 300 301 /* build scatterlist to decrypt packet payload */ 302 ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len); 303 if (unlikely(ret < 0)) { 304 netdev_err(peer->ovpn->dev, 305 "decrypt: cannot map skb to sg: %d\n", ret); 306 return ret; 307 } 308 309 /* append auth_tag onto scatterlist */ 310 sg_set_buf(sg + ret + 1, skb->data + OVPN_AAD_SIZE, tag_size); 311 312 /* copy nonce into IV buffer */ 313 memcpy(iv, skb->data + OVPN_OPCODE_SIZE, OVPN_NONCE_WIRE_SIZE); 314 memcpy(iv + OVPN_NONCE_WIRE_SIZE, ks->nonce_tail_recv, 315 OVPN_NONCE_TAIL_SIZE); 316 317 /* setup async crypto operation */ 318 aead_request_set_tfm(req, ks->decrypt); 319 aead_request_set_callback(req, 0, ovpn_decrypt_post, skb); 320 aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv); 321 322 aead_request_set_ad(req, OVPN_AAD_SIZE); 323 324 /* decrypt it */ 325 return crypto_aead_decrypt(req); 326 } 327 328 /* Initialize a struct crypto_aead object */ 329 static struct crypto_aead *ovpn_aead_init(const char *title, 330 const char *alg_name, 331 const unsigned char *key, 332 unsigned int keylen) 333 { 334 struct crypto_aead *aead; 335 int ret; 336 337 aead = crypto_alloc_aead(alg_name, 0, 0); 338 if (IS_ERR(aead)) { 339 ret = PTR_ERR(aead); 340 pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret); 341 aead = NULL; 342 goto error; 343 } 344 345 ret = crypto_aead_setkey(aead, key, keylen); 346 if (ret) { 347 pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title, 348 keylen, ret); 349 goto error; 350 } 351 352 ret = crypto_aead_setauthsize(aead, OVPN_AUTH_TAG_SIZE); 353 if (ret) { 354 pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title, 355 ret); 356 goto error; 357 } 358 359 /* basic AEAD assumption 360 * all current algorithms use OVPN_NONCE_SIZE. 361 * ovpn_aead_crypto_tmp_size and ovpn_aead_encrypt/decrypt 362 * expect this. 363 */ 364 if (crypto_aead_ivsize(aead) != OVPN_NONCE_SIZE) { 365 pr_err("%s IV size must be %d\n", title, OVPN_NONCE_SIZE); 366 ret = -EINVAL; 367 goto error; 368 } 369 370 pr_debug("********* Cipher %s (%s)\n", alg_name, title); 371 pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead)); 372 pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead)); 373 pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead)); 374 pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead)); 375 pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead)); 376 377 return aead; 378 379 error: 380 crypto_free_aead(aead); 381 return ERR_PTR(ret); 382 } 383 384 static void ovpn_aead_crypto_key_slot_free(struct ovpn_crypto_key_slot *ks) 385 { 386 crypto_free_aead(ks->encrypt); 387 crypto_free_aead(ks->decrypt); 388 } 389 390 static void ovpn_aead_crypto_key_slot_free_work(struct work_struct *work) 391 { 392 struct ovpn_crypto_key_slot *ks; 393 394 ks = container_of(to_rcu_work(work), struct ovpn_crypto_key_slot, 395 free_work); 396 ovpn_aead_crypto_key_slot_free(ks); 397 kfree(ks); 398 } 399 400 struct ovpn_crypto_key_slot * 401 ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc) 402 { 403 struct ovpn_crypto_key_slot *ks = NULL; 404 const char *alg_name; 405 int ret; 406 407 /* validate crypto alg */ 408 switch (kc->cipher_alg) { 409 case OVPN_CIPHER_ALG_AES_GCM: 410 alg_name = ALG_NAME_AES; 411 break; 412 case OVPN_CIPHER_ALG_CHACHA20_POLY1305: 413 alg_name = ALG_NAME_CHACHAPOLY; 414 break; 415 default: 416 return ERR_PTR(-EOPNOTSUPP); 417 } 418 419 if (kc->encrypt.nonce_tail_size != OVPN_NONCE_TAIL_SIZE || 420 kc->decrypt.nonce_tail_size != OVPN_NONCE_TAIL_SIZE) 421 return ERR_PTR(-EINVAL); 422 423 /* build the key slot */ 424 ks = kmalloc_obj(*ks); 425 if (!ks) 426 return ERR_PTR(-ENOMEM); 427 428 ks->encrypt = NULL; 429 ks->decrypt = NULL; 430 INIT_RCU_WORK(&ks->free_work, ovpn_aead_crypto_key_slot_free_work); 431 kref_init(&ks->refcount); 432 ks->key_id = kc->key_id; 433 434 ks->encrypt = ovpn_aead_init("encrypt", alg_name, 435 kc->encrypt.cipher_key, 436 kc->encrypt.cipher_key_size); 437 if (IS_ERR(ks->encrypt)) { 438 ret = PTR_ERR(ks->encrypt); 439 ks->encrypt = NULL; 440 goto destroy_ks; 441 } 442 443 ks->decrypt = ovpn_aead_init("decrypt", alg_name, 444 kc->decrypt.cipher_key, 445 kc->decrypt.cipher_key_size); 446 if (IS_ERR(ks->decrypt)) { 447 ret = PTR_ERR(ks->decrypt); 448 ks->decrypt = NULL; 449 goto destroy_ks; 450 } 451 452 memcpy(ks->nonce_tail_xmit, kc->encrypt.nonce_tail, 453 OVPN_NONCE_TAIL_SIZE); 454 memcpy(ks->nonce_tail_recv, kc->decrypt.nonce_tail, 455 OVPN_NONCE_TAIL_SIZE); 456 457 /* init packet ID generation/validation */ 458 ovpn_pktid_xmit_init(&ks->pid_xmit); 459 ovpn_pktid_recv_init(&ks->pid_recv); 460 461 return ks; 462 463 destroy_ks: 464 ovpn_aead_crypto_key_slot_free(ks); 465 kfree(ks); 466 return ERR_PTR(ret); 467 } 468 469 enum ovpn_cipher_alg ovpn_aead_crypto_alg(struct ovpn_crypto_key_slot *ks) 470 { 471 const char *alg_name; 472 473 if (!ks->encrypt) 474 return OVPN_CIPHER_ALG_NONE; 475 476 alg_name = crypto_tfm_alg_name(crypto_aead_tfm(ks->encrypt)); 477 478 if (!strcmp(alg_name, ALG_NAME_AES)) 479 return OVPN_CIPHER_ALG_AES_GCM; 480 else if (!strcmp(alg_name, ALG_NAME_CHACHAPOLY)) 481 return OVPN_CIPHER_ALG_CHACHA20_POLY1305; 482 else 483 return OVPN_CIPHER_ALG_NONE; 484 } 485