1 // SPDX-License-Identifier: GPL-2.0
2 /* OpenVPN data channel offload
3 *
4 * Copyright (C) 2020-2025 OpenVPN, Inc.
5 *
6 * Author: James Yonan <james@openvpn.net>
7 * Antonio Quartulli <antonio@openvpn.net>
8 */
9
10 #include <crypto/aead.h>
11 #include <linux/skbuff.h>
12 #include <linux/workqueue.h>
13 #include <net/ip.h>
14 #include <net/ipv6.h>
15 #include <net/udp.h>
16
17 #include "ovpnpriv.h"
18 #include "main.h"
19 #include "io.h"
20 #include "pktid.h"
21 #include "crypto_aead.h"
22 #include "crypto.h"
23 #include "peer.h"
24 #include "proto.h"
25 #include "skb.h"
26
27 #define OVPN_AUTH_TAG_SIZE 16
28 #define OVPN_AAD_SIZE (OVPN_OPCODE_SIZE + OVPN_NONCE_WIRE_SIZE)
29
30 #define ALG_NAME_AES "gcm(aes)"
31 #define ALG_NAME_CHACHAPOLY "rfc7539(chacha20,poly1305)"
32
ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot * ks)33 static int ovpn_aead_encap_overhead(const struct ovpn_crypto_key_slot *ks)
34 {
35 return OVPN_OPCODE_SIZE + /* OP header size */
36 sizeof(u32) + /* Packet ID */
37 crypto_aead_authsize(ks->encrypt); /* Auth Tag */
38 }
39
40 /**
41 * ovpn_aead_crypto_tmp_size - compute the size of a temporary object containing
42 * an AEAD request structure with extra space for SG
43 * and IV.
44 * @tfm: the AEAD cipher handle
45 * @nfrags: the number of fragments in the skb
46 *
47 * This function calculates the size of a contiguous memory block that includes
48 * the initialization vector (IV), the AEAD request, and an array of scatterlist
49 * entries. For alignment considerations, the IV is placed first, followed by
50 * the request, and then the scatterlist.
51 * Additional alignment is applied according to the requirements of the
52 * underlying structures.
53 *
54 * Return: the size of the temporary memory that needs to be allocated
55 */
ovpn_aead_crypto_tmp_size(struct crypto_aead * tfm,const unsigned int nfrags)56 static unsigned int ovpn_aead_crypto_tmp_size(struct crypto_aead *tfm,
57 const unsigned int nfrags)
58 {
59 unsigned int len = OVPN_NONCE_SIZE;
60
61 DEBUG_NET_WARN_ON_ONCE(crypto_aead_ivsize(tfm) != OVPN_NONCE_SIZE);
62
63 /* min size for a buffer of ivsize, aligned to alignmask */
64 len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1);
65 /* round up to the next multiple of the crypto ctx alignment */
66 len = ALIGN(len, crypto_tfm_ctx_alignment());
67
68 /* reserve space for the AEAD request */
69 len += sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
70 /* round up to the next multiple of the scatterlist alignment */
71 len = ALIGN(len, __alignof__(struct scatterlist));
72
73 /* add enough space for nfrags + 2 scatterlist entries */
74 len += array_size(sizeof(struct scatterlist), nfrags + 2);
75 return len;
76 }
77
78 /**
79 * ovpn_aead_crypto_tmp_iv - retrieve the pointer to the IV within a temporary
80 * buffer allocated using ovpn_aead_crypto_tmp_size
81 * @aead: the AEAD cipher handle
82 * @tmp: a pointer to the beginning of the temporary buffer
83 *
84 * This function retrieves a pointer to the initialization vector (IV) in the
85 * temporary buffer. If the AEAD cipher specifies an IV size, the pointer is
86 * adjusted using the AEAD's alignment mask to ensure proper alignment.
87 *
88 * Returns: a pointer to the IV within the temporary buffer
89 */
ovpn_aead_crypto_tmp_iv(struct crypto_aead * aead,void * tmp)90 static u8 *ovpn_aead_crypto_tmp_iv(struct crypto_aead *aead, void *tmp)
91 {
92 return likely(crypto_aead_ivsize(aead)) ?
93 PTR_ALIGN((u8 *)tmp, crypto_aead_alignmask(aead) + 1) :
94 tmp;
95 }
96
97 /**
98 * ovpn_aead_crypto_tmp_req - retrieve the pointer to the AEAD request structure
99 * within a temporary buffer allocated using
100 * ovpn_aead_crypto_tmp_size
101 * @aead: the AEAD cipher handle
102 * @iv: a pointer to the initialization vector in the temporary buffer
103 *
104 * This function computes the location of the AEAD request structure that
105 * immediately follows the IV in the temporary buffer and it ensures the request
106 * is aligned to the crypto transform context alignment.
107 *
108 * Returns: a pointer to the AEAD request structure
109 */
ovpn_aead_crypto_tmp_req(struct crypto_aead * aead,const u8 * iv)110 static struct aead_request *ovpn_aead_crypto_tmp_req(struct crypto_aead *aead,
111 const u8 *iv)
112 {
113 return (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
114 crypto_tfm_ctx_alignment());
115 }
116
117 /**
118 * ovpn_aead_crypto_req_sg - locate the scatterlist following the AEAD request
119 * within a temporary buffer allocated using
120 * ovpn_aead_crypto_tmp_size
121 * @aead: the AEAD cipher handle
122 * @req: a pointer to the AEAD request structure in the temporary buffer
123 *
124 * This function computes the starting address of the scatterlist that is
125 * allocated immediately after the AEAD request structure. It aligns the pointer
126 * based on the alignment requirements of the scatterlist structure.
127 *
128 * Returns: a pointer to the scatterlist
129 */
ovpn_aead_crypto_req_sg(struct crypto_aead * aead,struct aead_request * req)130 static struct scatterlist *ovpn_aead_crypto_req_sg(struct crypto_aead *aead,
131 struct aead_request *req)
132 {
133 return (void *)ALIGN((unsigned long)(req + 1) +
134 crypto_aead_reqsize(aead),
135 __alignof__(struct scatterlist));
136 }
137
ovpn_aead_encrypt(struct ovpn_peer * peer,struct ovpn_crypto_key_slot * ks,struct sk_buff * skb)138 int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
139 struct sk_buff *skb)
140 {
141 const unsigned int tag_size = crypto_aead_authsize(ks->encrypt);
142 struct aead_request *req;
143 struct sk_buff *trailer;
144 struct scatterlist *sg;
145 int nfrags, ret;
146 u32 pktid, op;
147 void *tmp;
148 u8 *iv;
149
150 ovpn_skb_cb(skb)->peer = peer;
151 ovpn_skb_cb(skb)->ks = ks;
152
153 /* Sample AEAD header format:
154 * 48000001 00000005 7e7046bd 444a7e28 cc6387b1 64a4d6c1 380275a...
155 * [ OP32 ] [seq # ] [ auth tag ] [ payload ... ]
156 * [4-byte
157 * IV head]
158 */
159
160 /* check that there's enough headroom in the skb for packet
161 * encapsulation
162 */
163 if (unlikely(skb_cow_head(skb, OVPN_HEAD_ROOM)))
164 return -ENOBUFS;
165
166 /* get number of skb frags and ensure that packet data is writable */
167 nfrags = skb_cow_data(skb, 0, &trailer);
168 if (unlikely(nfrags < 0))
169 return nfrags;
170
171 if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2)))
172 return -ENOSPC;
173
174 /* allocate temporary memory for iv, sg and req */
175 tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->encrypt, nfrags),
176 GFP_ATOMIC);
177 if (unlikely(!tmp))
178 return -ENOMEM;
179
180 ovpn_skb_cb(skb)->crypto_tmp = tmp;
181
182 iv = ovpn_aead_crypto_tmp_iv(ks->encrypt, tmp);
183 req = ovpn_aead_crypto_tmp_req(ks->encrypt, iv);
184 sg = ovpn_aead_crypto_req_sg(ks->encrypt, req);
185
186 /* sg table:
187 * 0: op, wire nonce (AD, len=OVPN_OP_SIZE_V2+OVPN_NONCE_WIRE_SIZE),
188 * 1, 2, 3, ..., n: payload,
189 * n+1: auth_tag (len=tag_size)
190 */
191 sg_init_table(sg, nfrags + 2);
192
193 /* build scatterlist to encrypt packet payload */
194 ret = skb_to_sgvec_nomark(skb, sg + 1, 0, skb->len);
195 if (unlikely(ret < 0)) {
196 netdev_err(peer->ovpn->dev,
197 "encrypt: cannot map skb to sg: %d\n", ret);
198 return ret;
199 }
200
201 /* append auth_tag onto scatterlist */
202 __skb_push(skb, tag_size);
203 sg_set_buf(sg + ret + 1, skb->data, tag_size);
204
205 /* obtain packet ID, which is used both as a first
206 * 4 bytes of nonce and last 4 bytes of associated data.
207 */
208 ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid);
209 if (unlikely(ret < 0))
210 return ret;
211
212 /* concat 4 bytes packet id and 8 bytes nonce tail into 12 bytes
213 * nonce
214 */
215 ovpn_pktid_aead_write(pktid, ks->nonce_tail_xmit, iv);
216
217 /* make space for packet id and push it to the front */
218 __skb_push(skb, OVPN_NONCE_WIRE_SIZE);
219 memcpy(skb->data, iv, OVPN_NONCE_WIRE_SIZE);
220
221 /* add packet op as head of additional data */
222 op = ovpn_opcode_compose(OVPN_DATA_V2, ks->key_id, peer->tx_id);
223 __skb_push(skb, OVPN_OPCODE_SIZE);
224 BUILD_BUG_ON(sizeof(op) != OVPN_OPCODE_SIZE);
225 *((__force __be32 *)skb->data) = htonl(op);
226
227 /* AEAD Additional data */
228 sg_set_buf(sg, skb->data, OVPN_AAD_SIZE);
229
230 /* setup async crypto operation */
231 aead_request_set_tfm(req, ks->encrypt);
232 aead_request_set_callback(req, 0, ovpn_encrypt_post, skb);
233 aead_request_set_crypt(req, sg, sg,
234 skb->len - ovpn_aead_encap_overhead(ks), iv);
235 aead_request_set_ad(req, OVPN_AAD_SIZE);
236
237 /* encrypt it */
238 return crypto_aead_encrypt(req);
239 }
240
ovpn_aead_decrypt(struct ovpn_peer * peer,struct ovpn_crypto_key_slot * ks,struct sk_buff * skb)241 int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks,
242 struct sk_buff *skb)
243 {
244 const unsigned int tag_size = crypto_aead_authsize(ks->decrypt);
245 int ret, payload_len, nfrags;
246 unsigned int payload_offset;
247 struct aead_request *req;
248 struct sk_buff *trailer;
249 struct scatterlist *sg;
250 void *tmp;
251 u8 *iv;
252
253 payload_offset = OVPN_AAD_SIZE + tag_size;
254 payload_len = skb->len - payload_offset;
255
256 ovpn_skb_cb(skb)->payload_offset = payload_offset;
257 ovpn_skb_cb(skb)->peer = peer;
258 ovpn_skb_cb(skb)->ks = ks;
259
260 /* sanity check on packet size, payload size must be >= 0 */
261 if (unlikely(payload_len < 0))
262 return -EINVAL;
263
264 /* Prepare the skb data buffer to be accessed up until the auth tag.
265 * This is required because this area is directly mapped into the sg
266 * list.
267 */
268 if (unlikely(!pskb_may_pull(skb, payload_offset)))
269 return -ENODATA;
270
271 /* get number of skb frags and ensure that packet data is writable */
272 nfrags = skb_cow_data(skb, 0, &trailer);
273 if (unlikely(nfrags < 0))
274 return nfrags;
275
276 if (unlikely(nfrags + 2 > (MAX_SKB_FRAGS + 2)))
277 return -ENOSPC;
278
279 /* allocate temporary memory for iv, sg and req */
280 tmp = kmalloc(ovpn_aead_crypto_tmp_size(ks->decrypt, nfrags),
281 GFP_ATOMIC);
282 if (unlikely(!tmp))
283 return -ENOMEM;
284
285 ovpn_skb_cb(skb)->crypto_tmp = tmp;
286
287 iv = ovpn_aead_crypto_tmp_iv(ks->decrypt, tmp);
288 req = ovpn_aead_crypto_tmp_req(ks->decrypt, iv);
289 sg = ovpn_aead_crypto_req_sg(ks->decrypt, req);
290
291 /* sg table:
292 * 0: op, wire nonce (AD, len=OVPN_OPCODE_SIZE+OVPN_NONCE_WIRE_SIZE),
293 * 1, 2, 3, ..., n: payload,
294 * n+1: auth_tag (len=tag_size)
295 */
296 sg_init_table(sg, nfrags + 2);
297
298 /* packet op is head of additional data */
299 sg_set_buf(sg, skb->data, OVPN_AAD_SIZE);
300
301 /* build scatterlist to decrypt packet payload */
302 ret = skb_to_sgvec_nomark(skb, sg + 1, payload_offset, payload_len);
303 if (unlikely(ret < 0)) {
304 netdev_err(peer->ovpn->dev,
305 "decrypt: cannot map skb to sg: %d\n", ret);
306 return ret;
307 }
308
309 /* append auth_tag onto scatterlist */
310 sg_set_buf(sg + ret + 1, skb->data + OVPN_AAD_SIZE, tag_size);
311
312 /* copy nonce into IV buffer */
313 memcpy(iv, skb->data + OVPN_OPCODE_SIZE, OVPN_NONCE_WIRE_SIZE);
314 memcpy(iv + OVPN_NONCE_WIRE_SIZE, ks->nonce_tail_recv,
315 OVPN_NONCE_TAIL_SIZE);
316
317 /* setup async crypto operation */
318 aead_request_set_tfm(req, ks->decrypt);
319 aead_request_set_callback(req, 0, ovpn_decrypt_post, skb);
320 aead_request_set_crypt(req, sg, sg, payload_len + tag_size, iv);
321
322 aead_request_set_ad(req, OVPN_AAD_SIZE);
323
324 /* decrypt it */
325 return crypto_aead_decrypt(req);
326 }
327
328 /* Initialize a struct crypto_aead object */
ovpn_aead_init(const char * title,const char * alg_name,const unsigned char * key,unsigned int keylen)329 static struct crypto_aead *ovpn_aead_init(const char *title,
330 const char *alg_name,
331 const unsigned char *key,
332 unsigned int keylen)
333 {
334 struct crypto_aead *aead;
335 int ret;
336
337 aead = crypto_alloc_aead(alg_name, 0, 0);
338 if (IS_ERR(aead)) {
339 ret = PTR_ERR(aead);
340 pr_err("%s crypto_alloc_aead failed, err=%d\n", title, ret);
341 aead = NULL;
342 goto error;
343 }
344
345 ret = crypto_aead_setkey(aead, key, keylen);
346 if (ret) {
347 pr_err("%s crypto_aead_setkey size=%u failed, err=%d\n", title,
348 keylen, ret);
349 goto error;
350 }
351
352 ret = crypto_aead_setauthsize(aead, OVPN_AUTH_TAG_SIZE);
353 if (ret) {
354 pr_err("%s crypto_aead_setauthsize failed, err=%d\n", title,
355 ret);
356 goto error;
357 }
358
359 /* basic AEAD assumption
360 * all current algorithms use OVPN_NONCE_SIZE.
361 * ovpn_aead_crypto_tmp_size and ovpn_aead_encrypt/decrypt
362 * expect this.
363 */
364 if (crypto_aead_ivsize(aead) != OVPN_NONCE_SIZE) {
365 pr_err("%s IV size must be %d\n", title, OVPN_NONCE_SIZE);
366 ret = -EINVAL;
367 goto error;
368 }
369
370 pr_debug("********* Cipher %s (%s)\n", alg_name, title);
371 pr_debug("*** IV size=%u\n", crypto_aead_ivsize(aead));
372 pr_debug("*** req size=%u\n", crypto_aead_reqsize(aead));
373 pr_debug("*** block size=%u\n", crypto_aead_blocksize(aead));
374 pr_debug("*** auth size=%u\n", crypto_aead_authsize(aead));
375 pr_debug("*** alignmask=0x%x\n", crypto_aead_alignmask(aead));
376
377 return aead;
378
379 error:
380 crypto_free_aead(aead);
381 return ERR_PTR(ret);
382 }
383
ovpn_aead_crypto_key_slot_free(struct ovpn_crypto_key_slot * ks)384 static void ovpn_aead_crypto_key_slot_free(struct ovpn_crypto_key_slot *ks)
385 {
386 crypto_free_aead(ks->encrypt);
387 crypto_free_aead(ks->decrypt);
388 }
389
ovpn_aead_crypto_key_slot_free_work(struct work_struct * work)390 static void ovpn_aead_crypto_key_slot_free_work(struct work_struct *work)
391 {
392 struct ovpn_crypto_key_slot *ks;
393
394 ks = container_of(to_rcu_work(work), struct ovpn_crypto_key_slot,
395 free_work);
396 ovpn_aead_crypto_key_slot_free(ks);
397 kfree(ks);
398 }
399
400 struct ovpn_crypto_key_slot *
ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config * kc)401 ovpn_aead_crypto_key_slot_new(const struct ovpn_key_config *kc)
402 {
403 struct ovpn_crypto_key_slot *ks = NULL;
404 const char *alg_name;
405 int ret;
406
407 /* validate crypto alg */
408 switch (kc->cipher_alg) {
409 case OVPN_CIPHER_ALG_AES_GCM:
410 alg_name = ALG_NAME_AES;
411 break;
412 case OVPN_CIPHER_ALG_CHACHA20_POLY1305:
413 alg_name = ALG_NAME_CHACHAPOLY;
414 break;
415 default:
416 return ERR_PTR(-EOPNOTSUPP);
417 }
418
419 if (kc->encrypt.nonce_tail_size != OVPN_NONCE_TAIL_SIZE ||
420 kc->decrypt.nonce_tail_size != OVPN_NONCE_TAIL_SIZE)
421 return ERR_PTR(-EINVAL);
422
423 /* build the key slot */
424 ks = kmalloc_obj(*ks);
425 if (!ks)
426 return ERR_PTR(-ENOMEM);
427
428 ks->encrypt = NULL;
429 ks->decrypt = NULL;
430 INIT_RCU_WORK(&ks->free_work, ovpn_aead_crypto_key_slot_free_work);
431 kref_init(&ks->refcount);
432 ks->key_id = kc->key_id;
433
434 ks->encrypt = ovpn_aead_init("encrypt", alg_name,
435 kc->encrypt.cipher_key,
436 kc->encrypt.cipher_key_size);
437 if (IS_ERR(ks->encrypt)) {
438 ret = PTR_ERR(ks->encrypt);
439 ks->encrypt = NULL;
440 goto destroy_ks;
441 }
442
443 ks->decrypt = ovpn_aead_init("decrypt", alg_name,
444 kc->decrypt.cipher_key,
445 kc->decrypt.cipher_key_size);
446 if (IS_ERR(ks->decrypt)) {
447 ret = PTR_ERR(ks->decrypt);
448 ks->decrypt = NULL;
449 goto destroy_ks;
450 }
451
452 memcpy(ks->nonce_tail_xmit, kc->encrypt.nonce_tail,
453 OVPN_NONCE_TAIL_SIZE);
454 memcpy(ks->nonce_tail_recv, kc->decrypt.nonce_tail,
455 OVPN_NONCE_TAIL_SIZE);
456
457 /* init packet ID generation/validation */
458 ovpn_pktid_xmit_init(&ks->pid_xmit);
459 ovpn_pktid_recv_init(&ks->pid_recv);
460
461 return ks;
462
463 destroy_ks:
464 ovpn_aead_crypto_key_slot_free(ks);
465 kfree(ks);
466 return ERR_PTR(ret);
467 }
468
ovpn_aead_crypto_alg(struct ovpn_crypto_key_slot * ks)469 enum ovpn_cipher_alg ovpn_aead_crypto_alg(struct ovpn_crypto_key_slot *ks)
470 {
471 const char *alg_name;
472
473 if (!ks->encrypt)
474 return OVPN_CIPHER_ALG_NONE;
475
476 alg_name = crypto_tfm_alg_name(crypto_aead_tfm(ks->encrypt));
477
478 if (!strcmp(alg_name, ALG_NAME_AES))
479 return OVPN_CIPHER_ALG_AES_GCM;
480 else if (!strcmp(alg_name, ALG_NAME_CHACHAPOLY))
481 return OVPN_CIPHER_ALG_CHACHA20_POLY1305;
482 else
483 return OVPN_CIPHER_ALG_NONE;
484 }
485