1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * algif_aead: User-space interface for AEAD algorithms
4 *
5 * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
6 *
7 * This file provides the user-space API for AEAD ciphers.
8 *
9 * The following concept of the memory management is used:
10 *
11 * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
12 * filled by user space with the data submitted via sendmsg (maybe with
13 * MSG_SPLICE_PAGES). Filling up the TX SGL does not cause a crypto operation
14 * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg
15 * call, the caller must provide a buffer which is tracked with the RX SGL.
16 *
17 * During the processing of the recvmsg operation, the cipher request is
18 * allocated and prepared. As part of the recvmsg operation, the processed
19 * TX buffers are extracted from the TX SGL into a separate SGL.
20 *
21 * After the completion of the crypto operation, the RX SGL and the cipher
22 * request is released. The extracted TX SGL parts are released together with
23 * the RX SGL release.
24 */
25
26 #include <crypto/internal/aead.h>
27 #include <crypto/scatterwalk.h>
28 #include <crypto/if_alg.h>
29 #include <linux/init.h>
30 #include <linux/list.h>
31 #include <linux/kernel.h>
32 #include <linux/mm.h>
33 #include <linux/module.h>
34 #include <linux/net.h>
35 #include <net/sock.h>
36
aead_sufficient_data(struct sock * sk)37 static inline bool aead_sufficient_data(struct sock *sk)
38 {
39 struct alg_sock *ask = alg_sk(sk);
40 struct sock *psk = ask->parent;
41 struct alg_sock *pask = alg_sk(psk);
42 struct af_alg_ctx *ctx = ask->private;
43 struct crypto_aead *tfm = pask->private;
44 unsigned int as = crypto_aead_authsize(tfm);
45
46 /*
47 * The minimum amount of memory needed for an AEAD cipher is
48 * the AAD and in case of decryption the tag.
49 */
50 return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
51 }
52
aead_sendmsg(struct socket * sock,struct msghdr * msg,size_t size)53 static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
54 {
55 struct sock *sk = sock->sk;
56 struct alg_sock *ask = alg_sk(sk);
57 struct sock *psk = ask->parent;
58 struct alg_sock *pask = alg_sk(psk);
59 struct crypto_aead *tfm = pask->private;
60 unsigned int ivsize = crypto_aead_ivsize(tfm);
61
62 return af_alg_sendmsg(sock, msg, size, ivsize);
63 }
64
_aead_recvmsg(struct socket * sock,struct msghdr * msg,size_t ignored,int flags)65 static int _aead_recvmsg(struct socket *sock, struct msghdr *msg,
66 size_t ignored, int flags)
67 {
68 struct sock *sk = sock->sk;
69 struct alg_sock *ask = alg_sk(sk);
70 struct sock *psk = ask->parent;
71 struct alg_sock *pask = alg_sk(psk);
72 struct af_alg_ctx *ctx = ask->private;
73 struct crypto_aead *tfm = pask->private;
74 unsigned int as = crypto_aead_authsize(tfm);
75 unsigned int ivsize = crypto_aead_ivsize(tfm);
76 struct af_alg_async_req *areq;
77 struct scatterlist *rsgl_src, *tsgl_src = NULL;
78 void *iv;
79 int err = 0;
80 size_t used = 0; /* [in] TX bufs to be en/decrypted */
81 size_t outlen = 0; /* [out] RX bufs produced by kernel */
82 size_t usedpages = 0; /* [in] RX bufs to be used from user */
83 size_t processed = 0; /* [in] TX bufs to be consumed */
84
85 if (!ctx->init || ctx->more) {
86 err = af_alg_wait_for_data(sk, flags, 0);
87 if (err)
88 return err;
89 }
90
91 /*
92 * Data length provided by caller via sendmsg that has not yet been
93 * processed.
94 */
95 used = ctx->used;
96
97 /*
98 * Make sure sufficient data is present -- note, the same check is also
99 * present in sendmsg. The checks in sendmsg shall provide an
100 * information to the data sender that something is wrong, but they are
101 * irrelevant to maintain the kernel integrity. We need this check
102 * here too in case user space decides to not honor the error message
103 * in sendmsg and still call recvmsg. This check here protects the
104 * kernel integrity.
105 */
106 if (!aead_sufficient_data(sk))
107 return -EINVAL;
108
109 /*
110 * Calculate the minimum output buffer size holding the result of the
111 * cipher operation. When encrypting data, the receiving buffer is
112 * larger by the tag length compared to the input buffer as the
113 * encryption operation generates the tag. For decryption, the input
114 * buffer provides the tag which is consumed resulting in only the
115 * plaintext without a buffer for the tag returned to the caller.
116 */
117 if (ctx->enc)
118 outlen = used + as;
119 else
120 outlen = used - as;
121
122 /*
123 * The cipher operation input data is reduced by the associated data
124 * length as this data is processed separately later on.
125 */
126 used -= ctx->aead_assoclen;
127
128 /* Allocate cipher request for current operation. */
129 areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) +
130 crypto_aead_reqsize(tfm) + ivsize);
131 if (IS_ERR(areq))
132 return PTR_ERR(areq);
133
134 iv = (u8 *)aead_request_ctx(&areq->cra_u.aead_req) +
135 crypto_aead_reqsize(tfm);
136 memcpy(iv, ctx->iv, ivsize);
137
138 /* convert iovecs of output buffers into RX SGL */
139 err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages);
140 if (err)
141 goto free;
142
143 /*
144 * Ensure output buffer is sufficiently large. If the caller provides
145 * less buffer space, only use the relative required input size. This
146 * allows AIO operation where the caller sent all data to be processed
147 * and the AIO operation performs the operation on the different chunks
148 * of the input data.
149 */
150 if (usedpages < outlen) {
151 size_t less = outlen - usedpages;
152
153 if (used < less + (ctx->enc ? 0 : as)) {
154 err = -EINVAL;
155 goto free;
156 }
157 used -= less;
158 outlen -= less;
159 }
160
161 /*
162 * Create a per request TX SGL for this request which tracks the
163 * SG entries from the global TX SGL.
164 */
165 processed = used + ctx->aead_assoclen;
166 areq->tsgl_entries = af_alg_count_tsgl(sk, processed);
167 if (!areq->tsgl_entries)
168 areq->tsgl_entries = 1;
169 areq->tsgl = sock_kmalloc(sk, array_size(sizeof(*areq->tsgl),
170 areq->tsgl_entries),
171 GFP_KERNEL);
172 if (!areq->tsgl) {
173 err = -ENOMEM;
174 goto free;
175 }
176 sg_init_table(areq->tsgl, areq->tsgl_entries);
177 af_alg_pull_tsgl(sk, processed, areq->tsgl);
178 tsgl_src = areq->tsgl;
179
180 /*
181 * Copy of AAD from source to destination
182 *
183 * The AAD is copied to the destination buffer without change. Even
184 * when user space uses an in-place cipher operation, the kernel
185 * will copy the data as it does not see whether such in-place operation
186 * is initiated.
187 */
188
189 /* Use the RX SGL as source (and destination) for crypto op. */
190 rsgl_src = areq->first_rsgl.sgl.sgt.sgl;
191
192 memcpy_sglist(rsgl_src, tsgl_src, ctx->aead_assoclen);
193
194 /* Initialize the crypto operation */
195 aead_request_set_crypt(&areq->cra_u.aead_req, tsgl_src,
196 areq->first_rsgl.sgl.sgt.sgl, used, iv);
197 aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen);
198 aead_request_set_tfm(&areq->cra_u.aead_req, tfm);
199
200 if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) {
201 /* AIO operation */
202 sock_hold(sk);
203 areq->iocb = msg->msg_iocb;
204
205 /* Remember output size that will be generated. */
206 areq->outlen = outlen;
207
208 aead_request_set_callback(&areq->cra_u.aead_req,
209 CRYPTO_TFM_REQ_MAY_SLEEP,
210 af_alg_async_cb, areq);
211 err = ctx->enc ? crypto_aead_encrypt(&areq->cra_u.aead_req) :
212 crypto_aead_decrypt(&areq->cra_u.aead_req);
213
214 /* AIO operation in progress */
215 if (err == -EINPROGRESS)
216 return -EIOCBQUEUED;
217
218 sock_put(sk);
219 } else {
220 /* Synchronous operation */
221 aead_request_set_callback(&areq->cra_u.aead_req,
222 CRYPTO_TFM_REQ_MAY_SLEEP |
223 CRYPTO_TFM_REQ_MAY_BACKLOG,
224 crypto_req_done, &ctx->wait);
225 err = crypto_wait_req(ctx->enc ?
226 crypto_aead_encrypt(&areq->cra_u.aead_req) :
227 crypto_aead_decrypt(&areq->cra_u.aead_req),
228 &ctx->wait);
229 }
230
231
232 free:
233 af_alg_free_resources(areq);
234
235 return err ? err : outlen;
236 }
237
aead_recvmsg(struct socket * sock,struct msghdr * msg,size_t ignored,int flags)238 static int aead_recvmsg(struct socket *sock, struct msghdr *msg,
239 size_t ignored, int flags)
240 {
241 struct sock *sk = sock->sk;
242 int ret = 0;
243
244 lock_sock(sk);
245 while (msg_data_left(msg)) {
246 int err = _aead_recvmsg(sock, msg, ignored, flags);
247
248 /*
249 * This error covers -EIOCBQUEUED which implies that we can
250 * only handle one AIO request. If the caller wants to have
251 * multiple AIO requests in parallel, he must make multiple
252 * separate AIO calls.
253 *
254 * Also return the error if no data has been processed so far.
255 */
256 if (err <= 0) {
257 if (err == -EIOCBQUEUED || err == -EBADMSG || !ret)
258 ret = err;
259 goto out;
260 }
261
262 ret += err;
263 }
264
265 out:
266 af_alg_wmem_wakeup(sk);
267 release_sock(sk);
268 return ret;
269 }
270
271 static struct proto_ops algif_aead_ops = {
272 .family = PF_ALG,
273
274 .connect = sock_no_connect,
275 .socketpair = sock_no_socketpair,
276 .getname = sock_no_getname,
277 .ioctl = sock_no_ioctl,
278 .listen = sock_no_listen,
279 .shutdown = sock_no_shutdown,
280 .mmap = sock_no_mmap,
281 .bind = sock_no_bind,
282 .accept = sock_no_accept,
283
284 .release = af_alg_release,
285 .sendmsg = aead_sendmsg,
286 .recvmsg = aead_recvmsg,
287 .poll = af_alg_poll,
288 };
289
aead_check_key(struct socket * sock)290 static int aead_check_key(struct socket *sock)
291 {
292 int err = 0;
293 struct sock *psk;
294 struct alg_sock *pask;
295 struct crypto_aead *tfm;
296 struct sock *sk = sock->sk;
297 struct alg_sock *ask = alg_sk(sk);
298
299 lock_sock(sk);
300 if (!atomic_read(&ask->nokey_refcnt))
301 goto unlock_child;
302
303 psk = ask->parent;
304 pask = alg_sk(ask->parent);
305 tfm = pask->private;
306
307 err = -ENOKEY;
308 lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
309 if (crypto_aead_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
310 goto unlock;
311
312 atomic_dec(&pask->nokey_refcnt);
313 atomic_set(&ask->nokey_refcnt, 0);
314
315 err = 0;
316
317 unlock:
318 release_sock(psk);
319 unlock_child:
320 release_sock(sk);
321
322 return err;
323 }
324
aead_sendmsg_nokey(struct socket * sock,struct msghdr * msg,size_t size)325 static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
326 size_t size)
327 {
328 int err;
329
330 err = aead_check_key(sock);
331 if (err)
332 return err;
333
334 return aead_sendmsg(sock, msg, size);
335 }
336
aead_recvmsg_nokey(struct socket * sock,struct msghdr * msg,size_t ignored,int flags)337 static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
338 size_t ignored, int flags)
339 {
340 int err;
341
342 err = aead_check_key(sock);
343 if (err)
344 return err;
345
346 return aead_recvmsg(sock, msg, ignored, flags);
347 }
348
349 static struct proto_ops algif_aead_ops_nokey = {
350 .family = PF_ALG,
351
352 .connect = sock_no_connect,
353 .socketpair = sock_no_socketpair,
354 .getname = sock_no_getname,
355 .ioctl = sock_no_ioctl,
356 .listen = sock_no_listen,
357 .shutdown = sock_no_shutdown,
358 .mmap = sock_no_mmap,
359 .bind = sock_no_bind,
360 .accept = sock_no_accept,
361
362 .release = af_alg_release,
363 .sendmsg = aead_sendmsg_nokey,
364 .recvmsg = aead_recvmsg_nokey,
365 .poll = af_alg_poll,
366 };
367
aead_bind(const char * name,u32 type,u32 mask)368 static void *aead_bind(const char *name, u32 type, u32 mask)
369 {
370 return crypto_alloc_aead(name, type, mask);
371 }
372
aead_release(void * private)373 static void aead_release(void *private)
374 {
375 crypto_free_aead(private);
376 }
377
aead_setauthsize(void * private,unsigned int authsize)378 static int aead_setauthsize(void *private, unsigned int authsize)
379 {
380 return crypto_aead_setauthsize(private, authsize);
381 }
382
aead_setkey(void * private,const u8 * key,unsigned int keylen)383 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
384 {
385 return crypto_aead_setkey(private, key, keylen);
386 }
387
aead_sock_destruct(struct sock * sk)388 static void aead_sock_destruct(struct sock *sk)
389 {
390 struct alg_sock *ask = alg_sk(sk);
391 struct af_alg_ctx *ctx = ask->private;
392 struct sock *psk = ask->parent;
393 struct alg_sock *pask = alg_sk(psk);
394 struct crypto_aead *tfm = pask->private;
395 unsigned int ivlen = crypto_aead_ivsize(tfm);
396
397 af_alg_pull_tsgl(sk, ctx->used, NULL);
398 sock_kzfree_s(sk, ctx->iv, ivlen);
399 sock_kfree_s(sk, ctx, ctx->len);
400 af_alg_release_parent(sk);
401 }
402
aead_accept_parent_nokey(void * private,struct sock * sk)403 static int aead_accept_parent_nokey(void *private, struct sock *sk)
404 {
405 struct af_alg_ctx *ctx;
406 struct alg_sock *ask = alg_sk(sk);
407 struct crypto_aead *tfm = private;
408 unsigned int len = sizeof(*ctx);
409 unsigned int ivlen = crypto_aead_ivsize(tfm);
410
411 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
412 if (!ctx)
413 return -ENOMEM;
414 memset(ctx, 0, len);
415
416 ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
417 if (!ctx->iv) {
418 sock_kfree_s(sk, ctx, len);
419 return -ENOMEM;
420 }
421 memset(ctx->iv, 0, ivlen);
422
423 INIT_LIST_HEAD(&ctx->tsgl_list);
424 ctx->len = len;
425 crypto_init_wait(&ctx->wait);
426
427 ask->private = ctx;
428
429 sk->sk_destruct = aead_sock_destruct;
430
431 return 0;
432 }
433
aead_accept_parent(void * private,struct sock * sk)434 static int aead_accept_parent(void *private, struct sock *sk)
435 {
436 struct crypto_aead *tfm = private;
437
438 if (crypto_aead_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
439 return -ENOKEY;
440
441 return aead_accept_parent_nokey(private, sk);
442 }
443
444 static const struct af_alg_type algif_type_aead = {
445 .bind = aead_bind,
446 .release = aead_release,
447 .setkey = aead_setkey,
448 .setauthsize = aead_setauthsize,
449 .accept = aead_accept_parent,
450 .accept_nokey = aead_accept_parent_nokey,
451 .ops = &algif_aead_ops,
452 .ops_nokey = &algif_aead_ops_nokey,
453 .name = "aead",
454 .owner = THIS_MODULE
455 };
456
algif_aead_init(void)457 static int __init algif_aead_init(void)
458 {
459 return af_alg_register_type(&algif_type_aead);
460 }
461
algif_aead_exit(void)462 static void __exit algif_aead_exit(void)
463 {
464 int err = af_alg_unregister_type(&algif_type_aead);
465 BUG_ON(err);
466 }
467
468 module_init(algif_aead_init);
469 module_exit(algif_aead_exit);
470 MODULE_LICENSE("GPL");
471 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
472 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");
473