1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* XTS: as defined in IEEE1619/D16 3 * http://grouper.ieee.org/groups/1619/email/pdf00086.pdf 4 * 5 * Copyright (c) 2007 Rik Snel <rsnel@cube.dyndns.org> 6 * 7 * Based on ecb.c 8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> 9 */ 10 #include <crypto/internal/cipher.h> 11 #include <crypto/internal/skcipher.h> 12 #include <crypto/scatterwalk.h> 13 #include <linux/err.h> 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/scatterlist.h> 18 #include <linux/slab.h> 19 #include <linux/string.h> 20 21 #include <crypto/xts.h> 22 #include <crypto/b128ops.h> 23 #include <crypto/gf128mul.h> 24 25 struct xts_tfm_ctx { 26 struct crypto_skcipher *child; 27 struct crypto_cipher *tweak; 28 }; 29 30 struct xts_instance_ctx { 31 struct crypto_skcipher_spawn spawn; 32 struct crypto_cipher_spawn tweak_spawn; 33 }; 34 35 struct xts_request_ctx { 36 le128 t; 37 struct scatterlist *tail; 38 struct scatterlist sg[2]; 39 struct skcipher_request subreq; 40 }; 41 42 static int xts_setkey(struct crypto_skcipher *parent, const u8 *key, 43 unsigned int keylen) 44 { 45 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(parent); 46 struct crypto_skcipher *child; 47 struct crypto_cipher *tweak; 48 int err; 49 50 err = xts_verify_key(parent, key, keylen); 51 if (err) 52 return err; 53 54 keylen /= 2; 55 56 /* we need two cipher instances: one to compute the initial 'tweak' 57 * by encrypting the IV (usually the 'plain' iv) and the other 58 * one to encrypt and decrypt the data */ 59 60 /* tweak cipher, uses Key2 i.e. the second half of *key */ 61 tweak = ctx->tweak; 62 crypto_cipher_clear_flags(tweak, CRYPTO_TFM_REQ_MASK); 63 crypto_cipher_set_flags(tweak, crypto_skcipher_get_flags(parent) & 64 CRYPTO_TFM_REQ_MASK); 65 err = crypto_cipher_setkey(tweak, key + keylen, keylen); 66 if (err) 67 return err; 68 69 /* data cipher, uses Key1 i.e. the first half of *key */ 70 child = ctx->child; 71 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 72 crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) & 73 CRYPTO_TFM_REQ_MASK); 74 return crypto_skcipher_setkey(child, key, keylen); 75 } 76 77 /* 78 * We compute the tweak masks twice (both before and after the ECB encryption or 79 * decryption) to avoid having to allocate a temporary buffer and/or make 80 * multiple calls to the 'ecb(..)' instance, which usually would be slower than 81 * just doing the gf128mul_x_ble() calls again. 82 */ 83 static int xts_xor_tweak(struct skcipher_request *req, bool second_pass, 84 bool enc) 85 { 86 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 87 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 88 const bool cts = (req->cryptlen % XTS_BLOCK_SIZE); 89 const int bs = XTS_BLOCK_SIZE; 90 struct skcipher_walk w; 91 le128 t = rctx->t; 92 int err; 93 94 if (second_pass) { 95 req = &rctx->subreq; 96 /* set to our TFM to enforce correct alignment: */ 97 skcipher_request_set_tfm(req, tfm); 98 } 99 err = skcipher_walk_virt(&w, req, false); 100 101 while (w.nbytes) { 102 unsigned int avail = w.nbytes; 103 const le128 *wsrc; 104 le128 *wdst; 105 106 wsrc = w.src.virt.addr; 107 wdst = w.dst.virt.addr; 108 109 do { 110 if (unlikely(cts) && 111 w.total - w.nbytes + avail < 2 * XTS_BLOCK_SIZE) { 112 if (!enc) { 113 if (second_pass) 114 rctx->t = t; 115 gf128mul_x_ble(&t, &t); 116 } 117 le128_xor(wdst, &t, wsrc); 118 if (enc && second_pass) 119 gf128mul_x_ble(&rctx->t, &t); 120 skcipher_walk_done(&w, avail - bs); 121 return 0; 122 } 123 124 le128_xor(wdst++, &t, wsrc++); 125 gf128mul_x_ble(&t, &t); 126 } while ((avail -= bs) >= bs); 127 128 err = skcipher_walk_done(&w, avail); 129 } 130 131 return err; 132 } 133 134 static int xts_xor_tweak_pre(struct skcipher_request *req, bool enc) 135 { 136 return xts_xor_tweak(req, false, enc); 137 } 138 139 static int xts_xor_tweak_post(struct skcipher_request *req, bool enc) 140 { 141 return xts_xor_tweak(req, true, enc); 142 } 143 144 static void xts_cts_done(void *data, int err) 145 { 146 struct skcipher_request *req = data; 147 le128 b; 148 149 if (!err) { 150 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 151 152 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); 153 le128_xor(&b, &rctx->t, &b); 154 scatterwalk_map_and_copy(&b, rctx->tail, 0, XTS_BLOCK_SIZE, 1); 155 } 156 157 skcipher_request_complete(req, err); 158 } 159 160 static int xts_cts_final(struct skcipher_request *req, 161 int (*crypt)(struct skcipher_request *req)) 162 { 163 const struct xts_tfm_ctx *ctx = 164 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 165 int offset = req->cryptlen & ~(XTS_BLOCK_SIZE - 1); 166 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 167 struct skcipher_request *subreq = &rctx->subreq; 168 int tail = req->cryptlen % XTS_BLOCK_SIZE; 169 le128 b[2]; 170 int err; 171 172 rctx->tail = scatterwalk_ffwd(rctx->sg, req->dst, 173 offset - XTS_BLOCK_SIZE); 174 175 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); 176 b[1] = b[0]; 177 scatterwalk_map_and_copy(b, req->src, offset, tail, 0); 178 179 le128_xor(b, &rctx->t, b); 180 181 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE + tail, 1); 182 183 skcipher_request_set_tfm(subreq, ctx->child); 184 skcipher_request_set_callback(subreq, req->base.flags, xts_cts_done, 185 req); 186 skcipher_request_set_crypt(subreq, rctx->tail, rctx->tail, 187 XTS_BLOCK_SIZE, NULL); 188 189 err = crypt(subreq); 190 if (err) 191 return err; 192 193 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 0); 194 le128_xor(b, &rctx->t, b); 195 scatterwalk_map_and_copy(b, rctx->tail, 0, XTS_BLOCK_SIZE, 1); 196 197 return 0; 198 } 199 200 static void xts_encrypt_done(void *data, int err) 201 { 202 struct skcipher_request *req = data; 203 204 if (!err) { 205 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 206 207 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG; 208 err = xts_xor_tweak_post(req, true); 209 210 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) { 211 err = xts_cts_final(req, crypto_skcipher_encrypt); 212 if (err == -EINPROGRESS || err == -EBUSY) 213 return; 214 } 215 } 216 217 skcipher_request_complete(req, err); 218 } 219 220 static void xts_decrypt_done(void *data, int err) 221 { 222 struct skcipher_request *req = data; 223 224 if (!err) { 225 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 226 227 rctx->subreq.base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG; 228 err = xts_xor_tweak_post(req, false); 229 230 if (!err && unlikely(req->cryptlen % XTS_BLOCK_SIZE)) { 231 err = xts_cts_final(req, crypto_skcipher_decrypt); 232 if (err == -EINPROGRESS || err == -EBUSY) 233 return; 234 } 235 } 236 237 skcipher_request_complete(req, err); 238 } 239 240 static int xts_init_crypt(struct skcipher_request *req, 241 crypto_completion_t compl) 242 { 243 const struct xts_tfm_ctx *ctx = 244 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 245 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 246 struct skcipher_request *subreq = &rctx->subreq; 247 248 if (req->cryptlen < XTS_BLOCK_SIZE) 249 return -EINVAL; 250 251 skcipher_request_set_tfm(subreq, ctx->child); 252 skcipher_request_set_callback(subreq, req->base.flags, compl, req); 253 skcipher_request_set_crypt(subreq, req->dst, req->dst, 254 req->cryptlen & ~(XTS_BLOCK_SIZE - 1), NULL); 255 256 /* calculate first value of T */ 257 crypto_cipher_encrypt_one(ctx->tweak, (u8 *)&rctx->t, req->iv); 258 259 return 0; 260 } 261 262 static int xts_encrypt(struct skcipher_request *req) 263 { 264 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 265 struct skcipher_request *subreq = &rctx->subreq; 266 int err; 267 268 err = xts_init_crypt(req, xts_encrypt_done) ?: 269 xts_xor_tweak_pre(req, true) ?: 270 crypto_skcipher_encrypt(subreq) ?: 271 xts_xor_tweak_post(req, true); 272 273 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0)) 274 return err; 275 276 return xts_cts_final(req, crypto_skcipher_encrypt); 277 } 278 279 static int xts_decrypt(struct skcipher_request *req) 280 { 281 struct xts_request_ctx *rctx = skcipher_request_ctx(req); 282 struct skcipher_request *subreq = &rctx->subreq; 283 int err; 284 285 err = xts_init_crypt(req, xts_decrypt_done) ?: 286 xts_xor_tweak_pre(req, false) ?: 287 crypto_skcipher_decrypt(subreq) ?: 288 xts_xor_tweak_post(req, false); 289 290 if (err || likely((req->cryptlen % XTS_BLOCK_SIZE) == 0)) 291 return err; 292 293 return xts_cts_final(req, crypto_skcipher_decrypt); 294 } 295 296 static int xts_init_tfm(struct crypto_skcipher *tfm) 297 { 298 struct skcipher_instance *inst = skcipher_alg_instance(tfm); 299 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst); 300 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 301 struct crypto_skcipher *child; 302 struct crypto_cipher *tweak; 303 304 child = crypto_spawn_skcipher(&ictx->spawn); 305 if (IS_ERR(child)) 306 return PTR_ERR(child); 307 308 ctx->child = child; 309 310 tweak = crypto_spawn_cipher(&ictx->tweak_spawn); 311 if (IS_ERR(tweak)) { 312 crypto_free_skcipher(ctx->child); 313 return PTR_ERR(tweak); 314 } 315 316 ctx->tweak = tweak; 317 318 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + 319 sizeof(struct xts_request_ctx)); 320 321 return 0; 322 } 323 324 static void xts_exit_tfm(struct crypto_skcipher *tfm) 325 { 326 struct xts_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 327 328 crypto_free_skcipher(ctx->child); 329 crypto_free_cipher(ctx->tweak); 330 } 331 332 static void xts_free_instance(struct skcipher_instance *inst) 333 { 334 struct xts_instance_ctx *ictx = skcipher_instance_ctx(inst); 335 336 crypto_drop_skcipher(&ictx->spawn); 337 crypto_drop_cipher(&ictx->tweak_spawn); 338 kfree(inst); 339 } 340 341 static int xts_create(struct crypto_template *tmpl, struct rtattr **tb) 342 { 343 struct skcipher_alg_common *alg; 344 char name[CRYPTO_MAX_ALG_NAME]; 345 struct skcipher_instance *inst; 346 struct xts_instance_ctx *ctx; 347 const char *cipher_name; 348 u32 mask; 349 int err; 350 351 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SKCIPHER, &mask); 352 if (err) 353 return err; 354 355 cipher_name = crypto_attr_alg_name(tb[1]); 356 if (IS_ERR(cipher_name)) 357 return PTR_ERR(cipher_name); 358 359 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 360 if (!inst) 361 return -ENOMEM; 362 363 ctx = skcipher_instance_ctx(inst); 364 365 err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), 366 cipher_name, 0, mask); 367 if (err == -ENOENT && memcmp(cipher_name, "ecb(", 4)) { 368 err = -ENAMETOOLONG; 369 if (snprintf(name, CRYPTO_MAX_ALG_NAME, "ecb(%s)", 370 cipher_name) >= CRYPTO_MAX_ALG_NAME) 371 goto err_free_inst; 372 373 err = crypto_grab_skcipher(&ctx->spawn, 374 skcipher_crypto_instance(inst), 375 name, 0, mask); 376 } 377 378 if (err) 379 goto err_free_inst; 380 381 alg = crypto_spawn_skcipher_alg_common(&ctx->spawn); 382 383 err = -EINVAL; 384 if (alg->base.cra_blocksize != XTS_BLOCK_SIZE) 385 goto err_free_inst; 386 387 if (alg->ivsize) 388 goto err_free_inst; 389 390 err = crypto_inst_setname(skcipher_crypto_instance(inst), "xts", 391 &alg->base); 392 if (err) 393 goto err_free_inst; 394 395 err = -EINVAL; 396 cipher_name = alg->base.cra_name; 397 398 /* Alas we screwed up the naming so we have to mangle the 399 * cipher name. 400 */ 401 if (!memcmp(cipher_name, "ecb(", 4)) { 402 int len; 403 404 len = strscpy(name, cipher_name + 4); 405 if (len < 2) 406 goto err_free_inst; 407 408 if (name[len - 1] != ')') 409 goto err_free_inst; 410 411 name[len - 1] = 0; 412 413 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, 414 "xts(%s)", name) >= CRYPTO_MAX_ALG_NAME) { 415 err = -ENAMETOOLONG; 416 goto err_free_inst; 417 } 418 } else 419 goto err_free_inst; 420 421 err = crypto_grab_cipher(&ctx->tweak_spawn, 422 skcipher_crypto_instance(inst), name, 0, mask); 423 if (err) 424 goto err_free_inst; 425 426 inst->alg.base.cra_priority = alg->base.cra_priority; 427 inst->alg.base.cra_blocksize = XTS_BLOCK_SIZE; 428 inst->alg.base.cra_alignmask = alg->base.cra_alignmask | 429 (__alignof__(u64) - 1); 430 431 inst->alg.ivsize = XTS_BLOCK_SIZE; 432 inst->alg.min_keysize = alg->min_keysize * 2; 433 inst->alg.max_keysize = alg->max_keysize * 2; 434 435 inst->alg.base.cra_ctxsize = sizeof(struct xts_tfm_ctx); 436 437 inst->alg.init = xts_init_tfm; 438 inst->alg.exit = xts_exit_tfm; 439 440 inst->alg.setkey = xts_setkey; 441 inst->alg.encrypt = xts_encrypt; 442 inst->alg.decrypt = xts_decrypt; 443 444 inst->free = xts_free_instance; 445 446 err = skcipher_register_instance(tmpl, inst); 447 if (err) { 448 err_free_inst: 449 xts_free_instance(inst); 450 } 451 return err; 452 } 453 454 static struct crypto_template xts_tmpl = { 455 .name = "xts", 456 .create = xts_create, 457 .module = THIS_MODULE, 458 }; 459 460 static int __init xts_module_init(void) 461 { 462 return crypto_register_template(&xts_tmpl); 463 } 464 465 static void __exit xts_module_exit(void) 466 { 467 crypto_unregister_template(&xts_tmpl); 468 } 469 470 module_init(xts_module_init); 471 module_exit(xts_module_exit); 472 473 MODULE_LICENSE("GPL"); 474 MODULE_DESCRIPTION("XTS block cipher mode"); 475 MODULE_ALIAS_CRYPTO("xts"); 476 MODULE_IMPORT_NS("CRYPTO_INTERNAL"); 477 MODULE_SOFTDEP("pre: ecb"); 478