1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Shared KUnit test cases for AEAD algorithms, including a benchmark 4 * 5 * Copyright 2026 Google LLC 6 */ 7 8 /* 9 * This file implements KUnit test cases shared by the different KUnit test 10 * suites for Authenticated Encryption with Associated Data (AEAD) algorithms. 11 * 12 * Test suites including this file must #define the following: 13 * 14 * Data structs: 15 * - AEAD_KEY: name of key struct 16 * - AEAD_CTX: name of context for incremental computation 17 * 18 * Constants: 19 * - AEAD_VALID_KEY_LENS: array of all valid key lengths in bytes 20 * - AEAD_VALID_NONCE_LENS: array of all valid nonce lengths in bytes 21 * - AEAD_VALID_TAG_LENS: array of all valid authtag lengths in bytes 22 * - AEAD_MAX_KEY_LEN: max key length in bytes (assumed to fit on stack) 23 * - AEAD_MAX_NONCE_LEN: max nonce length in bytes (assumed to fit on stack) 24 * - AEAD_MAX_TAG_LEN: max authtag length in bytes (assumed to fit on stack) 25 * - AEAD_MONTE_CARLO_CHECKSUM: checksum of a deterministically generated series 26 * of (ciphertext, authtag) pairs (see test_aead_monte_carlo()) 27 * 28 * Functions: 29 * - AEAD_PREPAREKEY: key preparation 30 * - AEAD_ENCRYPT and AEAD_DECRYPT: one-shot encryption and decryption 31 * - AEAD_INIT, AEAD_AUTH_UPDATE, AEAD_ENCRYPT_UPDATE, AEAD_ENCRYPT_FINAL, 32 * AEAD_DECRYPT_UPDATE, AEAD_DECRYPT_FINAL: functions for incremental 33 * encryption and decryption 34 * 35 * Function prototypes and their behavior must match the AES-CCM API. 36 */ 37 38 #include <crypto/blake2s.h> 39 #include <kunit/run-in-irq-context.h> 40 #include <kunit/test.h> 41 #include <linux/ktime.h> 42 #include <linux/preempt.h> 43 #include "test-utils.h" 44 45 /* 46 * Allocate a KUnit-managed struct AEAD_KEY and prepare it with a random key, 47 * using a random key length and random authentication tag length. 48 */ 49 static struct AEAD_KEY *aead_alloc_random_key(struct kunit *test, 50 size_t *tag_len_ret) 51 { 52 size_t key_len = 53 AEAD_VALID_KEY_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; 54 size_t tag_len = 55 AEAD_VALID_TAG_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; 56 u8 raw_key[AEAD_MAX_KEY_LEN]; 57 struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); 58 int err; 59 60 rand_bytes(raw_key, key_len); 61 err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); 62 KUNIT_ASSERT_EQ(test, 0, err); 63 *tag_len_ret = tag_len; 64 return key; 65 } 66 67 /* 68 * Allocate a KUnit-managed slab buffer of length @len bytes and initialize it 69 * with random data. 70 */ 71 static u8 *aead_alloc_random_data(struct kunit *test, size_t len) 72 { 73 u8 *buf = alloc_buf(test, len); 74 75 rand_bytes(buf, len); 76 return buf; 77 } 78 79 /* 80 * Allocate a KUnit-managed guarded buffer of length @len bytes and initialize 81 * it with random data. 82 */ 83 static u8 *aead_alloc_random_data_guarded(struct kunit *test, size_t len) 84 { 85 u8 *buf = alloc_guarded_buf(test, len); 86 87 rand_bytes(buf, len); 88 return buf; 89 } 90 91 /* Process the given associated data using a random incremental strategy. */ 92 static size_t aead_auth_incrementally(struct AEAD_CTX *ctx, const u8 *ad, 93 size_t ad_len) 94 { 95 size_t num_parts = 0; 96 size_t pos = 0; 97 98 while (rand_bool()) { 99 size_t part_len = rand_length(ad_len - pos); 100 101 AEAD_AUTH_UPDATE(ctx, &ad[pos], part_len); 102 pos += part_len; 103 num_parts++; 104 } 105 if (pos < ad_len || rand_bool()) { 106 AEAD_AUTH_UPDATE(ctx, &ad[pos], ad_len - pos); 107 num_parts++; 108 } 109 return num_parts; 110 } 111 112 /* Process the given en/decrypted data using a random incremental strategy. */ 113 static size_t aead_crypt_incrementally(struct AEAD_CTX *ctx, u8 *dst, 114 const u8 *src, size_t data_len, bool enc) 115 { 116 size_t num_parts = 0; 117 size_t pos = 0; 118 119 while (rand_bool()) { 120 size_t part_len = rand_length(data_len - pos); 121 122 if (enc) 123 AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], 124 part_len); 125 else 126 AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], 127 part_len); 128 pos += part_len; 129 num_parts++; 130 } 131 if (pos < data_len || rand_bool()) { 132 if (enc) 133 AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos], 134 data_len - pos); 135 else 136 AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos], 137 data_len - pos); 138 num_parts++; 139 } 140 return num_parts; 141 } 142 143 struct aead_incremental_info { 144 size_t num_data_parts; 145 size_t num_ad_parts; 146 }; 147 148 static const char *aead_incr_info_str(struct kunit *test, 149 const struct aead_incremental_info *info) 150 { 151 const size_t max_str_len = 64; 152 char *str = alloc_buf(test, max_str_len); 153 154 snprintf(str, max_str_len, "num_data_parts=%zu num_ad_parts=%zu", 155 info->num_data_parts, info->num_ad_parts); 156 return str; 157 } 158 159 /* 160 * Encrypt data using a random incremental strategy. 161 * Return information about the incremental strategy used. 162 */ 163 static struct aead_incremental_info 164 aead_encrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, 165 const u8 *src, size_t data_len, u8 *tag, 166 const u8 *ad, size_t ad_len, const u8 *nonce, 167 size_t nonce_len, const struct AEAD_KEY *key) 168 { 169 struct aead_incremental_info info; 170 int err; 171 172 err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); 173 KUNIT_ASSERT_EQ(test, 0, err); 174 info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); 175 info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, 176 /* enc= */ true); 177 AEAD_ENCRYPT_FINAL(ctx, tag); 178 KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), 179 "encrypt_final didn't zeroize context"); 180 return info; 181 } 182 183 /* 184 * Decrypt authentic data using a random incremental strategy. 185 * Return information about the incremental strategy used. 186 */ 187 static struct aead_incremental_info 188 aead_decrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst, 189 const u8 *src, size_t data_len, const u8 *tag, 190 const u8 *ad, size_t ad_len, const u8 *nonce, 191 size_t nonce_len, const struct AEAD_KEY *key) 192 { 193 struct aead_incremental_info info; 194 int err; 195 196 err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key); 197 KUNIT_ASSERT_EQ(test, 0, err); 198 info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len); 199 info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len, 200 /* enc= */ false); 201 err = AEAD_DECRYPT_FINAL(ctx, tag); 202 KUNIT_ASSERT_EQ(test, 0, err); 203 KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)), 204 "decrypt_final didn't zeroize context"); 205 return info; 206 } 207 208 /* Return true if key_len is declared to be a valid key length. */ 209 static bool aead_is_key_len_expected_valid(size_t key_len) 210 { 211 for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_KEY_LENS); i++) { 212 if (AEAD_VALID_KEY_LENS[i] == key_len) 213 return true; 214 } 215 return false; 216 } 217 218 /* Return true if nonce_len is declared to be a valid nonce length. */ 219 static bool aead_is_nonce_len_expected_valid(size_t nonce_len) 220 { 221 for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_NONCE_LENS); i++) { 222 if (AEAD_VALID_NONCE_LENS[i] == nonce_len) 223 return true; 224 } 225 return false; 226 } 227 228 /* Return true if tag_len is declared to be a valid tag length. */ 229 static bool aead_is_tag_len_expected_valid(size_t tag_len) 230 { 231 for (size_t i = 0; i < ARRAY_SIZE(AEAD_VALID_TAG_LENS); i++) { 232 if (AEAD_VALID_TAG_LENS[i] == tag_len) 233 return true; 234 } 235 return false; 236 } 237 238 struct aead_basic_validation_test_ctx { 239 struct AEAD_KEY key; 240 struct AEAD_CTX ctx; 241 u8 *raw_key_buf_end; 242 u8 *nonce_buf_end; 243 u8 *tag_buf_end; 244 u8 pt[64]; /* plaintext */ 245 u8 ct[64]; /* ciphertext */ 246 u8 decrypted[64]; 247 u8 ad[16]; /* associated data */ 248 u8 *unused_buf; 249 size_t data_len; 250 size_t ad_len; 251 }; 252 253 static struct aead_basic_validation_test_ctx * 254 aead_alloc_basic_validation_test_ctx(struct kunit *test) 255 { 256 struct aead_basic_validation_test_ctx *ctx = 257 alloc_buf(test, sizeof(*ctx)); 258 259 memset(ctx, 0, sizeof(*ctx)); 260 ctx->raw_key_buf_end = 261 aead_alloc_random_data_guarded(test, AEAD_MAX_KEY_LEN) + 262 AEAD_MAX_KEY_LEN; 263 ctx->nonce_buf_end = 264 aead_alloc_random_data_guarded(test, AEAD_MAX_NONCE_LEN) + 265 AEAD_MAX_NONCE_LEN; 266 ctx->tag_buf_end = 267 aead_alloc_random_data_guarded(test, AEAD_MAX_TAG_LEN) + 268 AEAD_MAX_TAG_LEN; 269 270 /* 271 * A pointer to this buffer is passed when passing a length that is 272 * expected to be invalid. It should never actually be accessed. 273 */ 274 ctx->unused_buf = 275 alloc_buf(test, max3(AEAD_MAX_KEY_LEN, AEAD_MAX_NONCE_LEN, 276 AEAD_MAX_TAG_LEN)); 277 278 ctx->data_len = sizeof(ctx->pt); 279 ctx->ad_len = sizeof(ctx->ad); 280 return ctx; 281 } 282 283 /* 284 * Given an expected-valid key_len, nonce_len, and tag_len, verify round-trip 285 * encryption and decryption with them. Use guarded buffers for each of the raw 286 * key, nonce, and tag to detect any buffer overruns in them. Also, verify that 287 * every byte of the tag is actually checked. 288 */ 289 static void aead_do_basic_checks(struct kunit *test, 290 struct aead_basic_validation_test_ctx *ctx, 291 size_t key_len, size_t nonce_len, 292 size_t tag_len) 293 { 294 /* Set up exact-size guarded buffers for (raw_key, nonce, tag). */ 295 const u8 *raw_key = ctx->raw_key_buf_end - key_len; 296 const u8 *nonce = ctx->nonce_buf_end - nonce_len; 297 u8 *tag = ctx->tag_buf_end - tag_len; 298 int err; 299 300 /* Key preparation should succeed. */ 301 err = AEAD_PREPAREKEY(&ctx->key, raw_key, key_len, tag_len); 302 KUNIT_ASSERT_EQ_MSG(test, 0, err, 303 "key_len=%zu, tag_len=%zu wasn't accepted", key_len, 304 tag_len); 305 306 /* Encryption should succeed. */ 307 err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, tag, ctx->ad, 308 ctx->ad_len, nonce, nonce_len, &ctx->key); 309 KUNIT_ASSERT_EQ_MSG( 310 test, 0, err, 311 "Encryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", 312 key_len, nonce_len, tag_len); 313 314 /* Decryption should succeed and give the original data. */ 315 err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, ctx->ad, 316 ctx->ad_len, nonce, nonce_len, &ctx->key); 317 KUNIT_ASSERT_EQ_MSG( 318 test, 0, err, 319 "Decryption failed with key_len=%zu, nonce_len=%zu, tag_len=%zu", 320 key_len, nonce_len, tag_len); 321 KUNIT_ASSERT_MEMEQ_MSG( 322 test, ctx->pt, ctx->decrypted, ctx->data_len, 323 "Decryption gave wrong output with key_len=%zu, nonce_len=%zu, tag_len=%zu", 324 key_len, nonce_len, tag_len); 325 326 /* 327 * Every byte of the tag should actually be checked. 328 * And on authentication failure, the dst buffer should be cleared. 329 */ 330 for (size_t i = 0; i < tag_len; i++) { 331 memset(ctx->decrypted, 0xff, ctx->data_len); 332 tag[i] ^= 1; 333 err = AEAD_DECRYPT(ctx->decrypted, ctx->ct, ctx->data_len, tag, 334 ctx->ad, ctx->ad_len, nonce, nonce_len, 335 &ctx->key); 336 KUNIT_ASSERT_EQ_MSG( 337 test, -EBADMSG, err, 338 "Decryption with bad auth tag with key_len=%zu, nonce_len=%zu, tag_len=%zu didn't fail with -EBADMSG", 339 key_len, nonce_len, tag_len); 340 KUNIT_ASSERT_TRUE_MSG( 341 test, mem_is_zero(ctx->decrypted, ctx->data_len), 342 "dst wasn't cleared on authentication failure"); 343 tag[i] ^= 1; 344 } 345 } 346 347 /* Verify that the given expected-invalid key_len is actually rejected. */ 348 static void 349 aead_verify_invalid_key_len(struct kunit *test, 350 struct aead_basic_validation_test_ctx *ctx, 351 size_t key_len) 352 { 353 int err; 354 355 /* 356 * The preparekey function should reject the key_len. It should do so 357 * before writing to the key struct. 358 */ 359 memset(&ctx->key, 0, sizeof(ctx->key)); 360 err = AEAD_PREPAREKEY(&ctx->key, ctx->unused_buf, key_len, 361 AEAD_MAX_TAG_LEN); 362 KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, 363 "key_len=%zu wasn't rejected with -EINVAL", 364 key_len); 365 KUNIT_ASSERT_TRUE_MSG( 366 test, mem_is_zero(&ctx->key, sizeof(ctx->key)), 367 "Key struct was written to before length validation"); 368 } 369 370 /* 371 * Test that every valid key length is accepted and basic checks pass with it, 372 * and test that invalid key lengths are rejected. 373 */ 374 static void test_aead_all_key_lens(struct kunit *test) 375 { 376 struct aead_basic_validation_test_ctx *ctx = 377 aead_alloc_basic_validation_test_ctx(test); 378 379 for (size_t key_len = 0; key_len <= AEAD_MAX_KEY_LEN; key_len++) { 380 if (aead_is_key_len_expected_valid(key_len)) 381 aead_do_basic_checks(test, ctx, key_len, 382 AEAD_MAX_NONCE_LEN, 383 AEAD_MAX_TAG_LEN); 384 else 385 aead_verify_invalid_key_len(test, ctx, key_len); 386 } 387 aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN + 1); 388 aead_verify_invalid_key_len(test, ctx, AEAD_MAX_KEY_LEN * 2); 389 aead_verify_invalid_key_len(test, ctx, U32_MAX); 390 aead_verify_invalid_key_len(test, ctx, SIZE_MAX); 391 } 392 393 /* Verify that the given expected-invalid nonce_len is actually rejected. */ 394 static void 395 aead_verify_invalid_nonce_len(struct kunit *test, 396 struct aead_basic_validation_test_ctx *ctx, 397 size_t nonce_len) 398 { 399 static const u8 raw_key[AEAD_MAX_KEY_LEN]; 400 int err; 401 402 /* Key preparation should succeed, as nonce_len isn't given yet. */ 403 err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), 404 AEAD_MAX_TAG_LEN); 405 KUNIT_ASSERT_EQ(test, 0, err); 406 407 /* The init function should reject the nonce_len. */ 408 memset(&ctx->ctx, 0, sizeof(ctx->ctx)); 409 err = AEAD_INIT(&ctx->ctx, ctx->data_len, ctx->ad_len, ctx->unused_buf, 410 nonce_len, &ctx->key); 411 KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, 412 "nonce_len=%zu wasn't rejected with -EINVAL (init)", 413 nonce_len); 414 KUNIT_ASSERT_TRUE_MSG( 415 test, mem_is_zero(&ctx->ctx, sizeof(ctx->ctx)), 416 "Context struct was written to before length validation"); 417 418 /* The encrypt function should reject the nonce_len. */ 419 err = AEAD_ENCRYPT(ctx->ct, ctx->pt, ctx->data_len, ctx->unused_buf, 420 ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, 421 &ctx->key); 422 KUNIT_ASSERT_EQ_MSG( 423 test, -EINVAL, err, 424 "nonce_len=%zu wasn't rejected with -EINVAL (encrypt)", 425 nonce_len); 426 427 /* The decrypt function should reject the nonce_len. */ 428 err = AEAD_DECRYPT(ctx->pt, ctx->ct, ctx->data_len, ctx->unused_buf, 429 ctx->ad, ctx->ad_len, ctx->unused_buf, nonce_len, 430 &ctx->key); 431 KUNIT_ASSERT_EQ_MSG( 432 test, -EINVAL, err, 433 "nonce_len=%zu wasn't rejected with -EINVAL (decrypt)", 434 nonce_len); 435 } 436 437 /* 438 * Test that every valid nonce length is accepted and basic checks pass with it, 439 * and test that invalid nonce lengths are rejected. 440 */ 441 static void test_aead_all_nonce_lens(struct kunit *test) 442 { 443 struct aead_basic_validation_test_ctx *ctx = 444 aead_alloc_basic_validation_test_ctx(test); 445 446 for (size_t nonce_len = 0; nonce_len <= AEAD_MAX_NONCE_LEN; 447 nonce_len++) { 448 if (aead_is_nonce_len_expected_valid(nonce_len)) 449 aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, 450 nonce_len, AEAD_MAX_TAG_LEN); 451 else 452 aead_verify_invalid_nonce_len(test, ctx, nonce_len); 453 } 454 aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN + 1); 455 aead_verify_invalid_nonce_len(test, ctx, AEAD_MAX_NONCE_LEN * 2); 456 aead_verify_invalid_nonce_len(test, ctx, U32_MAX); 457 aead_verify_invalid_nonce_len(test, ctx, SIZE_MAX); 458 } 459 460 /* Verify that the given expected-invalid tag_len is actually rejected. */ 461 static void 462 aead_verify_invalid_tag_len(struct kunit *test, 463 struct aead_basic_validation_test_ctx *ctx, 464 size_t tag_len) 465 { 466 static const u8 raw_key[AEAD_MAX_KEY_LEN]; 467 int err; 468 469 /* 470 * The preparekey function should reject the tag_len. It should do so 471 * before writing to the key struct. 472 */ 473 memset(&ctx->key, 0, sizeof(ctx->key)); 474 err = AEAD_PREPAREKEY(&ctx->key, raw_key, sizeof(raw_key), tag_len); 475 KUNIT_ASSERT_EQ_MSG(test, -EINVAL, err, 476 "tag_len=%zu wasn't rejected with -EINVAL", 477 tag_len); 478 KUNIT_ASSERT_TRUE_MSG( 479 test, mem_is_zero(&ctx->key, sizeof(ctx->key)), 480 "Key struct was written to before length validation"); 481 } 482 483 /* 484 * Test that every valid authentication tag length is accepted and basic checks 485 * pass with it, and test that invalid authentication tag lengths are rejected. 486 */ 487 static void test_aead_all_tag_lens(struct kunit *test) 488 { 489 struct aead_basic_validation_test_ctx *ctx = 490 aead_alloc_basic_validation_test_ctx(test); 491 492 for (size_t tag_len = 0; tag_len <= AEAD_MAX_TAG_LEN; tag_len++) { 493 if (aead_is_tag_len_expected_valid(tag_len)) 494 aead_do_basic_checks(test, ctx, AEAD_MAX_KEY_LEN, 495 AEAD_MAX_NONCE_LEN, tag_len); 496 else 497 aead_verify_invalid_tag_len(test, ctx, tag_len); 498 } 499 aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN + 1); 500 aead_verify_invalid_tag_len(test, ctx, AEAD_MAX_TAG_LEN * 2); 501 aead_verify_invalid_tag_len(test, ctx, U32_MAX); 502 aead_verify_invalid_tag_len(test, ctx, SIZE_MAX); 503 } 504 505 /* 506 * Test that one-shot encryption and decryption are consistent with each other 507 * and with incremental encryption and decryption. 508 */ 509 static void test_aead_incremental_updates(struct kunit *test) 510 { 511 const size_t max_data_len = 1024; 512 const size_t max_ad_len = 512; 513 const size_t nonce_len = AEAD_MAX_NONCE_LEN; 514 size_t tag_len; 515 struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); 516 struct AEAD_CTX *ctx = alloc_buf(test, sizeof(*ctx)); 517 u8 *pt = aead_alloc_random_data(test, max_data_len); 518 u8 *ad = aead_alloc_random_data(test, max_ad_len); 519 u8 *nonce = aead_alloc_random_data(test, nonce_len); 520 u8 *ct = alloc_buf(test, max_data_len); 521 u8 *ct2 = alloc_buf(test, max_data_len); 522 u8 *decrypted = alloc_buf(test, max_data_len); 523 u8 *tag = alloc_buf(test, tag_len); 524 u8 *tag2 = alloc_buf(test, tag_len); 525 int err; 526 527 for (int i = 0; i < 500; i++) { 528 /* Select the lengths to test. */ 529 const size_t data_len = rand_length(max_data_len); 530 const size_t ad_len = rand_length(max_ad_len); 531 struct aead_incremental_info incr_info; 532 533 /* Try one-shot encryption and decryption. */ 534 err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, 535 nonce_len, key); 536 KUNIT_ASSERT_EQ(test, 0, err); 537 err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, 538 nonce, nonce_len, key); 539 KUNIT_ASSERT_EQ(test, 0, err); 540 KUNIT_ASSERT_MEMEQ_MSG( 541 test, pt, decrypted, data_len, 542 "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", 543 data_len, ad_len); 544 545 /* Try incremental encryption and decryption. */ 546 incr_info = aead_encrypt_incrementally(test, ctx, ct2, pt, 547 data_len, tag2, ad, 548 ad_len, nonce, nonce_len, 549 key); 550 KUNIT_ASSERT_MEMEQ_MSG( 551 test, ct, ct2, data_len, 552 "One-shot and incremental encryption gave different ciphertexts; data_len=%zu ad_len=%zu %s", 553 data_len, ad_len, aead_incr_info_str(test, &incr_info)); 554 KUNIT_ASSERT_MEMEQ_MSG( 555 test, tag, tag2, tag_len, 556 "One-shot and incremental encryption gave different auth tags; data_len=%zu ad_len=%zu %s", 557 data_len, ad_len, aead_incr_info_str(test, &incr_info)); 558 incr_info = aead_decrypt_incrementally(test, ctx, decrypted, 559 ct2, data_len, tag2, ad, 560 ad_len, nonce, nonce_len, 561 key); 562 KUNIT_ASSERT_MEMEQ_MSG( 563 test, pt, decrypted, data_len, 564 "One-shot and incremental decryption gave different plaintexts; data_len=%zu ad_len=%zu %s", 565 data_len, ad_len, aead_incr_info_str(test, &incr_info)); 566 } 567 } 568 569 /* 570 * Test using guarded buffers for the plaintext, ciphertext, and associated 571 * data. This detects out-of-bounds accesses, even in assembly code. 572 * 573 * Note: other test cases cover overrun of raw_key, nonce, and tag. 574 */ 575 static void test_aead_data_buffer_overruns(struct kunit *test) 576 { 577 const size_t max_data_len = 1024; 578 const size_t max_ad_len = 512; 579 const size_t nonce_len = AEAD_MAX_NONCE_LEN; 580 size_t tag_len; 581 struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); 582 const u8 *nonce = aead_alloc_random_data(test, nonce_len); 583 const u8 *pt_end = aead_alloc_random_data_guarded(test, max_data_len) + 584 max_data_len; 585 const u8 *ad_end = 586 aead_alloc_random_data_guarded(test, max_ad_len) + max_ad_len; 587 u8 *ct_end = alloc_guarded_buf(test, max_data_len) + max_data_len; 588 u8 *decrypted_end = 589 alloc_guarded_buf(test, max_data_len) + max_data_len; 590 u8 *tag = alloc_buf(test, tag_len); 591 592 for (int i = 0; i < 200; i++) { 593 /* Select the lengths to test. */ 594 const size_t data_len = rand_length(max_data_len); 595 const size_t ad_len = rand_length(max_ad_len); 596 /* Set up exact-size guarded buffers. */ 597 const u8 *pt = pt_end - data_len; 598 const u8 *ad = ad_end - ad_len; 599 u8 *ct = ct_end - data_len; 600 u8 *decrypted = decrypted_end - data_len; 601 int err; 602 603 /* Encrypt and decrypt. */ 604 err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, 605 nonce_len, key); 606 KUNIT_ASSERT_EQ(test, 0, err); 607 err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, 608 nonce, nonce_len, key); 609 KUNIT_ASSERT_EQ(test, 0, err); 610 KUNIT_ASSERT_MEMEQ_MSG( 611 test, pt, decrypted, data_len, 612 "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", 613 data_len, ad_len); 614 } 615 } 616 617 /* 618 * Test that encryption and decryption produce the same results regardless of 619 * how the buffers are aligned in memory. 620 */ 621 static void test_aead_alignment_consistency(struct kunit *test) 622 { 623 const size_t max_data_len = 4096; 624 const size_t max_ad_len = 4096; 625 const size_t max_offset = 128; 626 const size_t nonce_len = AEAD_MAX_NONCE_LEN; 627 const size_t key_len = AEAD_MAX_KEY_LEN; 628 const size_t tag_len = AEAD_MAX_TAG_LEN; 629 u8 *raw_key1_buf = alloc_buf(test, key_len + max_offset); 630 u8 *raw_key2_buf = alloc_buf(test, key_len + max_offset); 631 u8 *nonce1_buf = alloc_buf(test, nonce_len + max_offset); 632 u8 *nonce2_buf = alloc_buf(test, nonce_len + max_offset); 633 u8 *pt1_buf = alloc_buf(test, max_data_len); 634 u8 *pt2_buf = alloc_buf(test, max_data_len); 635 u8 *ct1_buf = alloc_buf(test, max_data_len); 636 u8 *ct2_buf = alloc_buf(test, max_data_len); 637 u8 *ad1_buf = alloc_buf(test, max_ad_len); 638 u8 *ad2_buf = alloc_buf(test, max_ad_len); 639 u8 *tag1_buf = alloc_buf(test, tag_len + max_offset); 640 u8 *tag2_buf = alloc_buf(test, tag_len + max_offset); 641 struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); 642 int err; 643 644 for (int i = 0; i < 100; i++) { 645 /* Generate lengths. */ 646 size_t data_len = rand_length(max_data_len); 647 size_t ad_len = rand_length(max_ad_len); 648 649 /* Generate two sets of alignments. */ 650 u8 *raw_key1 = raw_key1_buf + rand_offset(max_offset); 651 u8 *raw_key2 = raw_key2_buf + rand_offset(max_offset); 652 u8 *nonce1 = nonce1_buf + rand_offset(max_offset); 653 u8 *nonce2 = nonce2_buf + rand_offset(max_offset); 654 u8 *pt1 = pt1_buf + rand_offset(max_data_len - data_len); 655 u8 *pt2 = pt2_buf + rand_offset(max_data_len - data_len); 656 u8 *ct1 = ct1_buf + rand_offset(max_data_len - data_len); 657 u8 *ct2 = ct2_buf + rand_offset(max_data_len - data_len); 658 u8 *ad1 = ad1_buf + rand_offset(max_ad_len - ad_len); 659 u8 *ad2 = ad2_buf + rand_offset(max_ad_len - ad_len); 660 u8 *tag1 = tag1_buf + rand_offset(max_offset); 661 u8 *tag2 = tag2_buf + rand_offset(max_offset); 662 663 /* 664 * Generate inputs in the first set of buffers using the first 665 * set of alignments. 666 */ 667 rand_bytes(raw_key1, key_len); 668 rand_bytes(nonce1, nonce_len); 669 rand_bytes(pt1, data_len); 670 rand_bytes(ad1, ad_len); 671 672 /* 673 * Copy the inputs to the second set of buffers using the second 674 * set of alignments. 675 */ 676 memcpy(raw_key2, raw_key1, key_len); 677 memcpy(nonce2, nonce1, nonce_len); 678 memcpy(pt2, pt1, data_len); 679 memcpy(ad2, ad1, ad_len); 680 681 /* Verify encryption consistency. */ 682 683 err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); 684 KUNIT_ASSERT_EQ(test, 0, err); 685 err = AEAD_ENCRYPT(ct1, pt1, data_len, tag1, ad1, ad_len, 686 nonce1, nonce_len, key); 687 KUNIT_ASSERT_EQ(test, 0, err); 688 689 err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); 690 KUNIT_ASSERT_EQ(test, 0, err); 691 err = AEAD_ENCRYPT(ct2, pt2, data_len, tag2, ad2, ad_len, 692 nonce2, nonce_len, key); 693 KUNIT_ASSERT_EQ(test, 0, err); 694 695 KUNIT_ASSERT_MEMEQ(test, ct1, ct2, data_len); 696 KUNIT_ASSERT_MEMEQ(test, tag1, tag2, tag_len); 697 698 /* Verify decryption consistency. */ 699 700 err = AEAD_PREPAREKEY(key, raw_key1, key_len, tag_len); 701 KUNIT_ASSERT_EQ(test, 0, err); 702 err = AEAD_DECRYPT(pt1, ct1, data_len, tag1, ad1, ad_len, 703 nonce1, nonce_len, key); 704 KUNIT_ASSERT_EQ(test, 0, err); 705 706 err = AEAD_PREPAREKEY(key, raw_key2, key_len, tag_len); 707 KUNIT_ASSERT_EQ(test, 0, err); 708 err = AEAD_DECRYPT(pt2, ct2, data_len, tag2, ad2, ad_len, 709 nonce2, nonce_len, key); 710 KUNIT_ASSERT_EQ(test, 0, err); 711 712 KUNIT_ASSERT_MEMEQ(test, pt1, pt2, data_len); 713 } 714 } 715 716 static void test_aead_inplace(struct kunit *test) 717 { 718 const size_t max_data_len = 1024; 719 const size_t max_ad_len = 512; 720 const size_t nonce_len = AEAD_MAX_NONCE_LEN; 721 size_t tag_len; 722 struct AEAD_KEY *key = aead_alloc_random_key(test, &tag_len); 723 u8 *data = aead_alloc_random_data(test, max_data_len + tag_len); 724 u8 *data2 = alloc_buf(test, max_data_len + tag_len); 725 u8 *ad = aead_alloc_random_data(test, max_ad_len); 726 const u8 *nonce = aead_alloc_random_data(test, nonce_len); 727 728 for (int i = 0; i < 100; i++) { 729 size_t data_len = rand_length(max_data_len); 730 size_t ad_len = rand_length(max_ad_len); 731 int err; 732 733 /* Encrypt out-of-place. */ 734 err = AEAD_ENCRYPT(data2, data, data_len, data2 + data_len, ad, 735 ad_len, nonce, nonce_len, key); 736 KUNIT_ASSERT_EQ(test, 0, err); 737 738 /* Encrypt in-place. */ 739 err = AEAD_ENCRYPT(data, data, data_len, data + data_len, ad, 740 ad_len, nonce, nonce_len, key); 741 KUNIT_ASSERT_EQ(test, 0, err); 742 743 /* Compare the results. */ 744 KUNIT_ASSERT_MEMEQ(test, data2, data, data_len + tag_len); 745 746 /* Decrypt out-of-place. */ 747 err = AEAD_DECRYPT(data2, data, data_len, data + data_len, ad, 748 ad_len, nonce, nonce_len, key); 749 KUNIT_ASSERT_EQ(test, 0, err); 750 751 /* Decrypt in-place. */ 752 err = AEAD_DECRYPT(data, data, data_len, data + data_len, ad, 753 ad_len, nonce, nonce_len, key); 754 KUNIT_ASSERT_EQ(test, 0, err); 755 756 /* Compare the results. */ 757 KUNIT_ASSERT_MEMEQ(test, data2, data, data_len); 758 } 759 } 760 761 /* 762 * Monte-Carlo test for AEAD algorithms. This deterministically generates 763 * random AEAD inputs, encrypts them, verifies decryption, and computes and 764 * verifies the checksum of all computed (ciphertext, tag) pairs. 765 */ 766 static void test_aead_monte_carlo(struct kunit *test) 767 { 768 const size_t max_data_len = 1024; 769 const size_t max_ad_len = 293; 770 u8 raw_key[AEAD_MAX_KEY_LEN]; 771 u8 nonce[AEAD_MAX_NONCE_LEN]; 772 u8 tag[AEAD_MAX_TAG_LEN]; 773 u8 *pt = alloc_buf(test, max_data_len); 774 u8 *ct = alloc_buf(test, max_data_len); 775 u8 *decrypted = alloc_buf(test, max_data_len); 776 u8 *ad = alloc_buf(test, max_ad_len); 777 struct AEAD_KEY *key = alloc_buf(test, sizeof(*key)); 778 struct blake2s_ctx checksum_ctx; 779 u8 actual_checksum[BLAKE2S_HASH_SIZE]; 780 int err; 781 782 blake2s_init(&checksum_ctx, BLAKE2S_HASH_SIZE); 783 784 for (size_t data_len = 0; data_len <= max_data_len; data_len++) { 785 size_t ad_len = data_len % max_ad_len; 786 size_t key_len = 787 AEAD_VALID_KEY_LENS[data_len % 788 ARRAY_SIZE(AEAD_VALID_KEY_LENS)]; 789 size_t nonce_len = 790 AEAD_VALID_NONCE_LENS[data_len % 791 ARRAY_SIZE(AEAD_VALID_NONCE_LENS)]; 792 size_t tag_len = 793 AEAD_VALID_TAG_LENS[data_len % 794 ARRAY_SIZE(AEAD_VALID_TAG_LENS)]; 795 796 rand_bytes_seeded_from_len(pt, data_len); 797 rand_bytes_seeded_from_len(ad, ad_len); 798 rand_bytes_seeded_from_len(raw_key, key_len); 799 rand_bytes_seeded_from_len(nonce, nonce_len); 800 801 err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); 802 KUNIT_ASSERT_EQ(test, 0, err); 803 804 err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, nonce, 805 nonce_len, key); 806 KUNIT_ASSERT_EQ_MSG( 807 test, 0, err, 808 "Encryption failed with data_len=%zu, ad_len=%zu", 809 data_len, ad_len); 810 err = AEAD_DECRYPT(decrypted, ct, data_len, tag, ad, ad_len, 811 nonce, nonce_len, key); 812 KUNIT_ASSERT_EQ_MSG( 813 test, 0, err, 814 "Decryption failed with data_len=%zu, ad_len=%zu", 815 data_len, ad_len); 816 KUNIT_ASSERT_MEMEQ_MSG( 817 test, pt, decrypted, data_len, 818 "Decryption didn't invert encryption; data_len=%zu, ad_len=%zu", 819 data_len, ad_len); 820 821 blake2s_update(&checksum_ctx, ct, data_len); 822 blake2s_update(&checksum_ctx, tag, tag_len); 823 } 824 825 blake2s_final(&checksum_ctx, actual_checksum); 826 KUNIT_EXPECT_MEMEQ_MSG(test, actual_checksum, AEAD_MONTE_CARLO_CHECKSUM, 827 BLAKE2S_HASH_SIZE, 828 "Monte-Carlo checksum mismatch"); 829 } 830 831 #define IRQ_TEST_DATA_LEN 256 832 #define IRQ_TEST_NUM_BUFFERS 3 /* matches max concurrency level */ 833 834 struct aead_irq_test_slot { 835 /* Fields written only at test case initialization time */ 836 u8 raw_key[AEAD_MAX_KEY_LEN]; 837 u8 nonce[AEAD_MAX_NONCE_LEN]; 838 u8 pt[IRQ_TEST_DATA_LEN]; 839 u8 ct[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; 840 u8 ad[IRQ_TEST_DATA_LEN]; 841 842 /* Fields written throughout the test case */ 843 struct AEAD_KEY key; 844 u8 scratch_buf[IRQ_TEST_DATA_LEN + AEAD_MAX_TAG_LEN]; 845 int phase; 846 atomic_t in_use; 847 }; 848 849 struct aead_irq_test_state { 850 struct aead_irq_test_slot slots[IRQ_TEST_NUM_BUFFERS]; 851 }; 852 853 static bool aead_irq_test_func(void *state_) 854 { 855 struct aead_irq_test_state *state = state_; 856 struct aead_irq_test_slot *slot; 857 size_t data_len; 858 bool ok = true; 859 860 /* 861 * Find a free slot. This should always succeed, since the number of 862 * slots is equal to the max concurrency level of kunit_run_irq_test(). 863 */ 864 for (slot = &state->slots[0]; 865 slot < &state->slots[ARRAY_SIZE(state->slots)]; slot++) { 866 if (atomic_cmpxchg(&slot->in_use, 0, 1) == 0) 867 break; 868 } 869 if (WARN_ON_ONCE(slot == &state->slots[ARRAY_SIZE(state->slots)])) 870 return false; 871 /* 872 * This execution context now has exclusive access to 'slot'. 873 * Next, execute the next operation that the slot is set to perform. 874 */ 875 876 data_len = sizeof(slot->pt); 877 if (slot->phase == 0) { 878 /* Phase 0: Prepare slot's key in current context. */ 879 ok = ok && AEAD_PREPAREKEY(&slot->key, slot->raw_key, 880 sizeof(slot->raw_key), 881 AEAD_MAX_TAG_LEN) == 0; 882 } else if (slot->phase == 1) { 883 /* 884 * Phase 1: Encrypt plaintext using key that may have been 885 * prepared in a different context. 886 */ 887 ok = ok && AEAD_ENCRYPT(slot->scratch_buf, slot->pt, data_len, 888 &slot->scratch_buf[data_len], slot->ad, 889 sizeof(slot->ad), slot->nonce, 890 sizeof(slot->nonce), &slot->key) == 0; 891 /* Verify the ciphertext (with concatenated auth tag) matches */ 892 ok = ok && 893 memcmp(slot->scratch_buf, slot->ct, sizeof(slot->ct)) == 0; 894 } else { 895 /* 896 * Phase 2: Decrypt ciphertext using key that may have been 897 * prepared in a different context. 898 */ 899 ok = ok && AEAD_DECRYPT(slot->scratch_buf, slot->ct, data_len, 900 &slot->ct[data_len], slot->ad, 901 sizeof(slot->ad), slot->nonce, 902 sizeof(slot->nonce), &slot->key) == 0; 903 /* Verify the plaintext matches. */ 904 ok = ok && memcmp(slot->scratch_buf, slot->pt, data_len) == 0; 905 } 906 slot->phase = (slot->phase + 1) % 3; 907 atomic_set_release(&slot->in_use, 0); 908 return ok; 909 } 910 911 /* 912 * Test that encryption and decryption produce the correct results in task, 913 * softirq, and hardirq contexts running concurrently -- including with keys 914 * prepared in other contexts. This is needed to cover fallback code paths that 915 * execute in contexts where FPU or vector registers cannot be used. 916 */ 917 static void test_aead_interrupt_context(struct kunit *test) 918 { 919 struct aead_irq_test_state *state = alloc_buf(test, sizeof(*state)); 920 921 memset(state, 0, sizeof(*state)); 922 923 /* 924 * For each slot, generate a set of AEAD inputs: a key, a nonce, a 925 * plaintext, and some associated data. Then generate the corresponding 926 * ciphertext with concatenated auth tag. 927 */ 928 for (int i = 0; i < IRQ_TEST_NUM_BUFFERS; i++) { 929 struct aead_irq_test_slot *slot = &state->slots[i]; 930 int err; 931 932 rand_bytes(slot->raw_key, sizeof(slot->raw_key)); 933 rand_bytes(slot->nonce, sizeof(slot->nonce)); 934 rand_bytes(slot->pt, sizeof(slot->pt)); 935 rand_bytes(slot->ad, sizeof(slot->ad)); 936 err = AEAD_PREPAREKEY(&slot->key, slot->raw_key, 937 sizeof(slot->raw_key), AEAD_MAX_TAG_LEN); 938 KUNIT_ASSERT_EQ(test, 0, err); 939 err = AEAD_ENCRYPT(slot->ct, slot->pt, sizeof(slot->pt), 940 &slot->ct[sizeof(slot->pt)], slot->ad, 941 sizeof(slot->ad), slot->nonce, 942 sizeof(slot->nonce), &slot->key); 943 KUNIT_ASSERT_EQ(test, 0, err); 944 } 945 946 kunit_run_irq_test(test, aead_irq_test_func, 100000, state); 947 } 948 949 /* Benchmark AEAD encryption and decryption on various data lengths. */ 950 static void benchmark_aead(struct kunit *test) 951 { 952 static const size_t data_lens_to_test[] = { 953 16, 64, 128, 256, 512, 1024, 1420, 4096, 16384, 954 }; 955 const size_t max_data_len = 16384; 956 const size_t ad_len = 16; 957 const size_t key_len = AEAD_MAX_KEY_LEN; 958 const size_t nonce_len = AEAD_MAX_NONCE_LEN; 959 const size_t tag_len = AEAD_MAX_TAG_LEN; 960 const u8 *raw_key, *nonce, *ad; 961 u8 *pt, *ct, *tag; 962 struct AEAD_KEY *key; 963 int err; 964 965 if (!IS_ENABLED(CONFIG_CRYPTO_LIB_BENCHMARK)) 966 kunit_skip(test, "not enabled"); 967 968 raw_key = aead_alloc_random_data(test, key_len); 969 nonce = aead_alloc_random_data(test, nonce_len); 970 ad = aead_alloc_random_data(test, ad_len); 971 pt = aead_alloc_random_data(test, max_data_len); 972 ct = alloc_buf(test, max_data_len); 973 tag = alloc_buf(test, tag_len); 974 975 key = alloc_buf(test, sizeof(*key)); 976 err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len); 977 KUNIT_ASSERT_EQ(test, 0, err); 978 979 /* Warm-up */ 980 for (size_t i = 0; i < 10000000; i += max_data_len) { 981 err = AEAD_ENCRYPT(ct, pt, max_data_len, tag, ad, ad_len, nonce, 982 nonce_len, key); 983 KUNIT_ASSERT_EQ(test, 0, err); 984 err = AEAD_DECRYPT(pt, ct, max_data_len, tag, ad, ad_len, nonce, 985 nonce_len, key); 986 KUNIT_ASSERT_EQ(test, 0, err); 987 } 988 989 for (size_t i = 0; i < ARRAY_SIZE(data_lens_to_test); i++) { 990 size_t data_len = data_lens_to_test[i]; 991 size_t num_iters = 10000000 / (data_len + 128); 992 u64 t_enc, t_dec; 993 bool ok = true; 994 995 KUNIT_ASSERT_LE(test, data_len, max_data_len); 996 997 preempt_disable(); 998 999 t_enc = ktime_get_ns(); 1000 for (size_t j = 0; j < num_iters; j++) { 1001 err = AEAD_ENCRYPT(ct, pt, data_len, tag, ad, ad_len, 1002 nonce, nonce_len, key); 1003 ok &= (err == 0); 1004 } 1005 t_enc = ktime_get_ns() - t_enc; 1006 1007 t_dec = ktime_get_ns(); 1008 for (size_t j = 0; j < num_iters; j++) { 1009 err = AEAD_DECRYPT(pt, ct, data_len, tag, ad, ad_len, 1010 nonce, nonce_len, key); 1011 ok &= (err == 0); 1012 } 1013 t_dec = ktime_get_ns() - t_dec; 1014 1015 preempt_enable(); 1016 1017 KUNIT_ASSERT_TRUE_MSG(test, ok, "data_len=%zu", data_len); 1018 1019 kunit_info(test, "data_len=%zu: enc %llu MB/s, dec %llu MB/s", 1020 data_len, 1021 div64_u64((u64)data_len * num_iters * 1000, 1022 t_enc ?: 1), 1023 div64_u64((u64)data_len * num_iters * 1000, 1024 t_dec ?: 1)); 1025 } 1026 } 1027 1028 /* clang-format off */ 1029 #define AEAD_KUNIT_CASES \ 1030 KUNIT_CASE(test_aead_all_key_lens), \ 1031 KUNIT_CASE(test_aead_all_nonce_lens), \ 1032 KUNIT_CASE(test_aead_all_tag_lens), \ 1033 KUNIT_CASE(test_aead_incremental_updates), \ 1034 KUNIT_CASE(test_aead_data_buffer_overruns), \ 1035 KUNIT_CASE(test_aead_alignment_consistency), \ 1036 KUNIT_CASE(test_aead_inplace), \ 1037 KUNIT_CASE(test_aead_monte_carlo), \ 1038 KUNIT_CASE(test_aead_interrupt_context), \ 1039 KUNIT_CASE(benchmark_aead) 1040