1 // SPDX-License-Identifier: GPL-2.0 2 #include <kunit/test.h> 3 #include <kunit/test-bug.h> 4 #include <linux/mm.h> 5 #include <linux/slab.h> 6 #include <linux/module.h> 7 #include <linux/kernel.h> 8 #include <linux/rcupdate.h> 9 #include <linux/delay.h> 10 #include <linux/perf_event.h> 11 #include <linux/kprobes.h> 12 #include "../mm/slab.h" 13 14 static struct kunit_resource resource; 15 static int slab_errors; 16 17 /* 18 * Wrapper function for kmem_cache_create(), which reduces 2 parameters: 19 * 'align' and 'ctor', and sets SLAB_SKIP_KFENCE flag to avoid getting an 20 * object from kfence pool, where the operation could be caught by both 21 * our test and kfence sanity check. 22 */ 23 static struct kmem_cache *test_kmem_cache_create(const char *name, 24 unsigned int size, slab_flags_t flags) 25 { 26 struct kmem_cache *s = kmem_cache_create(name, size, 0, 27 (flags | SLAB_NO_USER_FLAGS), NULL); 28 s->flags |= SLAB_SKIP_KFENCE; 29 return s; 30 } 31 32 static void test_clobber_zone(struct kunit *test) 33 { 34 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_alloc", 64, 35 SLAB_RED_ZONE); 36 u8 *p = kmem_cache_alloc(s, GFP_KERNEL); 37 38 kasan_disable_current(); 39 p[64] = 0x12; 40 41 validate_slab_cache(s); 42 KUNIT_EXPECT_EQ(test, 2, slab_errors); 43 44 kasan_enable_current(); 45 kmem_cache_free(s, p); 46 kmem_cache_destroy(s); 47 } 48 49 #ifndef CONFIG_KASAN 50 static void test_next_pointer(struct kunit *test) 51 { 52 struct kmem_cache *s = test_kmem_cache_create("TestSlub_next_ptr_free", 53 64, SLAB_POISON); 54 u8 *p = kmem_cache_alloc(s, GFP_KERNEL); 55 unsigned long tmp; 56 unsigned long *ptr_addr; 57 58 kmem_cache_free(s, p); 59 60 ptr_addr = (unsigned long *)(p + s->offset); 61 tmp = *ptr_addr; 62 p[s->offset] = ~p[s->offset]; 63 64 /* 65 * Expecting three errors. 66 * One for the corrupted freechain and the other one for the wrong 67 * count of objects in use. The third error is fixing broken cache. 68 */ 69 validate_slab_cache(s); 70 KUNIT_EXPECT_EQ(test, 3, slab_errors); 71 72 /* 73 * Try to repair corrupted freepointer. 74 * Still expecting two errors. The first for the wrong count 75 * of objects in use. 76 * The second error is for fixing broken cache. 77 */ 78 *ptr_addr = tmp; 79 slab_errors = 0; 80 81 validate_slab_cache(s); 82 KUNIT_EXPECT_EQ(test, 2, slab_errors); 83 84 /* 85 * Previous validation repaired the count of objects in use. 86 * Now expecting no error. 87 */ 88 slab_errors = 0; 89 validate_slab_cache(s); 90 KUNIT_EXPECT_EQ(test, 0, slab_errors); 91 92 kmem_cache_destroy(s); 93 } 94 95 static void test_first_word(struct kunit *test) 96 { 97 struct kmem_cache *s = test_kmem_cache_create("TestSlub_1th_word_free", 98 64, SLAB_POISON); 99 u8 *p = kmem_cache_alloc(s, GFP_KERNEL); 100 101 kmem_cache_free(s, p); 102 *p = 0x78; 103 104 validate_slab_cache(s); 105 KUNIT_EXPECT_EQ(test, 2, slab_errors); 106 107 kmem_cache_destroy(s); 108 } 109 110 static void test_clobber_50th_byte(struct kunit *test) 111 { 112 struct kmem_cache *s = test_kmem_cache_create("TestSlub_50th_word_free", 113 64, SLAB_POISON); 114 u8 *p = kmem_cache_alloc(s, GFP_KERNEL); 115 116 kmem_cache_free(s, p); 117 p[50] = 0x9a; 118 119 validate_slab_cache(s); 120 KUNIT_EXPECT_EQ(test, 2, slab_errors); 121 122 kmem_cache_destroy(s); 123 } 124 #endif 125 126 static void test_clobber_redzone_free(struct kunit *test) 127 { 128 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_free", 64, 129 SLAB_RED_ZONE); 130 u8 *p = kmem_cache_alloc(s, GFP_KERNEL); 131 132 kasan_disable_current(); 133 kmem_cache_free(s, p); 134 p[64] = 0xab; 135 136 validate_slab_cache(s); 137 KUNIT_EXPECT_EQ(test, 2, slab_errors); 138 139 kasan_enable_current(); 140 kmem_cache_destroy(s); 141 } 142 143 static void test_kmalloc_redzone_access(struct kunit *test) 144 { 145 struct kmem_cache *s = test_kmem_cache_create("TestSlub_RZ_kmalloc", 32, 146 SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE); 147 u8 *p = alloc_hooks(__kmalloc_cache_noprof(s, GFP_KERNEL, 18)); 148 149 kasan_disable_current(); 150 151 /* Suppress the -Warray-bounds warning */ 152 OPTIMIZER_HIDE_VAR(p); 153 p[18] = 0xab; 154 p[19] = 0xab; 155 156 validate_slab_cache(s); 157 KUNIT_EXPECT_EQ(test, 2, slab_errors); 158 159 kasan_enable_current(); 160 kmem_cache_free(s, p); 161 kmem_cache_destroy(s); 162 } 163 164 struct test_kfree_rcu_struct { 165 union { 166 struct rcu_head rcu; 167 struct kvfree_rcu_head kvrcu; 168 }; 169 }; 170 171 static void test_kfree_rcu(struct kunit *test) 172 { 173 struct kmem_cache *s; 174 struct test_kfree_rcu_struct *p; 175 176 if (IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST)) 177 kunit_skip(test, "can't do kfree_rcu() when test is built-in"); 178 179 s = test_kmem_cache_create("TestSlub_kfree_rcu", 180 sizeof(struct test_kfree_rcu_struct), 181 SLAB_NO_MERGE); 182 p = kmem_cache_alloc(s, GFP_KERNEL); 183 184 kfree_rcu(p, rcu); 185 kmem_cache_destroy(s); 186 187 KUNIT_EXPECT_EQ(test, 0, slab_errors); 188 } 189 190 struct cache_destroy_work { 191 struct work_struct work; 192 struct kmem_cache *s; 193 }; 194 195 static void cache_destroy_workfn(struct work_struct *w) 196 { 197 struct cache_destroy_work *cdw; 198 199 cdw = container_of(w, struct cache_destroy_work, work); 200 kmem_cache_destroy(cdw->s); 201 } 202 203 #define KMEM_CACHE_DESTROY_NR 10 204 205 static void test_kfree_rcu_wq_destroy(struct kunit *test) 206 { 207 struct test_kfree_rcu_struct *p; 208 struct cache_destroy_work cdw; 209 struct workqueue_struct *wq; 210 struct kmem_cache *s; 211 unsigned int delay; 212 int i; 213 214 if (IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST)) 215 kunit_skip(test, "can't do kfree_rcu() when test is built-in"); 216 217 INIT_WORK_ONSTACK(&cdw.work, cache_destroy_workfn); 218 wq = alloc_workqueue("test_kfree_rcu_destroy_wq", 219 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 0); 220 221 if (!wq) 222 kunit_skip(test, "failed to alloc wq"); 223 224 for (i = 0; i < KMEM_CACHE_DESTROY_NR; i++) { 225 s = test_kmem_cache_create("TestSlub_kfree_rcu_wq_destroy", 226 sizeof(struct test_kfree_rcu_struct), 227 SLAB_NO_MERGE); 228 229 if (!s) 230 kunit_skip(test, "failed to create cache"); 231 232 delay = get_random_u8(); 233 p = kmem_cache_alloc(s, GFP_KERNEL); 234 kfree_rcu(p, rcu); 235 236 cdw.s = s; 237 238 msleep(delay); 239 queue_work(wq, &cdw.work); 240 flush_work(&cdw.work); 241 } 242 243 destroy_workqueue(wq); 244 KUNIT_EXPECT_EQ(test, 0, slab_errors); 245 } 246 247 static void test_leak_destroy(struct kunit *test) 248 { 249 struct kmem_cache *s = test_kmem_cache_create("TestSlub_leak_destroy", 250 64, SLAB_NO_MERGE); 251 kmem_cache_alloc(s, GFP_KERNEL); 252 253 kmem_cache_destroy(s); 254 255 KUNIT_EXPECT_EQ(test, 2, slab_errors); 256 } 257 258 static void test_krealloc_redzone_zeroing(struct kunit *test) 259 { 260 u8 *p; 261 int i; 262 struct kmem_cache *s = test_kmem_cache_create("TestSlub_krealloc", 64, 263 SLAB_KMALLOC|SLAB_STORE_USER|SLAB_RED_ZONE); 264 265 p = alloc_hooks(__kmalloc_cache_noprof(s, GFP_KERNEL, 48)); 266 memset(p, 0xff, 48); 267 268 kasan_disable_current(); 269 OPTIMIZER_HIDE_VAR(p); 270 271 /* Test shrink */ 272 p = krealloc(p, 40, GFP_KERNEL | __GFP_ZERO); 273 for (i = 40; i < 64; i++) 274 KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE); 275 276 /* Test grow within the same 64B kmalloc object */ 277 p = krealloc(p, 56, GFP_KERNEL | __GFP_ZERO); 278 for (i = 40; i < 56; i++) 279 KUNIT_EXPECT_EQ(test, p[i], 0); 280 for (i = 56; i < 64; i++) 281 KUNIT_EXPECT_EQ(test, p[i], SLUB_RED_ACTIVE); 282 283 validate_slab_cache(s); 284 KUNIT_EXPECT_EQ(test, 0, slab_errors); 285 286 memset(p, 0xff, 56); 287 /* Test grow with allocating a bigger 128B object */ 288 p = krealloc(p, 112, GFP_KERNEL | __GFP_ZERO); 289 for (i = 0; i < 56; i++) 290 KUNIT_EXPECT_EQ(test, p[i], 0xff); 291 for (i = 56; i < 112; i++) 292 KUNIT_EXPECT_EQ(test, p[i], 0); 293 294 kfree(p); 295 kasan_enable_current(); 296 kmem_cache_destroy(s); 297 } 298 299 #if defined(CONFIG_PERF_EVENTS) || (defined(CONFIG_KPROBES) && defined(CONFIG_SMP)) 300 #define NR_ITERATIONS 1000 301 #define NR_OBJECTS 1000 302 static struct test_kfree_rcu_struct *objects[NR_OBJECTS]; 303 304 struct test_nolock_context { 305 struct kunit *test; 306 int callback_count; 307 int alloc_ok; 308 int alloc_fail; 309 #ifdef CONFIG_PERF_EVENTS 310 struct perf_event *event; 311 #endif 312 #if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) 313 struct kprobe kprobe; 314 #endif 315 }; 316 317 static void test_kmalloc_and_friends(void) 318 { 319 int i, j; 320 bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); 321 322 for (i = 0; i < NR_ITERATIONS; i++) { 323 for (j = 0; j < NR_OBJECTS; j++) { 324 gfp_t gfp = (i & 1) ? GFP_KERNEL : GFP_KERNEL_ACCOUNT; 325 326 objects[j] = kmalloc_obj(*objects[j], gfp); 327 if (!objects[j]) { 328 j--; 329 while (j >= 0) 330 kfree(objects[j--]); 331 return; 332 } 333 } 334 335 for (j = 0; j < NR_OBJECTS; j++) { 336 if (can_use_kfree_rcu && (i & 2)) 337 kfree_rcu(objects[j], rcu); 338 else 339 kfree(objects[j]); 340 } 341 } 342 } 343 344 static void test_nolock(struct test_nolock_context *ctx) 345 { 346 struct test_kfree_rcu_struct *objp; 347 gfp_t gfp; 348 bool can_use_kfree_rcu = !IS_BUILTIN(CONFIG_SLUB_KUNIT_TEST); 349 350 /* __GFP_ACCOUNT to test kmalloc_nolock() in alloc_slab_obj_exts() */ 351 gfp = (ctx->callback_count & 1) ? 0 : __GFP_ACCOUNT; 352 objp = kmalloc_nolock(sizeof(*objp), gfp, NUMA_NO_NODE); 353 354 if (objp) 355 ctx->alloc_ok++; 356 else 357 ctx->alloc_fail++; 358 359 if (can_use_kfree_rcu && (ctx->callback_count & 2)) 360 kfree_rcu_nolock(objp, kvrcu); 361 else 362 kfree_nolock(objp); 363 364 ctx->callback_count++; 365 } 366 #endif 367 368 #ifdef CONFIG_PERF_EVENTS 369 static struct perf_event_attr hw_attr = { 370 .type = PERF_TYPE_HARDWARE, 371 .config = PERF_COUNT_HW_CPU_CYCLES, 372 .size = sizeof(struct perf_event_attr), 373 .pinned = 1, 374 .disabled = 1, 375 .freq = 1, 376 .sample_freq = 100000, 377 }; 378 379 static void overflow_handler_test_nolock(struct perf_event *event, 380 struct perf_sample_data *data, 381 struct pt_regs *regs) 382 { 383 struct test_nolock_context *ctx = event->overflow_handler_context; 384 385 test_nolock(ctx); 386 } 387 388 static bool enable_perf_events(struct test_nolock_context *ctx) 389 { 390 struct perf_event *event; 391 392 event = perf_event_create_kernel_counter(&hw_attr, -1, current, 393 overflow_handler_test_nolock, 394 ctx); 395 396 if (IS_ERR(event)) 397 return false; 398 399 ctx->event = event; 400 perf_event_enable(ctx->event); 401 return true; 402 } 403 404 static void disable_perf_events(struct test_nolock_context *ctx) 405 { 406 kunit_info(ctx->test, "HW perf events: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", 407 ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); 408 409 perf_event_disable(ctx->event); 410 perf_event_release_kernel(ctx->event); 411 } 412 413 static void test_kmalloc_nolock_and_friends_perf(struct kunit *test) 414 { 415 struct test_nolock_context ctx = { .test = test }; 416 417 if (!enable_perf_events(&ctx)) 418 kunit_skip(test, "Failed to enable perf event, skipping"); 419 420 test_kmalloc_and_friends(); 421 422 disable_perf_events(&ctx); 423 KUNIT_EXPECT_EQ(test, 0, slab_errors); 424 } 425 #endif 426 427 #if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) 428 static int slab_kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) 429 { 430 struct test_nolock_context *ctx; 431 432 ctx = container_of(p, struct test_nolock_context, kprobe); 433 test_nolock(ctx); 434 return 0; 435 } 436 437 static bool register_slab_kprobes(struct test_nolock_context *ctx) 438 { 439 ctx->kprobe.symbol_name = "slab_attach_kprobe_locked"; 440 ctx->kprobe.pre_handler = slab_kprobe_pre_handler; 441 442 if (register_kprobe(&ctx->kprobe)) 443 return false; 444 return true; 445 } 446 447 static void unregister_slab_kprobes(struct test_nolock_context *ctx) 448 { 449 kunit_info(ctx->test, "kprobes: callback_count: %d, alloc_ok: %d, alloc_fail: %d\n", 450 ctx->callback_count, ctx->alloc_ok, ctx->alloc_fail); 451 unregister_kprobe(&ctx->kprobe); 452 } 453 454 static void test_kmalloc_nolock_and_friends_kprobe(struct kunit *test) 455 { 456 struct test_nolock_context ctx = { .test = test }; 457 458 if (!register_slab_kprobes(&ctx)) 459 kunit_skip(test, "Failed to register kprobe, skipping"); 460 461 test_kmalloc_and_friends(); 462 463 unregister_slab_kprobes(&ctx); 464 KUNIT_EXPECT_EQ(test, 0, slab_errors); 465 } 466 #endif 467 468 static int test_init(struct kunit *test) 469 { 470 slab_errors = 0; 471 472 kunit_add_named_resource(test, NULL, NULL, &resource, 473 "slab_errors", &slab_errors); 474 return 0; 475 } 476 477 static struct kunit_case test_cases[] = { 478 KUNIT_CASE(test_clobber_zone), 479 480 #ifndef CONFIG_KASAN 481 KUNIT_CASE(test_next_pointer), 482 KUNIT_CASE(test_first_word), 483 KUNIT_CASE(test_clobber_50th_byte), 484 #endif 485 486 KUNIT_CASE(test_clobber_redzone_free), 487 KUNIT_CASE(test_kmalloc_redzone_access), 488 KUNIT_CASE(test_kfree_rcu), 489 KUNIT_CASE(test_kfree_rcu_wq_destroy), 490 KUNIT_CASE(test_leak_destroy), 491 KUNIT_CASE(test_krealloc_redzone_zeroing), 492 #ifdef CONFIG_PERF_EVENTS 493 KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_perf), 494 #endif 495 #if defined(CONFIG_KPROBES) && defined(CONFIG_SMP) 496 KUNIT_CASE_SLOW(test_kmalloc_nolock_and_friends_kprobe), 497 #endif 498 {} 499 }; 500 501 static struct kunit_suite test_suite = { 502 .name = "slub_test", 503 .init = test_init, 504 .test_cases = test_cases, 505 }; 506 kunit_test_suite(test_suite); 507 508 MODULE_DESCRIPTION("Kunit tests for slub allocator"); 509 MODULE_LICENSE("GPL"); 510