1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Test cases for SL[AOU]B/page initialization at alloc/free time. 4 */ 5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 6 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/mm.h> 10 #include <linux/module.h> 11 #include <linux/slab.h> 12 #include <linux/string.h> 13 #include <linux/vmalloc.h> 14 15 #define GARBAGE_INT (0x09A7BA9E) 16 #define GARBAGE_BYTE (0x9E) 17 18 #define REPORT_FAILURES_IN_FN() \ 19 do { \ 20 if (failures) \ 21 pr_info("%s failed %d out of %d times\n", \ 22 __func__, failures, num_tests); \ 23 else \ 24 pr_info("all %d tests in %s passed\n", \ 25 num_tests, __func__); \ 26 } while (0) 27 28 /* Calculate the number of uninitialized bytes in the buffer. */ 29 static int __init count_nonzero_bytes(void *ptr, size_t size) 30 { 31 int i, ret = 0; 32 unsigned char *p = (unsigned char *)ptr; 33 34 for (i = 0; i < size; i++) 35 if (p[i]) 36 ret++; 37 return ret; 38 } 39 40 /* Fill a buffer with garbage, skipping |skip| first bytes. */ 41 static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) 42 { 43 unsigned int *p = (unsigned int *)((char *)ptr + skip); 44 int i = 0; 45 46 WARN_ON(skip > size); 47 size -= skip; 48 49 while (size >= sizeof(*p)) { 50 p[i] = GARBAGE_INT; 51 i++; 52 size -= sizeof(*p); 53 } 54 if (size) 55 memset(&p[i], GARBAGE_BYTE, size); 56 } 57 58 static void __init fill_with_garbage(void *ptr, size_t size) 59 { 60 fill_with_garbage_skip(ptr, size, 0); 61 } 62 63 static int __init do_alloc_pages_order(int order, int *total_failures) 64 { 65 struct page *page; 66 void *buf; 67 size_t size = PAGE_SIZE << order; 68 69 page = alloc_pages(GFP_KERNEL, order); 70 if (!page) 71 goto err; 72 buf = page_address(page); 73 fill_with_garbage(buf, size); 74 __free_pages(page, order); 75 76 page = alloc_pages(GFP_KERNEL, order); 77 if (!page) 78 goto err; 79 buf = page_address(page); 80 if (count_nonzero_bytes(buf, size)) 81 (*total_failures)++; 82 fill_with_garbage(buf, size); 83 __free_pages(page, order); 84 return 1; 85 err: 86 (*total_failures)++; 87 return 1; 88 } 89 90 /* Test the page allocator by calling alloc_pages with different orders. */ 91 static int __init test_pages(int *total_failures) 92 { 93 int failures = 0, num_tests = 0; 94 int i; 95 96 for (i = 0; i < NR_PAGE_ORDERS; i++) 97 num_tests += do_alloc_pages_order(i, &failures); 98 99 REPORT_FAILURES_IN_FN(); 100 *total_failures += failures; 101 return num_tests; 102 } 103 104 /* Test kmalloc() with given parameters. */ 105 static int __init do_kmalloc_size(size_t size, int *total_failures) 106 { 107 void *buf; 108 109 buf = kmalloc(size, GFP_KERNEL); 110 if (!buf) 111 goto err; 112 fill_with_garbage(buf, size); 113 kfree(buf); 114 115 buf = kmalloc(size, GFP_KERNEL); 116 if (!buf) 117 goto err; 118 if (count_nonzero_bytes(buf, size)) 119 (*total_failures)++; 120 fill_with_garbage(buf, size); 121 kfree(buf); 122 return 1; 123 err: 124 (*total_failures)++; 125 return 1; 126 } 127 128 /* Test vmalloc() with given parameters. */ 129 static int __init do_vmalloc_size(size_t size, int *total_failures) 130 { 131 void *buf; 132 133 buf = vmalloc(size); 134 if (!buf) 135 goto err; 136 fill_with_garbage(buf, size); 137 vfree(buf); 138 139 buf = vmalloc(size); 140 if (!buf) 141 goto err; 142 if (count_nonzero_bytes(buf, size)) 143 (*total_failures)++; 144 fill_with_garbage(buf, size); 145 vfree(buf); 146 return 1; 147 err: 148 (*total_failures)++; 149 return 1; 150 } 151 152 /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ 153 static int __init test_kvmalloc(int *total_failures) 154 { 155 int failures = 0, num_tests = 0; 156 int i, size; 157 158 for (i = 0; i < 20; i++) { 159 size = 1 << i; 160 num_tests += do_kmalloc_size(size, &failures); 161 num_tests += do_vmalloc_size(size, &failures); 162 } 163 164 REPORT_FAILURES_IN_FN(); 165 *total_failures += failures; 166 return num_tests; 167 } 168 169 #define CTOR_BYTES (sizeof(unsigned int)) 170 #define CTOR_PATTERN (0x41414141) 171 /* Initialize the first 4 bytes of the object. */ 172 static void test_ctor(void *obj) 173 { 174 *(unsigned int *)obj = CTOR_PATTERN; 175 } 176 177 /* 178 * Check the invariants for the buffer allocated from a slab cache. 179 * If the cache has a test constructor, the first 4 bytes of the object must 180 * always remain equal to CTOR_PATTERN. 181 * If the cache isn't an RCU-typesafe one, or if the allocation is done with 182 * __GFP_ZERO, then the object contents must be zeroed after allocation. 183 * If the cache is an RCU-typesafe one, the object contents must never be 184 * zeroed after the first use. This is checked by memcmp() in 185 * do_kmem_cache_size(). 186 */ 187 static bool __init check_buf(void *buf, int size, bool want_ctor, 188 bool want_rcu, bool want_zero) 189 { 190 int bytes; 191 bool fail = false; 192 193 bytes = count_nonzero_bytes(buf, size); 194 WARN_ON(want_ctor && want_zero); 195 if (want_zero) 196 return bytes; 197 if (want_ctor) { 198 if (*(unsigned int *)buf != CTOR_PATTERN) 199 fail = 1; 200 } else { 201 if (bytes) 202 fail = !want_rcu; 203 } 204 return fail; 205 } 206 207 #define BULK_SIZE 100 208 static void *bulk_array[BULK_SIZE]; 209 210 /* 211 * Test kmem_cache with given parameters: 212 * want_ctor - use a constructor; 213 * want_rcu - use SLAB_TYPESAFE_BY_RCU; 214 * want_zero - use __GFP_ZERO. 215 */ 216 static int __init do_kmem_cache_size(size_t size, bool want_ctor, 217 bool want_rcu, bool want_zero, 218 int *total_failures) 219 { 220 struct kmem_cache *c; 221 int iter; 222 bool fail = false; 223 gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); 224 void *buf, *buf_copy; 225 226 c = kmem_cache_create("test_cache", size, 1, 227 want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, 228 want_ctor ? test_ctor : NULL); 229 for (iter = 0; iter < 10; iter++) { 230 /* Do a test of bulk allocations */ 231 if (!want_rcu && !want_ctor) { 232 if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, 233 bulk_array)) { 234 fail = true; 235 } else { 236 int i; 237 for (i = 0; i < BULK_SIZE; i++) 238 fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); 239 kmem_cache_free_bulk(c, BULK_SIZE, bulk_array); 240 } 241 } 242 243 buf = kmem_cache_alloc(c, alloc_mask); 244 /* Check that buf is zeroed, if it must be. */ 245 fail |= check_buf(buf, size, want_ctor, want_rcu, want_zero); 246 fill_with_garbage_skip(buf, size, want_ctor ? CTOR_BYTES : 0); 247 248 if (!want_rcu) { 249 kmem_cache_free(c, buf); 250 continue; 251 } 252 253 /* 254 * If this is an RCU cache, use a critical section to ensure we 255 * can touch objects after they're freed. 256 */ 257 rcu_read_lock(); 258 /* 259 * Copy the buffer to check that it's not wiped on 260 * free(). 261 */ 262 buf_copy = kmalloc(size, GFP_ATOMIC); 263 if (buf_copy) 264 memcpy(buf_copy, buf, size); 265 266 kmem_cache_free(c, buf); 267 /* 268 * Check that |buf| is intact after kmem_cache_free(). 269 * |want_zero| is false, because we wrote garbage to 270 * the buffer already. 271 */ 272 fail |= check_buf(buf, size, want_ctor, want_rcu, 273 false); 274 if (buf_copy) { 275 fail |= (bool)memcmp(buf, buf_copy, size); 276 kfree(buf_copy); 277 } 278 rcu_read_unlock(); 279 } 280 kmem_cache_destroy(c); 281 282 *total_failures += fail; 283 return 1; 284 } 285 286 /* 287 * Check that the data written to an RCU-allocated object survives 288 * reallocation. 289 */ 290 static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) 291 { 292 struct kmem_cache *c; 293 void *buf, *buf_contents, *saved_ptr; 294 void **used_objects; 295 int i, iter, maxiter = 1024; 296 bool fail = false; 297 298 c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, 299 NULL); 300 buf = kmem_cache_alloc(c, GFP_KERNEL); 301 if (!buf) 302 goto out; 303 saved_ptr = buf; 304 fill_with_garbage(buf, size); 305 buf_contents = kmalloc(size, GFP_KERNEL); 306 if (!buf_contents) { 307 kmem_cache_free(c, buf); 308 goto out; 309 } 310 used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); 311 if (!used_objects) { 312 kmem_cache_free(c, buf); 313 kfree(buf_contents); 314 goto out; 315 } 316 memcpy(buf_contents, buf, size); 317 kmem_cache_free(c, buf); 318 /* 319 * Run for a fixed number of iterations. If we never hit saved_ptr, 320 * assume the test passes. 321 */ 322 for (iter = 0; iter < maxiter; iter++) { 323 buf = kmem_cache_alloc(c, GFP_KERNEL); 324 used_objects[iter] = buf; 325 if (buf == saved_ptr) { 326 fail = memcmp(buf_contents, buf, size); 327 for (i = 0; i <= iter; i++) 328 kmem_cache_free(c, used_objects[i]); 329 goto free_out; 330 } 331 } 332 333 for (iter = 0; iter < maxiter; iter++) 334 kmem_cache_free(c, used_objects[iter]); 335 336 free_out: 337 kfree(buf_contents); 338 kfree(used_objects); 339 out: 340 kmem_cache_destroy(c); 341 *total_failures += fail; 342 return 1; 343 } 344 345 static int __init do_kmem_cache_size_bulk(int size, int *total_failures) 346 { 347 struct kmem_cache *c; 348 int i, iter, maxiter = 1024; 349 int bytes; 350 bool fail = false; 351 void *objects[10]; 352 353 c = kmem_cache_create("test_cache", size, size, 0, NULL); 354 for (iter = 0; (iter < maxiter) && !fail; iter++) { 355 if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), 356 objects)) 357 continue; 358 359 for (i = 0; i < ARRAY_SIZE(objects); i++) { 360 bytes = count_nonzero_bytes(objects[i], size); 361 if (bytes) 362 fail = true; 363 fill_with_garbage(objects[i], size); 364 } 365 366 kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects); 367 } 368 kmem_cache_destroy(c); 369 *total_failures += fail; 370 return 1; 371 } 372 373 /* 374 * Test kmem_cache allocation by creating caches of different sizes, with and 375 * without constructors, with and without SLAB_TYPESAFE_BY_RCU. 376 */ 377 static int __init test_kmemcache(int *total_failures) 378 { 379 int failures = 0, num_tests = 0; 380 int i, flags, size; 381 bool ctor, rcu, zero; 382 383 for (i = 0; i < 10; i++) { 384 size = 8 << i; 385 for (flags = 0; flags < 8; flags++) { 386 ctor = flags & 1; 387 rcu = flags & 2; 388 zero = flags & 4; 389 if (ctor && zero) 390 continue; 391 num_tests += do_kmem_cache_size(size, ctor, rcu, zero, 392 &failures); 393 } 394 num_tests += do_kmem_cache_size_bulk(size, &failures); 395 } 396 REPORT_FAILURES_IN_FN(); 397 *total_failures += failures; 398 return num_tests; 399 } 400 401 /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ 402 static int __init test_rcu_persistent(int *total_failures) 403 { 404 int failures = 0, num_tests = 0; 405 int i, size; 406 407 for (i = 0; i < 10; i++) { 408 size = 8 << i; 409 num_tests += do_kmem_cache_rcu_persistent(size, &failures); 410 } 411 REPORT_FAILURES_IN_FN(); 412 *total_failures += failures; 413 return num_tests; 414 } 415 416 /* 417 * Run the tests. Each test function returns the number of executed tests and 418 * updates |failures| with the number of failed tests. 419 */ 420 static int __init test_meminit_init(void) 421 { 422 int failures = 0, num_tests = 0; 423 424 num_tests += test_pages(&failures); 425 num_tests += test_kvmalloc(&failures); 426 num_tests += test_kmemcache(&failures); 427 num_tests += test_rcu_persistent(&failures); 428 429 if (failures == 0) 430 pr_info("all %d tests passed!\n", num_tests); 431 else 432 pr_info("failures: %d out of %d\n", failures, num_tests); 433 434 return failures ? -EINVAL : 0; 435 } 436 module_init(test_meminit_init); 437 438 MODULE_DESCRIPTION("Test cases for SL[AOU]B/page initialization at alloc/free time"); 439 MODULE_LICENSE("GPL"); 440