1 // SPDX-License-Identifier: GPL-2.0 2 #include <dirent.h> 3 #include <fcntl.h> 4 #include <limits.h> 5 #include <signal.h> 6 #include <stddef.h> 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <unistd.h> 11 12 #include "vm_util.h" 13 #include "hugepage_settings.h" 14 15 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" 16 #define MAX_SETTINGS_DEPTH 4 17 static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; 18 static int settings_index; 19 static struct thp_settings saved_settings; 20 static char dev_queue_read_ahead_path[PATH_MAX]; 21 static bool thp_settings_saved; 22 23 static const char * const thp_enabled_strings[] = { 24 "never", 25 "always", 26 "inherit", 27 "madvise", 28 NULL 29 }; 30 31 static const char * const thp_defrag_strings[] = { 32 "always", 33 "defer", 34 "defer+madvise", 35 "madvise", 36 "never", 37 NULL 38 }; 39 40 static const char * const shmem_enabled_strings[] = { 41 "never", 42 "always", 43 "within_size", 44 "advise", 45 "inherit", 46 "deny", 47 "force", 48 NULL 49 }; 50 51 int thp_read_string(const char *name, const char * const strings[]) 52 { 53 char path[PATH_MAX]; 54 char buf[256]; 55 char *c; 56 int ret; 57 58 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 59 if (ret >= PATH_MAX) { 60 printf("%s: Pathname is too long\n", __func__); 61 exit(EXIT_FAILURE); 62 } 63 64 if (!read_file(path, buf, sizeof(buf))) { 65 perror(path); 66 exit(EXIT_FAILURE); 67 } 68 69 c = strchr(buf, '['); 70 if (!c) { 71 printf("%s: Parse failure\n", __func__); 72 exit(EXIT_FAILURE); 73 } 74 75 c++; 76 memmove(buf, c, sizeof(buf) - (c - buf)); 77 78 c = strchr(buf, ']'); 79 if (!c) { 80 printf("%s: Parse failure\n", __func__); 81 exit(EXIT_FAILURE); 82 } 83 *c = '\0'; 84 85 ret = 0; 86 while (strings[ret]) { 87 if (!strcmp(strings[ret], buf)) 88 return ret; 89 ret++; 90 } 91 92 printf("Failed to parse %s\n", name); 93 exit(EXIT_FAILURE); 94 } 95 96 void thp_write_string(const char *name, const char *val) 97 { 98 char path[PATH_MAX]; 99 int ret; 100 101 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 102 if (ret >= PATH_MAX) { 103 printf("%s: Pathname is too long\n", __func__); 104 exit(EXIT_FAILURE); 105 } 106 write_file(path, val, strlen(val) + 1); 107 } 108 109 unsigned long thp_read_num(const char *name) 110 { 111 char path[PATH_MAX]; 112 int ret; 113 114 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 115 if (ret >= PATH_MAX) { 116 printf("%s: Pathname is too long\n", __func__); 117 exit(EXIT_FAILURE); 118 } 119 return read_num(path); 120 } 121 122 void thp_write_num(const char *name, unsigned long num) 123 { 124 char path[PATH_MAX]; 125 int ret; 126 127 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 128 if (ret >= PATH_MAX) { 129 printf("%s: Pathname is too long\n", __func__); 130 exit(EXIT_FAILURE); 131 } 132 write_num(path, num); 133 } 134 135 void thp_read_settings(struct thp_settings *settings) 136 { 137 unsigned long orders = thp_supported_orders(); 138 unsigned long shmem_orders = thp_shmem_supported_orders(); 139 char path[PATH_MAX]; 140 int i; 141 142 *settings = (struct thp_settings) { 143 .thp_enabled = thp_read_string("enabled", thp_enabled_strings), 144 .thp_defrag = thp_read_string("defrag", thp_defrag_strings), 145 .shmem_enabled = 146 thp_read_string("shmem_enabled", shmem_enabled_strings), 147 .use_zero_page = thp_read_num("use_zero_page"), 148 }; 149 settings->khugepaged = (struct khugepaged_settings) { 150 .defrag = thp_read_num("khugepaged/defrag"), 151 .alloc_sleep_millisecs = 152 thp_read_num("khugepaged/alloc_sleep_millisecs"), 153 .scan_sleep_millisecs = 154 thp_read_num("khugepaged/scan_sleep_millisecs"), 155 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"), 156 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"), 157 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"), 158 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"), 159 }; 160 if (dev_queue_read_ahead_path[0]) 161 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path); 162 163 for (i = 0; i < NR_ORDERS; i++) { 164 if (!((1 << i) & orders)) { 165 settings->hugepages[i].enabled = THP_NEVER; 166 continue; 167 } 168 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 169 (getpagesize() >> 10) << i); 170 settings->hugepages[i].enabled = 171 thp_read_string(path, thp_enabled_strings); 172 } 173 174 for (i = 0; i < NR_ORDERS; i++) { 175 if (!((1 << i) & shmem_orders)) { 176 settings->shmem_hugepages[i].enabled = SHMEM_NEVER; 177 continue; 178 } 179 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 180 (getpagesize() >> 10) << i); 181 settings->shmem_hugepages[i].enabled = 182 thp_read_string(path, shmem_enabled_strings); 183 } 184 } 185 186 void thp_write_settings(struct thp_settings *settings) 187 { 188 struct khugepaged_settings *khugepaged = &settings->khugepaged; 189 unsigned long orders = thp_supported_orders(); 190 unsigned long shmem_orders = thp_shmem_supported_orders(); 191 char path[PATH_MAX]; 192 int enabled; 193 int i; 194 195 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]); 196 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]); 197 thp_write_string("shmem_enabled", 198 shmem_enabled_strings[settings->shmem_enabled]); 199 thp_write_num("use_zero_page", settings->use_zero_page); 200 201 thp_write_num("khugepaged/defrag", khugepaged->defrag); 202 thp_write_num("khugepaged/alloc_sleep_millisecs", 203 khugepaged->alloc_sleep_millisecs); 204 thp_write_num("khugepaged/scan_sleep_millisecs", 205 khugepaged->scan_sleep_millisecs); 206 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); 207 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); 208 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); 209 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); 210 211 if (dev_queue_read_ahead_path[0]) 212 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb); 213 214 for (i = 0; i < NR_ORDERS; i++) { 215 if (!((1 << i) & orders)) 216 continue; 217 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 218 (getpagesize() >> 10) << i); 219 enabled = settings->hugepages[i].enabled; 220 thp_write_string(path, thp_enabled_strings[enabled]); 221 } 222 223 for (i = 0; i < NR_ORDERS; i++) { 224 if (!((1 << i) & shmem_orders)) 225 continue; 226 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 227 (getpagesize() >> 10) << i); 228 enabled = settings->shmem_hugepages[i].enabled; 229 thp_write_string(path, shmem_enabled_strings[enabled]); 230 } 231 } 232 233 struct thp_settings *thp_current_settings(void) 234 { 235 if (!settings_index) { 236 printf("Fail: No settings set"); 237 exit(EXIT_FAILURE); 238 } 239 return settings_stack + settings_index - 1; 240 } 241 242 void thp_push_settings(struct thp_settings *settings) 243 { 244 if (settings_index >= MAX_SETTINGS_DEPTH) { 245 printf("Fail: Settings stack exceeded"); 246 exit(EXIT_FAILURE); 247 } 248 settings_stack[settings_index++] = *settings; 249 thp_write_settings(thp_current_settings()); 250 } 251 252 void thp_pop_settings(void) 253 { 254 if (settings_index <= 0) { 255 printf("Fail: Settings stack empty"); 256 exit(EXIT_FAILURE); 257 } 258 --settings_index; 259 thp_write_settings(thp_current_settings()); 260 } 261 262 void thp_restore_settings(void) 263 { 264 if (thp_settings_saved) 265 thp_write_settings(&saved_settings); 266 } 267 268 static void __thp_save_settings(void) 269 { 270 if (!thp_available()) 271 return; 272 273 if (thp_settings_saved) 274 return; 275 276 thp_read_settings(&saved_settings); 277 thp_settings_saved = true; 278 } 279 280 void thp_set_read_ahead_path(char *path) 281 { 282 if (!path) { 283 dev_queue_read_ahead_path[0] = '\0'; 284 return; 285 } 286 287 strncpy(dev_queue_read_ahead_path, path, 288 sizeof(dev_queue_read_ahead_path)); 289 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0'; 290 } 291 292 static unsigned long __thp_supported_orders(bool is_shmem) 293 { 294 unsigned long orders = 0; 295 char path[PATH_MAX]; 296 char buf[256]; 297 int ret, i; 298 char anon_dir[] = "enabled"; 299 char shmem_dir[] = "shmem_enabled"; 300 301 for (i = 0; i < NR_ORDERS; i++) { 302 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s", 303 (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir); 304 if (ret >= PATH_MAX) { 305 printf("%s: Pathname is too long\n", __func__); 306 exit(EXIT_FAILURE); 307 } 308 309 ret = read_file(path, buf, sizeof(buf)); 310 if (ret) 311 orders |= 1UL << i; 312 } 313 314 return orders; 315 } 316 317 unsigned long thp_supported_orders(void) 318 { 319 return __thp_supported_orders(false); 320 } 321 322 unsigned long thp_shmem_supported_orders(void) 323 { 324 return __thp_supported_orders(true); 325 } 326 327 bool thp_available(void) 328 { 329 if (access(THP_SYSFS, F_OK) != 0) 330 return false; 331 return true; 332 } 333 334 bool thp_is_enabled(void) 335 { 336 if (!thp_available()) 337 return false; 338 339 int mode = thp_read_string("enabled", thp_enabled_strings); 340 341 /* THP is considered enabled if it's either "always" or "madvise" */ 342 return mode == 1 || mode == 3; 343 } 344 345 #define HUGETLB_MAX_NR_PAGESIZES 10 346 struct hugetlb_settings { 347 unsigned long nr_hugepages[HUGETLB_MAX_NR_PAGESIZES]; 348 unsigned long sizes[HUGETLB_MAX_NR_PAGESIZES]; 349 unsigned long default_size; 350 int nr_sizes; 351 }; 352 353 static struct hugetlb_settings hugetlb_saved_settings; 354 static bool hugetlb_settings_saved; 355 356 int detect_hugetlb_page_sizes(unsigned long sizes[], int max) 357 { 358 static struct hugetlb_settings *settings = &hugetlb_saved_settings; 359 DIR *dir; 360 int count = 0; 361 362 if (settings->nr_sizes) { 363 if (settings->nr_sizes < max) 364 max = settings->nr_sizes; 365 for (count = 0; count < max; count++) 366 sizes[count] = settings->sizes[count]; 367 return count; 368 } 369 370 dir = opendir("/sys/kernel/mm/hugepages/"); 371 if (!dir) 372 return 0; 373 374 while (count < max) { 375 struct dirent *entry = readdir(dir); 376 size_t kb; 377 378 if (!entry) 379 break; 380 if (entry->d_type != DT_DIR) 381 continue; 382 if (sscanf(entry->d_name, "hugepages-%zukB", &kb) != 1) 383 continue; 384 sizes[count++] = kb * 1024; 385 ksft_print_msg("[INFO] detected hugetlb page size: %zu KiB\n", 386 kb); 387 } 388 closedir(dir); 389 return count; 390 } 391 392 unsigned long default_huge_page_size(void) 393 { 394 static struct hugetlb_settings *settings = &hugetlb_saved_settings; 395 unsigned long hps = 0; 396 char *line = NULL; 397 size_t linelen = 0; 398 FILE *f; 399 400 if (settings->default_size) 401 return settings->default_size; 402 403 f = fopen("/proc/meminfo", "r"); 404 if (!f) 405 return 0; 406 while (getline(&line, &linelen, f) > 0) { 407 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { 408 hps <<= 10; 409 break; 410 } 411 } 412 413 free(line); 414 fclose(f); 415 return hps; 416 } 417 418 static void hugetlb_sysfs_path(char *buf, size_t buflen, 419 unsigned long size, const char *attr) 420 { 421 snprintf(buf, buflen, "/sys/kernel/mm/hugepages/hugepages-%lukB/%s", 422 size / 1024, attr); 423 } 424 425 unsigned long hugetlb_nr_pages(unsigned long size) 426 { 427 char path[PATH_MAX]; 428 429 hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); 430 431 return read_num(path); 432 } 433 434 void hugetlb_set_nr_pages(unsigned long size, unsigned long nr) 435 { 436 char path[PATH_MAX]; 437 438 hugetlb_sysfs_path(path, sizeof(path), size, "nr_hugepages"); 439 440 write_num(path, nr); 441 } 442 443 unsigned long hugetlb_free_pages(unsigned long size) 444 { 445 char path[PATH_MAX]; 446 447 hugetlb_sysfs_path(path, sizeof(path), size, "free_hugepages"); 448 449 return read_num(path); 450 } 451 452 static bool __hugetlb_setup(unsigned long size, unsigned long nr) 453 { 454 unsigned long free = hugetlb_free_pages(size); 455 unsigned long total = hugetlb_nr_pages(size); 456 457 if (free >= nr) 458 return true; 459 460 hugetlb_set_nr_pages(size, total + (nr - free)); 461 462 return hugetlb_free_pages(size) >= nr; 463 } 464 465 bool hugetlb_setup_default(unsigned long nr) 466 { 467 unsigned long size; 468 469 hugetlb_save_settings(); 470 size = default_huge_page_size(); 471 if (!size) 472 return false; 473 474 return __hugetlb_setup(size, nr); 475 } 476 477 bool hugetlb_setup_default_exact(unsigned long nr) 478 { 479 unsigned long size; 480 481 hugetlb_save_settings(); 482 size = default_huge_page_size(); 483 if (!size) 484 return false; 485 486 hugetlb_set_nr_pages(size, nr); 487 488 return hugetlb_free_pages(size) == nr; 489 } 490 491 unsigned long hugetlb_setup(unsigned long nr, unsigned long sizes[], 492 int max) 493 { 494 unsigned long enabled[10]; 495 int nr_sizes = 0; 496 int nr_enabled; 497 498 hugetlb_save_settings(); 499 500 nr_enabled = detect_hugetlb_page_sizes(enabled, ARRAY_SIZE(enabled)); 501 if (!nr_enabled) 502 return 0; 503 504 if (nr_enabled > max) { 505 ksft_print_msg("detected %d huge page sizes, will only test %d\n", nr_enabled, max); 506 nr_enabled = max; 507 } 508 509 /* request nr HugeTLB pages of every size. */ 510 for (int i = 0; i < nr_enabled; i++) { 511 if (!__hugetlb_setup(enabled[i], nr)) 512 continue; 513 sizes[nr_sizes++] = enabled[i]; 514 } 515 516 return nr_sizes; 517 } 518 519 static void __hugetlb_save_settings(void) 520 { 521 struct hugetlb_settings *settings = &hugetlb_saved_settings; 522 int nr_sizes; 523 524 if (hugetlb_settings_saved) 525 return; 526 527 settings->default_size = default_huge_page_size(); 528 if (!settings->default_size) 529 return; 530 531 nr_sizes = detect_hugetlb_page_sizes(settings->sizes, 532 HUGETLB_MAX_NR_PAGESIZES); 533 if (!nr_sizes) { 534 settings->default_size = 0; 535 return; 536 } 537 538 for (int i = 0; i < nr_sizes; i++) { 539 unsigned long sz = settings->sizes[i]; 540 541 if (!sz) 542 continue; 543 settings->nr_hugepages[i] = hugetlb_nr_pages(sz); 544 } 545 546 settings->nr_sizes = nr_sizes; 547 hugetlb_settings_saved = true; 548 } 549 550 void hugetlb_restore_settings(void) 551 { 552 struct hugetlb_settings *settings = &hugetlb_saved_settings; 553 554 if (!hugetlb_settings_saved || !settings->default_size) 555 return; 556 557 for (int i = 0; i < HUGETLB_MAX_NR_PAGESIZES; i++) { 558 unsigned long sz = settings->sizes[i]; 559 560 if (!sz) 561 continue; 562 563 hugetlb_set_nr_pages(sz, settings->nr_hugepages[i]); 564 } 565 } 566 567 static void hugepage_restore_settings_atexit(void) 568 { 569 if (thp_settings_saved) 570 thp_restore_settings(); 571 if (hugetlb_settings_saved) 572 hugetlb_restore_settings(); 573 } 574 575 static void hugepage_restore_settings_sighandler(int sig) 576 { 577 /* exit() will invoke the hugepage_restore_settings_atexit handler. */ 578 exit(KSFT_FAIL); 579 } 580 581 void hugepage_save_settings(bool thp, bool hugetlb) 582 { 583 if (!thp && !hugetlb) 584 return; 585 586 if (thp) 587 __thp_save_settings(); 588 if (hugetlb) 589 __hugetlb_save_settings(); 590 591 /* 592 * setup exit hooks to make sure THP and HugeTLB settings are 593 * restored on graceful and error exits and signals 594 */ 595 atexit(hugepage_restore_settings_atexit); 596 signal(SIGTERM, hugepage_restore_settings_sighandler); 597 signal(SIGINT, hugepage_restore_settings_sighandler); 598 signal(SIGHUP, hugepage_restore_settings_sighandler); 599 signal(SIGQUIT, hugepage_restore_settings_sighandler); 600 } 601