1 // SPDX-License-Identifier: GPL-2.0 2 #include <fcntl.h> 3 #include <limits.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <unistd.h> 8 9 #include "thp_settings.h" 10 11 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" 12 #define MAX_SETTINGS_DEPTH 4 13 static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; 14 static int settings_index; 15 static struct thp_settings saved_settings; 16 static char dev_queue_read_ahead_path[PATH_MAX]; 17 18 static const char * const thp_enabled_strings[] = { 19 "never", 20 "always", 21 "inherit", 22 "madvise", 23 NULL 24 }; 25 26 static const char * const thp_defrag_strings[] = { 27 "always", 28 "defer", 29 "defer+madvise", 30 "madvise", 31 "never", 32 NULL 33 }; 34 35 static const char * const shmem_enabled_strings[] = { 36 "never", 37 "always", 38 "within_size", 39 "advise", 40 "inherit", 41 "deny", 42 "force", 43 NULL 44 }; 45 46 int read_file(const char *path, char *buf, size_t buflen) 47 { 48 int fd; 49 ssize_t numread; 50 51 fd = open(path, O_RDONLY); 52 if (fd == -1) 53 return 0; 54 55 numread = read(fd, buf, buflen - 1); 56 if (numread < 1) { 57 close(fd); 58 return 0; 59 } 60 61 buf[numread] = '\0'; 62 close(fd); 63 64 return (unsigned int) numread; 65 } 66 67 int write_file(const char *path, const char *buf, size_t buflen) 68 { 69 int fd; 70 ssize_t numwritten; 71 72 fd = open(path, O_WRONLY); 73 if (fd == -1) { 74 printf("open(%s)\n", path); 75 exit(EXIT_FAILURE); 76 return 0; 77 } 78 79 numwritten = write(fd, buf, buflen - 1); 80 close(fd); 81 if (numwritten < 1) { 82 printf("write(%s)\n", buf); 83 exit(EXIT_FAILURE); 84 return 0; 85 } 86 87 return (unsigned int) numwritten; 88 } 89 90 const unsigned long read_num(const char *path) 91 { 92 char buf[21]; 93 94 if (read_file(path, buf, sizeof(buf)) < 0) { 95 perror("read_file()"); 96 exit(EXIT_FAILURE); 97 } 98 99 return strtoul(buf, NULL, 10); 100 } 101 102 void write_num(const char *path, unsigned long num) 103 { 104 char buf[21]; 105 106 sprintf(buf, "%ld", num); 107 if (!write_file(path, buf, strlen(buf) + 1)) { 108 perror(path); 109 exit(EXIT_FAILURE); 110 } 111 } 112 113 int thp_read_string(const char *name, const char * const strings[]) 114 { 115 char path[PATH_MAX]; 116 char buf[256]; 117 char *c; 118 int ret; 119 120 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 121 if (ret >= PATH_MAX) { 122 printf("%s: Pathname is too long\n", __func__); 123 exit(EXIT_FAILURE); 124 } 125 126 if (!read_file(path, buf, sizeof(buf))) { 127 perror(path); 128 exit(EXIT_FAILURE); 129 } 130 131 c = strchr(buf, '['); 132 if (!c) { 133 printf("%s: Parse failure\n", __func__); 134 exit(EXIT_FAILURE); 135 } 136 137 c++; 138 memmove(buf, c, sizeof(buf) - (c - buf)); 139 140 c = strchr(buf, ']'); 141 if (!c) { 142 printf("%s: Parse failure\n", __func__); 143 exit(EXIT_FAILURE); 144 } 145 *c = '\0'; 146 147 ret = 0; 148 while (strings[ret]) { 149 if (!strcmp(strings[ret], buf)) 150 return ret; 151 ret++; 152 } 153 154 printf("Failed to parse %s\n", name); 155 exit(EXIT_FAILURE); 156 } 157 158 void thp_write_string(const char *name, const char *val) 159 { 160 char path[PATH_MAX]; 161 int ret; 162 163 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 164 if (ret >= PATH_MAX) { 165 printf("%s: Pathname is too long\n", __func__); 166 exit(EXIT_FAILURE); 167 } 168 169 if (!write_file(path, val, strlen(val) + 1)) { 170 perror(path); 171 exit(EXIT_FAILURE); 172 } 173 } 174 175 const unsigned long thp_read_num(const char *name) 176 { 177 char path[PATH_MAX]; 178 int ret; 179 180 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 181 if (ret >= PATH_MAX) { 182 printf("%s: Pathname is too long\n", __func__); 183 exit(EXIT_FAILURE); 184 } 185 return read_num(path); 186 } 187 188 void thp_write_num(const char *name, unsigned long num) 189 { 190 char path[PATH_MAX]; 191 int ret; 192 193 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 194 if (ret >= PATH_MAX) { 195 printf("%s: Pathname is too long\n", __func__); 196 exit(EXIT_FAILURE); 197 } 198 write_num(path, num); 199 } 200 201 void thp_read_settings(struct thp_settings *settings) 202 { 203 unsigned long orders = thp_supported_orders(); 204 unsigned long shmem_orders = thp_shmem_supported_orders(); 205 char path[PATH_MAX]; 206 int i; 207 208 *settings = (struct thp_settings) { 209 .thp_enabled = thp_read_string("enabled", thp_enabled_strings), 210 .thp_defrag = thp_read_string("defrag", thp_defrag_strings), 211 .shmem_enabled = 212 thp_read_string("shmem_enabled", shmem_enabled_strings), 213 .use_zero_page = thp_read_num("use_zero_page"), 214 }; 215 settings->khugepaged = (struct khugepaged_settings) { 216 .defrag = thp_read_num("khugepaged/defrag"), 217 .alloc_sleep_millisecs = 218 thp_read_num("khugepaged/alloc_sleep_millisecs"), 219 .scan_sleep_millisecs = 220 thp_read_num("khugepaged/scan_sleep_millisecs"), 221 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"), 222 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"), 223 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"), 224 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"), 225 }; 226 if (dev_queue_read_ahead_path[0]) 227 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path); 228 229 for (i = 0; i < NR_ORDERS; i++) { 230 if (!((1 << i) & orders)) { 231 settings->hugepages[i].enabled = THP_NEVER; 232 continue; 233 } 234 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 235 (getpagesize() >> 10) << i); 236 settings->hugepages[i].enabled = 237 thp_read_string(path, thp_enabled_strings); 238 } 239 240 for (i = 0; i < NR_ORDERS; i++) { 241 if (!((1 << i) & shmem_orders)) { 242 settings->shmem_hugepages[i].enabled = SHMEM_NEVER; 243 continue; 244 } 245 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 246 (getpagesize() >> 10) << i); 247 settings->shmem_hugepages[i].enabled = 248 thp_read_string(path, shmem_enabled_strings); 249 } 250 } 251 252 void thp_write_settings(struct thp_settings *settings) 253 { 254 struct khugepaged_settings *khugepaged = &settings->khugepaged; 255 unsigned long orders = thp_supported_orders(); 256 unsigned long shmem_orders = thp_shmem_supported_orders(); 257 char path[PATH_MAX]; 258 int enabled; 259 int i; 260 261 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]); 262 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]); 263 thp_write_string("shmem_enabled", 264 shmem_enabled_strings[settings->shmem_enabled]); 265 thp_write_num("use_zero_page", settings->use_zero_page); 266 267 thp_write_num("khugepaged/defrag", khugepaged->defrag); 268 thp_write_num("khugepaged/alloc_sleep_millisecs", 269 khugepaged->alloc_sleep_millisecs); 270 thp_write_num("khugepaged/scan_sleep_millisecs", 271 khugepaged->scan_sleep_millisecs); 272 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); 273 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); 274 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); 275 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); 276 277 if (dev_queue_read_ahead_path[0]) 278 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb); 279 280 for (i = 0; i < NR_ORDERS; i++) { 281 if (!((1 << i) & orders)) 282 continue; 283 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 284 (getpagesize() >> 10) << i); 285 enabled = settings->hugepages[i].enabled; 286 thp_write_string(path, thp_enabled_strings[enabled]); 287 } 288 289 for (i = 0; i < NR_ORDERS; i++) { 290 if (!((1 << i) & shmem_orders)) 291 continue; 292 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 293 (getpagesize() >> 10) << i); 294 enabled = settings->shmem_hugepages[i].enabled; 295 thp_write_string(path, shmem_enabled_strings[enabled]); 296 } 297 } 298 299 struct thp_settings *thp_current_settings(void) 300 { 301 if (!settings_index) { 302 printf("Fail: No settings set"); 303 exit(EXIT_FAILURE); 304 } 305 return settings_stack + settings_index - 1; 306 } 307 308 void thp_push_settings(struct thp_settings *settings) 309 { 310 if (settings_index >= MAX_SETTINGS_DEPTH) { 311 printf("Fail: Settings stack exceeded"); 312 exit(EXIT_FAILURE); 313 } 314 settings_stack[settings_index++] = *settings; 315 thp_write_settings(thp_current_settings()); 316 } 317 318 void thp_pop_settings(void) 319 { 320 if (settings_index <= 0) { 321 printf("Fail: Settings stack empty"); 322 exit(EXIT_FAILURE); 323 } 324 --settings_index; 325 thp_write_settings(thp_current_settings()); 326 } 327 328 void thp_restore_settings(void) 329 { 330 thp_write_settings(&saved_settings); 331 } 332 333 void thp_save_settings(void) 334 { 335 thp_read_settings(&saved_settings); 336 } 337 338 void thp_set_read_ahead_path(char *path) 339 { 340 if (!path) { 341 dev_queue_read_ahead_path[0] = '\0'; 342 return; 343 } 344 345 strncpy(dev_queue_read_ahead_path, path, 346 sizeof(dev_queue_read_ahead_path)); 347 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0'; 348 } 349 350 static unsigned long __thp_supported_orders(bool is_shmem) 351 { 352 unsigned long orders = 0; 353 char path[PATH_MAX]; 354 char buf[256]; 355 int ret, i; 356 char anon_dir[] = "enabled"; 357 char shmem_dir[] = "shmem_enabled"; 358 359 for (i = 0; i < NR_ORDERS; i++) { 360 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s", 361 (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir); 362 if (ret >= PATH_MAX) { 363 printf("%s: Pathname is too long\n", __func__); 364 exit(EXIT_FAILURE); 365 } 366 367 ret = read_file(path, buf, sizeof(buf)); 368 if (ret) 369 orders |= 1UL << i; 370 } 371 372 return orders; 373 } 374 375 unsigned long thp_supported_orders(void) 376 { 377 return __thp_supported_orders(false); 378 } 379 380 unsigned long thp_shmem_supported_orders(void) 381 { 382 return __thp_supported_orders(true); 383 } 384