1 // SPDX-License-Identifier: GPL-2.0 2 #include <fcntl.h> 3 #include <limits.h> 4 #include <stdio.h> 5 #include <stdlib.h> 6 #include <string.h> 7 #include <unistd.h> 8 9 #include "vm_util.h" 10 #include "thp_settings.h" 11 12 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" 13 #define MAX_SETTINGS_DEPTH 4 14 static struct thp_settings settings_stack[MAX_SETTINGS_DEPTH]; 15 static int settings_index; 16 static struct thp_settings saved_settings; 17 static char dev_queue_read_ahead_path[PATH_MAX]; 18 19 static const char * const thp_enabled_strings[] = { 20 "never", 21 "always", 22 "inherit", 23 "madvise", 24 NULL 25 }; 26 27 static const char * const thp_defrag_strings[] = { 28 "always", 29 "defer", 30 "defer+madvise", 31 "madvise", 32 "never", 33 NULL 34 }; 35 36 static const char * const shmem_enabled_strings[] = { 37 "never", 38 "always", 39 "within_size", 40 "advise", 41 "inherit", 42 "deny", 43 "force", 44 NULL 45 }; 46 47 int read_file(const char *path, char *buf, size_t buflen) 48 { 49 int fd; 50 ssize_t numread; 51 52 fd = open(path, O_RDONLY); 53 if (fd == -1) 54 return 0; 55 56 numread = read(fd, buf, buflen - 1); 57 if (numread < 1) { 58 close(fd); 59 return 0; 60 } 61 62 buf[numread] = '\0'; 63 close(fd); 64 65 return (unsigned int) numread; 66 } 67 68 unsigned long read_num(const char *path) 69 { 70 char buf[21]; 71 72 if (read_file(path, buf, sizeof(buf)) < 0) { 73 perror("read_file()"); 74 exit(EXIT_FAILURE); 75 } 76 77 return strtoul(buf, NULL, 10); 78 } 79 80 void write_num(const char *path, unsigned long num) 81 { 82 char buf[21]; 83 84 sprintf(buf, "%ld", num); 85 write_file(path, buf, strlen(buf) + 1); 86 } 87 88 int thp_read_string(const char *name, const char * const strings[]) 89 { 90 char path[PATH_MAX]; 91 char buf[256]; 92 char *c; 93 int ret; 94 95 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 96 if (ret >= PATH_MAX) { 97 printf("%s: Pathname is too long\n", __func__); 98 exit(EXIT_FAILURE); 99 } 100 101 if (!read_file(path, buf, sizeof(buf))) { 102 perror(path); 103 exit(EXIT_FAILURE); 104 } 105 106 c = strchr(buf, '['); 107 if (!c) { 108 printf("%s: Parse failure\n", __func__); 109 exit(EXIT_FAILURE); 110 } 111 112 c++; 113 memmove(buf, c, sizeof(buf) - (c - buf)); 114 115 c = strchr(buf, ']'); 116 if (!c) { 117 printf("%s: Parse failure\n", __func__); 118 exit(EXIT_FAILURE); 119 } 120 *c = '\0'; 121 122 ret = 0; 123 while (strings[ret]) { 124 if (!strcmp(strings[ret], buf)) 125 return ret; 126 ret++; 127 } 128 129 printf("Failed to parse %s\n", name); 130 exit(EXIT_FAILURE); 131 } 132 133 void thp_write_string(const char *name, const char *val) 134 { 135 char path[PATH_MAX]; 136 int ret; 137 138 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 139 if (ret >= PATH_MAX) { 140 printf("%s: Pathname is too long\n", __func__); 141 exit(EXIT_FAILURE); 142 } 143 write_file(path, val, strlen(val) + 1); 144 } 145 146 unsigned long thp_read_num(const char *name) 147 { 148 char path[PATH_MAX]; 149 int ret; 150 151 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 152 if (ret >= PATH_MAX) { 153 printf("%s: Pathname is too long\n", __func__); 154 exit(EXIT_FAILURE); 155 } 156 return read_num(path); 157 } 158 159 void thp_write_num(const char *name, unsigned long num) 160 { 161 char path[PATH_MAX]; 162 int ret; 163 164 ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); 165 if (ret >= PATH_MAX) { 166 printf("%s: Pathname is too long\n", __func__); 167 exit(EXIT_FAILURE); 168 } 169 write_num(path, num); 170 } 171 172 void thp_read_settings(struct thp_settings *settings) 173 { 174 unsigned long orders = thp_supported_orders(); 175 unsigned long shmem_orders = thp_shmem_supported_orders(); 176 char path[PATH_MAX]; 177 int i; 178 179 *settings = (struct thp_settings) { 180 .thp_enabled = thp_read_string("enabled", thp_enabled_strings), 181 .thp_defrag = thp_read_string("defrag", thp_defrag_strings), 182 .shmem_enabled = 183 thp_read_string("shmem_enabled", shmem_enabled_strings), 184 .use_zero_page = thp_read_num("use_zero_page"), 185 }; 186 settings->khugepaged = (struct khugepaged_settings) { 187 .defrag = thp_read_num("khugepaged/defrag"), 188 .alloc_sleep_millisecs = 189 thp_read_num("khugepaged/alloc_sleep_millisecs"), 190 .scan_sleep_millisecs = 191 thp_read_num("khugepaged/scan_sleep_millisecs"), 192 .max_ptes_none = thp_read_num("khugepaged/max_ptes_none"), 193 .max_ptes_swap = thp_read_num("khugepaged/max_ptes_swap"), 194 .max_ptes_shared = thp_read_num("khugepaged/max_ptes_shared"), 195 .pages_to_scan = thp_read_num("khugepaged/pages_to_scan"), 196 }; 197 if (dev_queue_read_ahead_path[0]) 198 settings->read_ahead_kb = read_num(dev_queue_read_ahead_path); 199 200 for (i = 0; i < NR_ORDERS; i++) { 201 if (!((1 << i) & orders)) { 202 settings->hugepages[i].enabled = THP_NEVER; 203 continue; 204 } 205 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 206 (getpagesize() >> 10) << i); 207 settings->hugepages[i].enabled = 208 thp_read_string(path, thp_enabled_strings); 209 } 210 211 for (i = 0; i < NR_ORDERS; i++) { 212 if (!((1 << i) & shmem_orders)) { 213 settings->shmem_hugepages[i].enabled = SHMEM_NEVER; 214 continue; 215 } 216 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 217 (getpagesize() >> 10) << i); 218 settings->shmem_hugepages[i].enabled = 219 thp_read_string(path, shmem_enabled_strings); 220 } 221 } 222 223 void thp_write_settings(struct thp_settings *settings) 224 { 225 struct khugepaged_settings *khugepaged = &settings->khugepaged; 226 unsigned long orders = thp_supported_orders(); 227 unsigned long shmem_orders = thp_shmem_supported_orders(); 228 char path[PATH_MAX]; 229 int enabled; 230 int i; 231 232 thp_write_string("enabled", thp_enabled_strings[settings->thp_enabled]); 233 thp_write_string("defrag", thp_defrag_strings[settings->thp_defrag]); 234 thp_write_string("shmem_enabled", 235 shmem_enabled_strings[settings->shmem_enabled]); 236 thp_write_num("use_zero_page", settings->use_zero_page); 237 238 thp_write_num("khugepaged/defrag", khugepaged->defrag); 239 thp_write_num("khugepaged/alloc_sleep_millisecs", 240 khugepaged->alloc_sleep_millisecs); 241 thp_write_num("khugepaged/scan_sleep_millisecs", 242 khugepaged->scan_sleep_millisecs); 243 thp_write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none); 244 thp_write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); 245 thp_write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); 246 thp_write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); 247 248 if (dev_queue_read_ahead_path[0]) 249 write_num(dev_queue_read_ahead_path, settings->read_ahead_kb); 250 251 for (i = 0; i < NR_ORDERS; i++) { 252 if (!((1 << i) & orders)) 253 continue; 254 snprintf(path, PATH_MAX, "hugepages-%ukB/enabled", 255 (getpagesize() >> 10) << i); 256 enabled = settings->hugepages[i].enabled; 257 thp_write_string(path, thp_enabled_strings[enabled]); 258 } 259 260 for (i = 0; i < NR_ORDERS; i++) { 261 if (!((1 << i) & shmem_orders)) 262 continue; 263 snprintf(path, PATH_MAX, "hugepages-%ukB/shmem_enabled", 264 (getpagesize() >> 10) << i); 265 enabled = settings->shmem_hugepages[i].enabled; 266 thp_write_string(path, shmem_enabled_strings[enabled]); 267 } 268 } 269 270 struct thp_settings *thp_current_settings(void) 271 { 272 if (!settings_index) { 273 printf("Fail: No settings set"); 274 exit(EXIT_FAILURE); 275 } 276 return settings_stack + settings_index - 1; 277 } 278 279 void thp_push_settings(struct thp_settings *settings) 280 { 281 if (settings_index >= MAX_SETTINGS_DEPTH) { 282 printf("Fail: Settings stack exceeded"); 283 exit(EXIT_FAILURE); 284 } 285 settings_stack[settings_index++] = *settings; 286 thp_write_settings(thp_current_settings()); 287 } 288 289 void thp_pop_settings(void) 290 { 291 if (settings_index <= 0) { 292 printf("Fail: Settings stack empty"); 293 exit(EXIT_FAILURE); 294 } 295 --settings_index; 296 thp_write_settings(thp_current_settings()); 297 } 298 299 void thp_restore_settings(void) 300 { 301 thp_write_settings(&saved_settings); 302 } 303 304 void thp_save_settings(void) 305 { 306 thp_read_settings(&saved_settings); 307 } 308 309 void thp_set_read_ahead_path(char *path) 310 { 311 if (!path) { 312 dev_queue_read_ahead_path[0] = '\0'; 313 return; 314 } 315 316 strncpy(dev_queue_read_ahead_path, path, 317 sizeof(dev_queue_read_ahead_path)); 318 dev_queue_read_ahead_path[sizeof(dev_queue_read_ahead_path) - 1] = '\0'; 319 } 320 321 static unsigned long __thp_supported_orders(bool is_shmem) 322 { 323 unsigned long orders = 0; 324 char path[PATH_MAX]; 325 char buf[256]; 326 int ret, i; 327 char anon_dir[] = "enabled"; 328 char shmem_dir[] = "shmem_enabled"; 329 330 for (i = 0; i < NR_ORDERS; i++) { 331 ret = snprintf(path, PATH_MAX, THP_SYSFS "hugepages-%ukB/%s", 332 (getpagesize() >> 10) << i, is_shmem ? shmem_dir : anon_dir); 333 if (ret >= PATH_MAX) { 334 printf("%s: Pathname is too long\n", __func__); 335 exit(EXIT_FAILURE); 336 } 337 338 ret = read_file(path, buf, sizeof(buf)); 339 if (ret) 340 orders |= 1UL << i; 341 } 342 343 return orders; 344 } 345 346 unsigned long thp_supported_orders(void) 347 { 348 return __thp_supported_orders(false); 349 } 350 351 unsigned long thp_shmem_supported_orders(void) 352 { 353 return __thp_supported_orders(true); 354 } 355 356 bool thp_available(void) 357 { 358 if (access(THP_SYSFS, F_OK) != 0) 359 return false; 360 return true; 361 } 362 363 bool thp_is_enabled(void) 364 { 365 if (!thp_available()) 366 return false; 367 368 int mode = thp_read_string("enabled", thp_enabled_strings); 369 370 /* THP is considered enabled if it's either "always" or "madvise" */ 371 return mode == 1 || mode == 3; 372 } 373