1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * HugeTLB sysfs interfaces. 4 * (C) Nadia Yvette Chambers, April 2004 5 */ 6 7 #include <linux/swap.h> 8 #include <linux/page_owner.h> 9 #include <linux/page-isolation.h> 10 11 #include "hugetlb_vmemmap.h" 12 #include "hugetlb_internal.h" 13 14 #define HSTATE_ATTR_RO(_name) \ 15 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 16 17 #define HSTATE_ATTR_WO(_name) \ 18 static struct kobj_attribute _name##_attr = __ATTR_WO(_name) 19 20 #define HSTATE_ATTR(_name) \ 21 static struct kobj_attribute _name##_attr = __ATTR_RW(_name) 22 23 static struct kobject *hugepages_kobj; 24 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 25 26 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp); 27 28 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp) 29 { 30 int i; 31 32 for (i = 0; i < HUGE_MAX_HSTATE; i++) 33 if (hstate_kobjs[i] == kobj) { 34 if (nidp) 35 *nidp = NUMA_NO_NODE; 36 return &hstates[i]; 37 } 38 39 return kobj_to_node_hstate(kobj, nidp); 40 } 41 42 static ssize_t nr_hugepages_show_common(struct kobject *kobj, 43 struct kobj_attribute *attr, char *buf) 44 { 45 struct hstate *h; 46 unsigned long nr_huge_pages; 47 int nid; 48 49 h = kobj_to_hstate(kobj, &nid); 50 if (nid == NUMA_NO_NODE) 51 nr_huge_pages = h->nr_huge_pages; 52 else 53 nr_huge_pages = h->nr_huge_pages_node[nid]; 54 55 return sysfs_emit(buf, "%lu\n", nr_huge_pages); 56 } 57 58 static ssize_t nr_hugepages_store_common(bool obey_mempolicy, 59 struct kobject *kobj, const char *buf, 60 size_t len) 61 { 62 struct hstate *h; 63 unsigned long count; 64 int nid; 65 int err; 66 67 err = kstrtoul(buf, 10, &count); 68 if (err) 69 return err; 70 71 h = kobj_to_hstate(kobj, &nid); 72 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); 73 } 74 75 static ssize_t nr_hugepages_show(struct kobject *kobj, 76 struct kobj_attribute *attr, char *buf) 77 { 78 return nr_hugepages_show_common(kobj, attr, buf); 79 } 80 81 static ssize_t nr_hugepages_store(struct kobject *kobj, 82 struct kobj_attribute *attr, const char *buf, size_t len) 83 { 84 return nr_hugepages_store_common(false, kobj, buf, len); 85 } 86 HSTATE_ATTR(nr_hugepages); 87 88 #ifdef CONFIG_NUMA 89 90 /* 91 * hstate attribute for optionally mempolicy-based constraint on persistent 92 * huge page alloc/free. 93 */ 94 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, 95 struct kobj_attribute *attr, 96 char *buf) 97 { 98 return nr_hugepages_show_common(kobj, attr, buf); 99 } 100 101 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, 102 struct kobj_attribute *attr, const char *buf, size_t len) 103 { 104 return nr_hugepages_store_common(true, kobj, buf, len); 105 } 106 HSTATE_ATTR(nr_hugepages_mempolicy); 107 #endif 108 109 110 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, 111 struct kobj_attribute *attr, char *buf) 112 { 113 struct hstate *h = kobj_to_hstate(kobj, NULL); 114 return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages); 115 } 116 117 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, 118 struct kobj_attribute *attr, const char *buf, size_t count) 119 { 120 int err; 121 unsigned long input; 122 struct hstate *h = kobj_to_hstate(kobj, NULL); 123 124 if (hstate_is_gigantic_no_runtime(h)) 125 return -EINVAL; 126 127 err = kstrtoul(buf, 10, &input); 128 if (err) 129 return err; 130 131 spin_lock_irq(&hugetlb_lock); 132 h->nr_overcommit_huge_pages = input; 133 spin_unlock_irq(&hugetlb_lock); 134 135 return count; 136 } 137 HSTATE_ATTR(nr_overcommit_hugepages); 138 139 static ssize_t free_hugepages_show(struct kobject *kobj, 140 struct kobj_attribute *attr, char *buf) 141 { 142 struct hstate *h; 143 unsigned long free_huge_pages; 144 int nid; 145 146 h = kobj_to_hstate(kobj, &nid); 147 if (nid == NUMA_NO_NODE) 148 free_huge_pages = h->free_huge_pages; 149 else 150 free_huge_pages = h->free_huge_pages_node[nid]; 151 152 return sysfs_emit(buf, "%lu\n", free_huge_pages); 153 } 154 HSTATE_ATTR_RO(free_hugepages); 155 156 static ssize_t resv_hugepages_show(struct kobject *kobj, 157 struct kobj_attribute *attr, char *buf) 158 { 159 struct hstate *h = kobj_to_hstate(kobj, NULL); 160 return sysfs_emit(buf, "%lu\n", h->resv_huge_pages); 161 } 162 HSTATE_ATTR_RO(resv_hugepages); 163 164 static ssize_t surplus_hugepages_show(struct kobject *kobj, 165 struct kobj_attribute *attr, char *buf) 166 { 167 struct hstate *h; 168 unsigned long surplus_huge_pages; 169 int nid; 170 171 h = kobj_to_hstate(kobj, &nid); 172 if (nid == NUMA_NO_NODE) 173 surplus_huge_pages = h->surplus_huge_pages; 174 else 175 surplus_huge_pages = h->surplus_huge_pages_node[nid]; 176 177 return sysfs_emit(buf, "%lu\n", surplus_huge_pages); 178 } 179 HSTATE_ATTR_RO(surplus_hugepages); 180 181 static ssize_t demote_store(struct kobject *kobj, 182 struct kobj_attribute *attr, const char *buf, size_t len) 183 { 184 unsigned long nr_demote; 185 unsigned long nr_available; 186 nodemask_t nodes_allowed, *n_mask; 187 struct hstate *h; 188 int err; 189 int nid; 190 191 err = kstrtoul(buf, 10, &nr_demote); 192 if (err) 193 return err; 194 h = kobj_to_hstate(kobj, &nid); 195 196 if (nid != NUMA_NO_NODE) { 197 init_nodemask_of_node(&nodes_allowed, nid); 198 n_mask = &nodes_allowed; 199 } else { 200 n_mask = &node_states[N_MEMORY]; 201 } 202 203 /* Synchronize with other sysfs operations modifying huge pages */ 204 mutex_lock(&h->resize_lock); 205 spin_lock_irq(&hugetlb_lock); 206 207 while (nr_demote) { 208 long rc; 209 210 /* 211 * Check for available pages to demote each time thorough the 212 * loop as demote_pool_huge_page will drop hugetlb_lock. 213 */ 214 if (nid != NUMA_NO_NODE) 215 nr_available = h->free_huge_pages_node[nid]; 216 else 217 nr_available = h->free_huge_pages; 218 nr_available -= h->resv_huge_pages; 219 if (!nr_available) 220 break; 221 222 rc = demote_pool_huge_page(h, n_mask, nr_demote); 223 if (rc < 0) { 224 err = rc; 225 break; 226 } 227 228 nr_demote -= rc; 229 } 230 231 spin_unlock_irq(&hugetlb_lock); 232 mutex_unlock(&h->resize_lock); 233 234 if (err) 235 return err; 236 return len; 237 } 238 HSTATE_ATTR_WO(demote); 239 240 static ssize_t demote_size_show(struct kobject *kobj, 241 struct kobj_attribute *attr, char *buf) 242 { 243 struct hstate *h = kobj_to_hstate(kobj, NULL); 244 unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K; 245 246 return sysfs_emit(buf, "%lukB\n", demote_size); 247 } 248 249 static ssize_t demote_size_store(struct kobject *kobj, 250 struct kobj_attribute *attr, 251 const char *buf, size_t count) 252 { 253 struct hstate *h, *demote_hstate; 254 unsigned long demote_size; 255 unsigned int demote_order; 256 257 demote_size = (unsigned long)memparse(buf, NULL); 258 259 demote_hstate = size_to_hstate(demote_size); 260 if (!demote_hstate) 261 return -EINVAL; 262 demote_order = demote_hstate->order; 263 if (demote_order < HUGETLB_PAGE_ORDER) 264 return -EINVAL; 265 266 /* demote order must be smaller than hstate order */ 267 h = kobj_to_hstate(kobj, NULL); 268 if (demote_order >= h->order) 269 return -EINVAL; 270 271 /* resize_lock synchronizes access to demote size and writes */ 272 mutex_lock(&h->resize_lock); 273 h->demote_order = demote_order; 274 mutex_unlock(&h->resize_lock); 275 276 return count; 277 } 278 HSTATE_ATTR(demote_size); 279 280 static struct attribute *hstate_attrs[] = { 281 &nr_hugepages_attr.attr, 282 &nr_overcommit_hugepages_attr.attr, 283 &free_hugepages_attr.attr, 284 &resv_hugepages_attr.attr, 285 &surplus_hugepages_attr.attr, 286 #ifdef CONFIG_NUMA 287 &nr_hugepages_mempolicy_attr.attr, 288 #endif 289 NULL, 290 }; 291 292 static const struct attribute_group hstate_attr_group = { 293 .attrs = hstate_attrs, 294 }; 295 296 static struct attribute *hstate_demote_attrs[] = { 297 &demote_size_attr.attr, 298 &demote_attr.attr, 299 NULL, 300 }; 301 302 static const struct attribute_group hstate_demote_attr_group = { 303 .attrs = hstate_demote_attrs, 304 }; 305 306 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, 307 struct kobject **hstate_kobjs, 308 const struct attribute_group *hstate_attr_group) 309 { 310 int retval; 311 int hi = hstate_index(h); 312 313 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent); 314 if (!hstate_kobjs[hi]) 315 return -ENOMEM; 316 317 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group); 318 if (retval) { 319 kobject_put(hstate_kobjs[hi]); 320 hstate_kobjs[hi] = NULL; 321 return retval; 322 } 323 324 if (h->demote_order) { 325 retval = sysfs_create_group(hstate_kobjs[hi], 326 &hstate_demote_attr_group); 327 if (retval) { 328 pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name); 329 sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group); 330 kobject_put(hstate_kobjs[hi]); 331 hstate_kobjs[hi] = NULL; 332 return retval; 333 } 334 } 335 336 return 0; 337 } 338 339 #ifdef CONFIG_NUMA 340 static bool hugetlb_sysfs_initialized __ro_after_init; 341 342 /* 343 * node_hstate/s - associate per node hstate attributes, via their kobjects, 344 * with node devices in node_devices[] using a parallel array. The array 345 * index of a node device or _hstate == node id. 346 * This is here to avoid any static dependency of the node device driver, in 347 * the base kernel, on the hugetlb module. 348 */ 349 struct node_hstate { 350 struct kobject *hugepages_kobj; 351 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE]; 352 }; 353 static struct node_hstate node_hstates[MAX_NUMNODES]; 354 355 /* 356 * A subset of global hstate attributes for node devices 357 */ 358 static struct attribute *per_node_hstate_attrs[] = { 359 &nr_hugepages_attr.attr, 360 &free_hugepages_attr.attr, 361 &surplus_hugepages_attr.attr, 362 NULL, 363 }; 364 365 static const struct attribute_group per_node_hstate_attr_group = { 366 .attrs = per_node_hstate_attrs, 367 }; 368 369 /* 370 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj. 371 * Returns node id via non-NULL nidp. 372 */ 373 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 374 { 375 int nid; 376 377 for (nid = 0; nid < nr_node_ids; nid++) { 378 struct node_hstate *nhs = &node_hstates[nid]; 379 int i; 380 for (i = 0; i < HUGE_MAX_HSTATE; i++) 381 if (nhs->hstate_kobjs[i] == kobj) { 382 if (nidp) 383 *nidp = nid; 384 return &hstates[i]; 385 } 386 } 387 388 BUG(); 389 return NULL; 390 } 391 392 /* 393 * Unregister hstate attributes from a single node device. 394 * No-op if no hstate attributes attached. 395 */ 396 void hugetlb_unregister_node(struct node *node) 397 { 398 struct hstate *h; 399 struct node_hstate *nhs = &node_hstates[node->dev.id]; 400 401 if (!nhs->hugepages_kobj) 402 return; /* no hstate attributes */ 403 404 for_each_hstate(h) { 405 int idx = hstate_index(h); 406 struct kobject *hstate_kobj = nhs->hstate_kobjs[idx]; 407 408 if (!hstate_kobj) 409 continue; 410 if (h->demote_order) 411 sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group); 412 sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group); 413 kobject_put(hstate_kobj); 414 nhs->hstate_kobjs[idx] = NULL; 415 } 416 417 kobject_put(nhs->hugepages_kobj); 418 nhs->hugepages_kobj = NULL; 419 } 420 421 422 /* 423 * Register hstate attributes for a single node device. 424 * No-op if attributes already registered. 425 */ 426 void hugetlb_register_node(struct node *node) 427 { 428 struct hstate *h; 429 struct node_hstate *nhs = &node_hstates[node->dev.id]; 430 int err; 431 432 if (!hugetlb_sysfs_initialized) 433 return; 434 435 if (nhs->hugepages_kobj) 436 return; /* already allocated */ 437 438 nhs->hugepages_kobj = kobject_create_and_add("hugepages", 439 &node->dev.kobj); 440 if (!nhs->hugepages_kobj) 441 return; 442 443 for_each_hstate(h) { 444 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj, 445 nhs->hstate_kobjs, 446 &per_node_hstate_attr_group); 447 if (err) { 448 pr_err("HugeTLB: Unable to add hstate %s for node %d\n", 449 h->name, node->dev.id); 450 hugetlb_unregister_node(node); 451 break; 452 } 453 } 454 } 455 456 /* 457 * hugetlb init time: register hstate attributes for all registered node 458 * devices of nodes that have memory. All on-line nodes should have 459 * registered their associated device by this time. 460 */ 461 static void __init hugetlb_register_all_nodes(void) 462 { 463 int nid; 464 465 for_each_online_node(nid) 466 hugetlb_register_node(node_devices[nid]); 467 } 468 #else /* !CONFIG_NUMA */ 469 470 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) 471 { 472 BUG(); 473 if (nidp) 474 *nidp = -1; 475 return NULL; 476 } 477 478 static void hugetlb_register_all_nodes(void) { } 479 480 #endif 481 482 void __init hugetlb_sysfs_init(void) 483 { 484 struct hstate *h; 485 int err; 486 487 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); 488 if (!hugepages_kobj) 489 return; 490 491 for_each_hstate(h) { 492 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, 493 hstate_kobjs, &hstate_attr_group); 494 if (err) 495 pr_err("HugeTLB: Unable to add hstate %s\n", h->name); 496 } 497 498 #ifdef CONFIG_NUMA 499 hugetlb_sysfs_initialized = true; 500 #endif 501 hugetlb_register_all_nodes(); 502 } 503