1 // SPDX-License-Identifier: LGPL-2.1 2 /* 3 * 4 * Copyright IBM Corporation, 2012 5 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> 6 * 7 * Cgroup v2 8 * Copyright (C) 2019 Red Hat, Inc. 9 * Author: Giuseppe Scrivano <gscrivan@redhat.com> 10 * 11 */ 12 13 #include <linux/cgroup.h> 14 #include <linux/page_counter.h> 15 #include <linux/slab.h> 16 #include <linux/hugetlb.h> 17 #include <linux/hugetlb_cgroup.h> 18 19 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) 20 #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) 21 #define MEMFILE_ATTR(val) ((val) & 0xffff) 22 23 /* Use t->m[0] to encode the offset */ 24 #define MEMFILE_OFFSET(t, m0) (((offsetof(t, m0) << 16) | sizeof_field(t, m0))) 25 #define MEMFILE_OFFSET0(val) (((val) >> 16) & 0xffff) 26 #define MEMFILE_FIELD_SIZE(val) ((val) & 0xffff) 27 28 #define DFL_TMPL_SIZE ARRAY_SIZE(hugetlb_dfl_tmpl) 29 #define LEGACY_TMPL_SIZE ARRAY_SIZE(hugetlb_legacy_tmpl) 30 31 static struct hugetlb_cgroup *root_h_cgroup __read_mostly; 32 static struct cftype *dfl_files; 33 static struct cftype *legacy_files; 34 35 static inline struct page_counter * 36 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx, 37 bool rsvd) 38 { 39 if (rsvd) 40 return &h_cg->rsvd_hugepage[idx]; 41 return &h_cg->hugepage[idx]; 42 } 43 44 static inline struct page_counter * 45 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx) 46 { 47 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false); 48 } 49 50 static inline struct page_counter * 51 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx) 52 { 53 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true); 54 } 55 56 static inline 57 struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s) 58 { 59 return s ? container_of(s, struct hugetlb_cgroup, css) : NULL; 60 } 61 62 static inline 63 struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task) 64 { 65 return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id)); 66 } 67 68 static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg) 69 { 70 return (h_cg == root_h_cgroup); 71 } 72 73 static inline struct hugetlb_cgroup * 74 parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg) 75 { 76 return hugetlb_cgroup_from_css(h_cg->css.parent); 77 } 78 79 static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) 80 { 81 struct hstate *h; 82 83 for_each_hstate(h) { 84 if (page_counter_read( 85 hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h)))) 86 return true; 87 } 88 return false; 89 } 90 91 static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, 92 struct hugetlb_cgroup *parent_h_cgroup) 93 { 94 int idx; 95 96 for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) { 97 struct page_counter *fault, *fault_parent = NULL; 98 struct page_counter *rsvd, *rsvd_parent = NULL; 99 unsigned long limit; 100 int ret; 101 102 if (parent_h_cgroup) { 103 fault_parent = hugetlb_cgroup_counter_from_cgroup( 104 parent_h_cgroup, idx); 105 rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd( 106 parent_h_cgroup, idx); 107 } 108 fault = hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx); 109 rsvd = hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx); 110 111 page_counter_init(fault, fault_parent, false); 112 page_counter_init(rsvd, rsvd_parent, false); 113 114 if (!cgroup_subsys_on_dfl(hugetlb_cgrp_subsys)) { 115 fault->track_failcnt = true; 116 rsvd->track_failcnt = true; 117 } 118 119 limit = round_down(PAGE_COUNTER_MAX, 120 pages_per_huge_page(&hstates[idx])); 121 122 ret = page_counter_set_max(fault, limit); 123 VM_WARN_ON_ONCE(ret); 124 ret = page_counter_set_max(rsvd, limit); 125 VM_WARN_ON_ONCE(ret); 126 } 127 } 128 129 static void hugetlb_cgroup_free(struct hugetlb_cgroup *h_cgroup) 130 { 131 int node; 132 133 for_each_node(node) 134 kfree(h_cgroup->nodeinfo[node]); 135 kfree(h_cgroup); 136 } 137 138 static struct cgroup_subsys_state * 139 hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 140 { 141 struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css); 142 struct hugetlb_cgroup *h_cgroup; 143 int node; 144 145 h_cgroup = kzalloc_flex(*h_cgroup, nodeinfo, nr_node_ids); 146 147 if (!h_cgroup) 148 return ERR_PTR(-ENOMEM); 149 150 if (!parent_h_cgroup) 151 root_h_cgroup = h_cgroup; 152 153 /* 154 * TODO: this routine can waste much memory for nodes which will 155 * never be onlined. It's better to use memory hotplug callback 156 * function. 157 */ 158 for_each_node(node) { 159 /* Set node_to_alloc to NUMA_NO_NODE for offline nodes. */ 160 int node_to_alloc = 161 node_state(node, N_NORMAL_MEMORY) ? node : NUMA_NO_NODE; 162 h_cgroup->nodeinfo[node] = 163 kzalloc_node(sizeof(struct hugetlb_cgroup_per_node), 164 GFP_KERNEL, node_to_alloc); 165 if (!h_cgroup->nodeinfo[node]) 166 goto fail_alloc_nodeinfo; 167 } 168 169 hugetlb_cgroup_init(h_cgroup, parent_h_cgroup); 170 return &h_cgroup->css; 171 172 fail_alloc_nodeinfo: 173 hugetlb_cgroup_free(h_cgroup); 174 return ERR_PTR(-ENOMEM); 175 } 176 177 static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) 178 { 179 hugetlb_cgroup_free(hugetlb_cgroup_from_css(css)); 180 } 181 182 /* 183 * Should be called with hugetlb_lock held. 184 * Since we are holding hugetlb_lock, pages cannot get moved from 185 * active list or uncharged from the cgroup, So no need to get 186 * page reference and test for page active here. This function 187 * cannot fail. 188 */ 189 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, 190 struct folio *folio) 191 { 192 unsigned int nr_pages; 193 struct page_counter *counter; 194 struct hugetlb_cgroup *hcg; 195 struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg); 196 197 hcg = hugetlb_cgroup_from_folio(folio); 198 /* 199 * We can have pages in active list without any cgroup 200 * ie, hugepage with less than 3 pages. We can safely 201 * ignore those pages. 202 */ 203 if (!hcg || hcg != h_cg) 204 goto out; 205 206 nr_pages = folio_nr_pages(folio); 207 if (!parent) { 208 parent = root_h_cgroup; 209 /* root has no limit */ 210 page_counter_charge(&parent->hugepage[idx], nr_pages); 211 } 212 counter = &h_cg->hugepage[idx]; 213 /* Take the pages off the local counter */ 214 page_counter_cancel(counter, nr_pages); 215 216 set_hugetlb_cgroup(folio, parent); 217 out: 218 return; 219 } 220 221 /* 222 * Force the hugetlb cgroup to empty the hugetlb resources by moving them to 223 * the parent cgroup. 224 */ 225 static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) 226 { 227 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); 228 struct hstate *h; 229 struct folio *folio; 230 231 do { 232 for_each_hstate(h) { 233 spin_lock_irq(&hugetlb_lock); 234 list_for_each_entry(folio, &h->hugepage_activelist, lru) 235 hugetlb_cgroup_move_parent(hstate_index(h), h_cg, folio); 236 237 spin_unlock_irq(&hugetlb_lock); 238 } 239 cond_resched(); 240 } while (hugetlb_cgroup_have_usage(h_cg)); 241 } 242 243 static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx, 244 enum hugetlb_memory_event event) 245 { 246 atomic_long_inc(&hugetlb->events_local[idx][event]); 247 cgroup_file_notify(&hugetlb->events_local_file[idx]); 248 249 do { 250 atomic_long_inc(&hugetlb->events[idx][event]); 251 cgroup_file_notify(&hugetlb->events_file[idx]); 252 } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) && 253 !hugetlb_cgroup_is_root(hugetlb)); 254 } 255 256 static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, 257 struct hugetlb_cgroup **ptr, 258 bool rsvd) 259 { 260 int ret = 0; 261 struct page_counter *counter; 262 struct hugetlb_cgroup *h_cg = NULL; 263 264 if (hugetlb_cgroup_disabled()) 265 goto done; 266 again: 267 rcu_read_lock(); 268 h_cg = hugetlb_cgroup_from_task(current); 269 if (!css_tryget(&h_cg->css)) { 270 rcu_read_unlock(); 271 goto again; 272 } 273 rcu_read_unlock(); 274 275 if (!page_counter_try_charge( 276 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), 277 nr_pages, &counter)) { 278 ret = -ENOMEM; 279 hugetlb_event(h_cg, idx, HUGETLB_MAX); 280 css_put(&h_cg->css); 281 goto done; 282 } 283 /* Reservations take a reference to the css because they do not get 284 * reparented. 285 */ 286 if (!rsvd) 287 css_put(&h_cg->css); 288 done: 289 *ptr = h_cg; 290 return ret; 291 } 292 293 int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, 294 struct hugetlb_cgroup **ptr) 295 { 296 return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false); 297 } 298 299 int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages, 300 struct hugetlb_cgroup **ptr) 301 { 302 return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true); 303 } 304 305 /* Should be called with hugetlb_lock held */ 306 static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, 307 struct hugetlb_cgroup *h_cg, 308 struct folio *folio, bool rsvd) 309 { 310 if (hugetlb_cgroup_disabled() || !h_cg) 311 return; 312 lockdep_assert_held(&hugetlb_lock); 313 __set_hugetlb_cgroup(folio, h_cg, rsvd); 314 if (!rsvd) { 315 unsigned long usage = 316 h_cg->nodeinfo[folio_nid(folio)]->usage[idx]; 317 /* 318 * This write is not atomic due to fetching usage and writing 319 * to it, but that's fine because we call this with 320 * hugetlb_lock held anyway. 321 */ 322 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx], 323 usage + nr_pages); 324 } 325 } 326 327 void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, 328 struct hugetlb_cgroup *h_cg, 329 struct folio *folio) 330 { 331 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, false); 332 } 333 334 void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages, 335 struct hugetlb_cgroup *h_cg, 336 struct folio *folio) 337 { 338 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, true); 339 } 340 341 /* 342 * Should be called with hugetlb_lock held 343 */ 344 static void __hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages, 345 struct folio *folio, bool rsvd) 346 { 347 struct hugetlb_cgroup *h_cg; 348 349 if (hugetlb_cgroup_disabled()) 350 return; 351 lockdep_assert_held(&hugetlb_lock); 352 h_cg = __hugetlb_cgroup_from_folio(folio, rsvd); 353 if (unlikely(!h_cg)) 354 return; 355 __set_hugetlb_cgroup(folio, NULL, rsvd); 356 357 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, 358 rsvd), 359 nr_pages); 360 361 if (rsvd) 362 css_put(&h_cg->css); 363 else { 364 unsigned long usage = 365 h_cg->nodeinfo[folio_nid(folio)]->usage[idx]; 366 /* 367 * This write is not atomic due to fetching usage and writing 368 * to it, but that's fine because we call this with 369 * hugetlb_lock held anyway. 370 */ 371 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx], 372 usage - nr_pages); 373 } 374 } 375 376 void hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages, 377 struct folio *folio) 378 { 379 __hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, false); 380 } 381 382 void hugetlb_cgroup_uncharge_folio_rsvd(int idx, unsigned long nr_pages, 383 struct folio *folio) 384 { 385 __hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, true); 386 } 387 388 static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, 389 struct hugetlb_cgroup *h_cg, 390 bool rsvd) 391 { 392 if (hugetlb_cgroup_disabled() || !h_cg) 393 return; 394 395 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx, 396 rsvd), 397 nr_pages); 398 399 if (rsvd) 400 css_put(&h_cg->css); 401 } 402 403 void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, 404 struct hugetlb_cgroup *h_cg) 405 { 406 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false); 407 } 408 409 void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages, 410 struct hugetlb_cgroup *h_cg) 411 { 412 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true); 413 } 414 415 void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start, 416 unsigned long end) 417 { 418 if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter || 419 !resv->css) 420 return; 421 422 page_counter_uncharge(resv->reservation_counter, 423 (end - start) * resv->pages_per_hpage); 424 css_put(resv->css); 425 } 426 427 void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, 428 struct file_region *rg, 429 unsigned long nr_pages, 430 bool region_del) 431 { 432 if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages) 433 return; 434 435 if (rg->reservation_counter && resv->pages_per_hpage && 436 !resv->reservation_counter) { 437 page_counter_uncharge(rg->reservation_counter, 438 nr_pages * resv->pages_per_hpage); 439 /* 440 * Only do css_put(rg->css) when we delete the entire region 441 * because one file_region must hold exactly one css reference. 442 */ 443 if (region_del) 444 css_put(rg->css); 445 } 446 } 447 448 enum { 449 RES_USAGE, 450 RES_RSVD_USAGE, 451 RES_LIMIT, 452 RES_RSVD_LIMIT, 453 RES_MAX_USAGE, 454 RES_RSVD_MAX_USAGE, 455 RES_FAILCNT, 456 RES_RSVD_FAILCNT, 457 }; 458 459 static int hugetlb_cgroup_read_numa_stat(struct seq_file *seq, void *dummy) 460 { 461 int nid; 462 struct cftype *cft = seq_cft(seq); 463 int idx = MEMFILE_IDX(cft->private); 464 bool legacy = !cgroup_subsys_on_dfl(hugetlb_cgrp_subsys); 465 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); 466 struct cgroup_subsys_state *css; 467 unsigned long usage; 468 469 if (legacy) { 470 /* Add up usage across all nodes for the non-hierarchical total. */ 471 usage = 0; 472 for_each_node_state(nid, N_MEMORY) 473 usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]); 474 seq_printf(seq, "total=%lu", usage * PAGE_SIZE); 475 476 /* Simply print the per-node usage for the non-hierarchical total. */ 477 for_each_node_state(nid, N_MEMORY) 478 seq_printf(seq, " N%d=%lu", nid, 479 READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) * 480 PAGE_SIZE); 481 seq_putc(seq, '\n'); 482 } 483 484 /* 485 * The hierarchical total is pretty much the value recorded by the 486 * counter, so use that. 487 */ 488 seq_printf(seq, "%stotal=%lu", legacy ? "hierarchical_" : "", 489 page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE); 490 491 /* 492 * For each node, transverse the css tree to obtain the hierarchical 493 * node usage. 494 */ 495 for_each_node_state(nid, N_MEMORY) { 496 usage = 0; 497 rcu_read_lock(); 498 css_for_each_descendant_pre(css, &h_cg->css) { 499 usage += READ_ONCE(hugetlb_cgroup_from_css(css) 500 ->nodeinfo[nid] 501 ->usage[idx]); 502 } 503 rcu_read_unlock(); 504 seq_printf(seq, " N%d=%lu", nid, usage * PAGE_SIZE); 505 } 506 507 seq_putc(seq, '\n'); 508 509 return 0; 510 } 511 512 static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, 513 struct cftype *cft) 514 { 515 struct page_counter *counter; 516 struct page_counter *rsvd_counter; 517 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); 518 519 counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)]; 520 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)]; 521 522 switch (MEMFILE_ATTR(cft->private)) { 523 case RES_USAGE: 524 return (u64)page_counter_read(counter) * PAGE_SIZE; 525 case RES_RSVD_USAGE: 526 return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE; 527 case RES_LIMIT: 528 return (u64)counter->max * PAGE_SIZE; 529 case RES_RSVD_LIMIT: 530 return (u64)rsvd_counter->max * PAGE_SIZE; 531 case RES_MAX_USAGE: 532 return (u64)counter->watermark * PAGE_SIZE; 533 case RES_RSVD_MAX_USAGE: 534 return (u64)rsvd_counter->watermark * PAGE_SIZE; 535 case RES_FAILCNT: 536 return counter->failcnt; 537 case RES_RSVD_FAILCNT: 538 return rsvd_counter->failcnt; 539 default: 540 BUG(); 541 } 542 } 543 544 static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) 545 { 546 int idx; 547 u64 val; 548 struct cftype *cft = seq_cft(seq); 549 unsigned long limit; 550 struct page_counter *counter; 551 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); 552 553 idx = MEMFILE_IDX(cft->private); 554 counter = &h_cg->hugepage[idx]; 555 556 limit = round_down(PAGE_COUNTER_MAX, 557 pages_per_huge_page(&hstates[idx])); 558 559 switch (MEMFILE_ATTR(cft->private)) { 560 case RES_RSVD_USAGE: 561 counter = &h_cg->rsvd_hugepage[idx]; 562 fallthrough; 563 case RES_USAGE: 564 val = (u64)page_counter_read(counter); 565 seq_printf(seq, "%llu\n", val * PAGE_SIZE); 566 break; 567 case RES_RSVD_LIMIT: 568 counter = &h_cg->rsvd_hugepage[idx]; 569 fallthrough; 570 case RES_LIMIT: 571 val = (u64)counter->max; 572 if (val == limit) 573 seq_puts(seq, "max\n"); 574 else 575 seq_printf(seq, "%llu\n", val * PAGE_SIZE); 576 break; 577 default: 578 BUG(); 579 } 580 581 return 0; 582 } 583 584 static DEFINE_MUTEX(hugetlb_limit_mutex); 585 586 static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, 587 char *buf, size_t nbytes, loff_t off, 588 const char *max) 589 { 590 int ret, idx; 591 unsigned long nr_pages; 592 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); 593 bool rsvd = false; 594 595 if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */ 596 return -EINVAL; 597 598 buf = strstrip(buf); 599 ret = page_counter_memparse(buf, max, &nr_pages); 600 if (ret) 601 return ret; 602 603 idx = MEMFILE_IDX(of_cft(of)->private); 604 nr_pages = round_down(nr_pages, pages_per_huge_page(&hstates[idx])); 605 606 switch (MEMFILE_ATTR(of_cft(of)->private)) { 607 case RES_RSVD_LIMIT: 608 rsvd = true; 609 fallthrough; 610 case RES_LIMIT: 611 mutex_lock(&hugetlb_limit_mutex); 612 ret = page_counter_set_max( 613 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd), 614 nr_pages); 615 mutex_unlock(&hugetlb_limit_mutex); 616 break; 617 default: 618 ret = -EINVAL; 619 break; 620 } 621 return ret ?: nbytes; 622 } 623 624 static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of, 625 char *buf, size_t nbytes, loff_t off) 626 { 627 return hugetlb_cgroup_write(of, buf, nbytes, off, "-1"); 628 } 629 630 static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of, 631 char *buf, size_t nbytes, loff_t off) 632 { 633 return hugetlb_cgroup_write(of, buf, nbytes, off, "max"); 634 } 635 636 static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, 637 char *buf, size_t nbytes, loff_t off) 638 { 639 int ret = 0; 640 struct page_counter *counter, *rsvd_counter; 641 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); 642 643 counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)]; 644 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)]; 645 646 switch (MEMFILE_ATTR(of_cft(of)->private)) { 647 case RES_MAX_USAGE: 648 page_counter_reset_watermark(counter); 649 break; 650 case RES_RSVD_MAX_USAGE: 651 page_counter_reset_watermark(rsvd_counter); 652 break; 653 case RES_FAILCNT: 654 counter->failcnt = 0; 655 break; 656 case RES_RSVD_FAILCNT: 657 rsvd_counter->failcnt = 0; 658 break; 659 default: 660 ret = -EINVAL; 661 break; 662 } 663 return ret ?: nbytes; 664 } 665 666 static char *mem_fmt(char *buf, int size, unsigned long hsize) 667 { 668 if (hsize >= SZ_1G) 669 snprintf(buf, size, "%luGB", hsize / SZ_1G); 670 else if (hsize >= SZ_1M) 671 snprintf(buf, size, "%luMB", hsize / SZ_1M); 672 else 673 snprintf(buf, size, "%luKB", hsize / SZ_1K); 674 return buf; 675 } 676 677 static int __hugetlb_events_show(struct seq_file *seq, bool local) 678 { 679 int idx; 680 long max; 681 struct cftype *cft = seq_cft(seq); 682 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); 683 684 idx = MEMFILE_IDX(cft->private); 685 686 if (local) 687 max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]); 688 else 689 max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]); 690 691 seq_printf(seq, "max %lu\n", max); 692 693 return 0; 694 } 695 696 static int hugetlb_events_show(struct seq_file *seq, void *v) 697 { 698 return __hugetlb_events_show(seq, false); 699 } 700 701 static int hugetlb_events_local_show(struct seq_file *seq, void *v) 702 { 703 return __hugetlb_events_show(seq, true); 704 } 705 706 static struct cftype hugetlb_dfl_tmpl[] = { 707 { 708 .name = "max", 709 .private = RES_LIMIT, 710 .seq_show = hugetlb_cgroup_read_u64_max, 711 .write = hugetlb_cgroup_write_dfl, 712 .flags = CFTYPE_NOT_ON_ROOT, 713 }, 714 { 715 .name = "rsvd.max", 716 .private = RES_RSVD_LIMIT, 717 .seq_show = hugetlb_cgroup_read_u64_max, 718 .write = hugetlb_cgroup_write_dfl, 719 .flags = CFTYPE_NOT_ON_ROOT, 720 }, 721 { 722 .name = "current", 723 .private = RES_USAGE, 724 .seq_show = hugetlb_cgroup_read_u64_max, 725 .flags = CFTYPE_NOT_ON_ROOT, 726 }, 727 { 728 .name = "rsvd.current", 729 .private = RES_RSVD_USAGE, 730 .seq_show = hugetlb_cgroup_read_u64_max, 731 .flags = CFTYPE_NOT_ON_ROOT, 732 }, 733 { 734 .name = "events", 735 .seq_show = hugetlb_events_show, 736 .file_offset = MEMFILE_OFFSET(struct hugetlb_cgroup, events_file[0]), 737 .flags = CFTYPE_NOT_ON_ROOT, 738 }, 739 { 740 .name = "events.local", 741 .seq_show = hugetlb_events_local_show, 742 .file_offset = MEMFILE_OFFSET(struct hugetlb_cgroup, events_local_file[0]), 743 .flags = CFTYPE_NOT_ON_ROOT, 744 }, 745 { 746 .name = "numa_stat", 747 .seq_show = hugetlb_cgroup_read_numa_stat, 748 .flags = CFTYPE_NOT_ON_ROOT, 749 }, 750 /* don't need terminator here */ 751 }; 752 753 static struct cftype hugetlb_legacy_tmpl[] = { 754 { 755 .name = "limit_in_bytes", 756 .private = RES_LIMIT, 757 .read_u64 = hugetlb_cgroup_read_u64, 758 .write = hugetlb_cgroup_write_legacy, 759 }, 760 { 761 .name = "rsvd.limit_in_bytes", 762 .private = RES_RSVD_LIMIT, 763 .read_u64 = hugetlb_cgroup_read_u64, 764 .write = hugetlb_cgroup_write_legacy, 765 }, 766 { 767 .name = "usage_in_bytes", 768 .private = RES_USAGE, 769 .read_u64 = hugetlb_cgroup_read_u64, 770 }, 771 { 772 .name = "rsvd.usage_in_bytes", 773 .private = RES_RSVD_USAGE, 774 .read_u64 = hugetlb_cgroup_read_u64, 775 }, 776 { 777 .name = "max_usage_in_bytes", 778 .private = RES_MAX_USAGE, 779 .write = hugetlb_cgroup_reset, 780 .read_u64 = hugetlb_cgroup_read_u64, 781 }, 782 { 783 .name = "rsvd.max_usage_in_bytes", 784 .private = RES_RSVD_MAX_USAGE, 785 .write = hugetlb_cgroup_reset, 786 .read_u64 = hugetlb_cgroup_read_u64, 787 }, 788 { 789 .name = "failcnt", 790 .private = RES_FAILCNT, 791 .write = hugetlb_cgroup_reset, 792 .read_u64 = hugetlb_cgroup_read_u64, 793 }, 794 { 795 .name = "rsvd.failcnt", 796 .private = RES_RSVD_FAILCNT, 797 .write = hugetlb_cgroup_reset, 798 .read_u64 = hugetlb_cgroup_read_u64, 799 }, 800 { 801 .name = "numa_stat", 802 .seq_show = hugetlb_cgroup_read_numa_stat, 803 }, 804 /* don't need terminator here */ 805 }; 806 807 static void __init 808 hugetlb_cgroup_cfttypes_init(struct hstate *h, struct cftype *cft, 809 struct cftype *tmpl, int tmpl_size) 810 { 811 char buf[32]; 812 int i, idx = hstate_index(h); 813 814 /* format the size */ 815 mem_fmt(buf, sizeof(buf), huge_page_size(h)); 816 817 for (i = 0; i < tmpl_size; cft++, tmpl++, i++) { 818 *cft = *tmpl; 819 /* rebuild the name */ 820 scnprintf(cft->name, MAX_CFTYPE_NAME, "%s.%s", buf, tmpl->name); 821 /* rebuild the private */ 822 cft->private = MEMFILE_PRIVATE(idx, tmpl->private); 823 /* rebuild the file_offset */ 824 if (tmpl->file_offset) { 825 unsigned int offset = tmpl->file_offset; 826 827 cft->file_offset = MEMFILE_OFFSET0(offset) + 828 MEMFILE_FIELD_SIZE(offset) * idx; 829 } 830 831 lockdep_register_key(&cft->lockdep_key); 832 } 833 } 834 835 static void __init __hugetlb_cgroup_file_dfl_init(struct hstate *h) 836 { 837 int idx = hstate_index(h); 838 839 hugetlb_cgroup_cfttypes_init(h, dfl_files + idx * DFL_TMPL_SIZE, 840 hugetlb_dfl_tmpl, DFL_TMPL_SIZE); 841 } 842 843 static void __init __hugetlb_cgroup_file_legacy_init(struct hstate *h) 844 { 845 int idx = hstate_index(h); 846 847 hugetlb_cgroup_cfttypes_init(h, legacy_files + idx * LEGACY_TMPL_SIZE, 848 hugetlb_legacy_tmpl, LEGACY_TMPL_SIZE); 849 } 850 851 static void __init __hugetlb_cgroup_file_init(struct hstate *h) 852 { 853 __hugetlb_cgroup_file_dfl_init(h); 854 __hugetlb_cgroup_file_legacy_init(h); 855 } 856 857 static void __init __hugetlb_cgroup_file_pre_init(void) 858 { 859 int cft_count; 860 861 cft_count = hugetlb_max_hstate * DFL_TMPL_SIZE + 1; /* add terminator */ 862 dfl_files = kzalloc_objs(struct cftype, cft_count); 863 BUG_ON(!dfl_files); 864 cft_count = hugetlb_max_hstate * LEGACY_TMPL_SIZE + 1; /* add terminator */ 865 legacy_files = kzalloc_objs(struct cftype, cft_count); 866 BUG_ON(!legacy_files); 867 } 868 869 static void __init __hugetlb_cgroup_file_post_init(void) 870 { 871 WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys, 872 dfl_files)); 873 WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, 874 legacy_files)); 875 } 876 877 void __init hugetlb_cgroup_file_init(void) 878 { 879 struct hstate *h; 880 881 __hugetlb_cgroup_file_pre_init(); 882 for_each_hstate(h) 883 __hugetlb_cgroup_file_init(h); 884 __hugetlb_cgroup_file_post_init(); 885 } 886 887 /* 888 * hugetlb_lock will make sure a parallel cgroup rmdir won't happen 889 * when we migrate hugepages 890 */ 891 void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio) 892 { 893 struct hugetlb_cgroup *h_cg; 894 struct hugetlb_cgroup *h_cg_rsvd; 895 struct hstate *h = folio_hstate(old_folio); 896 897 if (hugetlb_cgroup_disabled()) 898 return; 899 900 spin_lock_irq(&hugetlb_lock); 901 h_cg = hugetlb_cgroup_from_folio(old_folio); 902 h_cg_rsvd = hugetlb_cgroup_from_folio_rsvd(old_folio); 903 set_hugetlb_cgroup(old_folio, NULL); 904 set_hugetlb_cgroup_rsvd(old_folio, NULL); 905 906 /* move the h_cg details to new cgroup */ 907 set_hugetlb_cgroup(new_folio, h_cg); 908 set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd); 909 list_move(&new_folio->lru, &h->hugepage_activelist); 910 spin_unlock_irq(&hugetlb_lock); 911 } 912 913 static struct cftype hugetlb_files[] = { 914 {} /* terminate */ 915 }; 916 917 struct cgroup_subsys hugetlb_cgrp_subsys = { 918 .css_alloc = hugetlb_cgroup_css_alloc, 919 .css_offline = hugetlb_cgroup_css_offline, 920 .css_free = hugetlb_cgroup_css_free, 921 .dfl_cftypes = hugetlb_files, 922 .legacy_cftypes = hugetlb_files, 923 }; 924