Lines Matching full:stock
150 * 1) CPU0: objcg == stock->cached_objcg in obj_cgroup_release()
154 * the stock if flushed, in obj_cgroup_release()
157 * 92 bytes are added to stock->nr_bytes in obj_cgroup_release()
158 * 6) CPU0: stock is flushed, in obj_cgroup_release()
1781 static void drain_obj_stock(struct obj_stock_pcp *stock);
1782 static bool obj_stock_flush_required(struct obj_stock_pcp *stock,
1798 struct memcg_stock_pcp *stock; in consume_stock() local
1807 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
1810 if (memcg != READ_ONCE(stock->cached[i])) in consume_stock()
1813 stock_pages = READ_ONCE(stock->nr_pages[i]); in consume_stock()
1815 WRITE_ONCE(stock->nr_pages[i], stock_pages - nr_pages); in consume_stock()
1836 static void drain_stock(struct memcg_stock_pcp *stock, int i) in drain_stock() argument
1838 struct mem_cgroup *old = READ_ONCE(stock->cached[i]); in drain_stock()
1844 stock_pages = READ_ONCE(stock->nr_pages[i]); in drain_stock()
1847 WRITE_ONCE(stock->nr_pages[i], 0); in drain_stock()
1851 WRITE_ONCE(stock->cached[i], NULL); in drain_stock()
1854 static void drain_stock_fully(struct memcg_stock_pcp *stock) in drain_stock_fully() argument
1859 drain_stock(stock, i); in drain_stock_fully()
1864 struct memcg_stock_pcp *stock; in drain_local_memcg_stock() local
1871 stock = this_cpu_ptr(&memcg_stock); in drain_local_memcg_stock()
1872 drain_stock_fully(stock); in drain_local_memcg_stock()
1873 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_memcg_stock()
1880 struct obj_stock_pcp *stock; in drain_local_obj_stock() local
1887 stock = this_cpu_ptr(&obj_stock); in drain_local_obj_stock()
1888 drain_obj_stock(stock); in drain_local_obj_stock()
1889 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_obj_stock()
1896 struct memcg_stock_pcp *stock; in refill_stock() local
1922 stock = this_cpu_ptr(&memcg_stock); in refill_stock()
1924 cached = READ_ONCE(stock->cached[i]); in refill_stock()
1927 if (memcg == READ_ONCE(stock->cached[i])) { in refill_stock()
1928 stock_pages = READ_ONCE(stock->nr_pages[i]) + nr_pages; in refill_stock()
1929 WRITE_ONCE(stock->nr_pages[i], stock_pages); in refill_stock()
1931 drain_stock(stock, i); in refill_stock()
1941 drain_stock(stock, i); in refill_stock()
1944 WRITE_ONCE(stock->cached[i], memcg); in refill_stock()
1945 WRITE_ONCE(stock->nr_pages[i], nr_pages); in refill_stock()
1951 static bool is_memcg_drain_needed(struct memcg_stock_pcp *stock, in is_memcg_drain_needed() argument
1960 memcg = READ_ONCE(stock->cached[i]); in is_memcg_drain_needed()
1964 if (READ_ONCE(stock->nr_pages[i]) && in is_memcg_drain_needed()
2317 /* Avoid the refill and flush of the older stock */ in try_charge_memcg()
2872 struct obj_stock_pcp *stock, int nr, in __account_obj_stock() argument
2878 * Save vmstat data in stock and skip vmstat array update unless in __account_obj_stock()
2881 if (stock->cached_pgdat != pgdat) { in __account_obj_stock()
2883 struct pglist_data *oldpg = stock->cached_pgdat; in __account_obj_stock()
2885 if (stock->nr_slab_reclaimable_b) { in __account_obj_stock()
2887 stock->nr_slab_reclaimable_b); in __account_obj_stock()
2888 stock->nr_slab_reclaimable_b = 0; in __account_obj_stock()
2890 if (stock->nr_slab_unreclaimable_b) { in __account_obj_stock()
2892 stock->nr_slab_unreclaimable_b); in __account_obj_stock()
2893 stock->nr_slab_unreclaimable_b = 0; in __account_obj_stock()
2895 stock->cached_pgdat = pgdat; in __account_obj_stock()
2898 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in __account_obj_stock()
2899 : &stock->nr_slab_unreclaimable_b; in __account_obj_stock()
2923 struct obj_stock_pcp *stock; in consume_obj_stock() local
2929 stock = this_cpu_ptr(&obj_stock); in consume_obj_stock()
2930 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
2931 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
2935 __account_obj_stock(objcg, stock, nr_bytes, pgdat, idx); in consume_obj_stock()
2943 static void drain_obj_stock(struct obj_stock_pcp *stock) in drain_obj_stock() argument
2945 struct obj_cgroup *old = READ_ONCE(stock->cached_objcg); in drain_obj_stock()
2950 if (stock->nr_bytes) { in drain_obj_stock()
2951 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
2952 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
2969 * On the next attempt to refill obj stock it will be moved in drain_obj_stock()
2970 * to a per-cpu stock (probably, on an other CPU), see in drain_obj_stock()
2978 stock->nr_bytes = 0; in drain_obj_stock()
2982 * Flush the vmstat data in current stock in drain_obj_stock()
2984 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
2985 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
2986 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
2988 stock->nr_slab_reclaimable_b); in drain_obj_stock()
2989 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
2991 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
2992 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
2994 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
2995 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
2997 stock->cached_pgdat = NULL; in drain_obj_stock()
3000 WRITE_ONCE(stock->cached_objcg, NULL); in drain_obj_stock()
3004 static bool obj_stock_flush_required(struct obj_stock_pcp *stock, in obj_stock_flush_required() argument
3007 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg); in obj_stock_flush_required()
3026 struct obj_stock_pcp *stock; in refill_obj_stock() local
3038 stock = this_cpu_ptr(&obj_stock); in refill_obj_stock()
3039 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */ in refill_obj_stock()
3040 drain_obj_stock(stock); in refill_obj_stock()
3042 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3044 WRITE_ONCE(stock->cached_objcg, objcg); in refill_obj_stock()
3048 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3051 __account_obj_stock(objcg, stock, nr_acct, pgdat, idx); in refill_obj_stock()
3053 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3054 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3055 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()
3081 * grab some new pages. The stock's nr_bytes will be flushed to in obj_cgroup_charge_account()
3084 * The stock's nr_bytes may contain enough pre-charged bytes in obj_cgroup_charge_account()