144 static bool mdtc_valid(struct dirty_throttle_control *dtc)  in mdtc_valid()  argument
146 	return dtc->dom;  in mdtc_valid()
149 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)  in dtc_dom()  argument
151 	return dtc->dom;  in dtc_dom()
198 static bool mdtc_valid(struct dirty_throttle_control *dtc)  in mdtc_valid()  argument
203 static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)  in dtc_dom()  argument
348  * @dtc: dirty_throttle_control of interest
350  * Calculate @dtc->thresh and ->bg_thresh considering
352  * must ensure that @dtc->avail is set before calling this function.  The
355 static void domain_dirty_limits(struct dirty_throttle_control *dtc)  in domain_dirty_limits()  argument
357 	const unsigned long available_memory = dtc->avail;  in domain_dirty_limits()
358 	struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);  in domain_dirty_limits()
368 	/* gdtc is !NULL iff @dtc is for memcg domain */  in domain_dirty_limits()
412 	dtc->thresh = thresh;  in domain_dirty_limits()
413 	dtc->bg_thresh = bg_thresh;  in domain_dirty_limits()
848 static inline bool dtc_is_global(struct dirty_throttle_control *dtc)  in dtc_is_global()  argument
850 	return mdtc_gdtc(dtc) == NULL;  in dtc_is_global()
857 static void domain_dirty_avail(struct dirty_throttle_control *dtc,  in domain_dirty_avail()  argument
860 	if (dtc_is_global(dtc)) {  in domain_dirty_avail()
861 		dtc->avail = global_dirtyable_memory();  in domain_dirty_avail()
862 		dtc->dirty = global_node_page_state(NR_FILE_DIRTY);  in domain_dirty_avail()
864 			dtc->dirty += global_node_page_state(NR_WRITEBACK);  in domain_dirty_avail()
868 		mem_cgroup_wb_stats(dtc->wb, &filepages, &headroom, &dtc->dirty,  in domain_dirty_avail()
871 			dtc->dirty += writeback;  in domain_dirty_avail()
872 		mdtc_calc_avail(dtc, filepages, headroom);  in domain_dirty_avail()
878  * @dtc: dirty_throttle_context of interest
879  * @thresh: dirty throttling or dirty background threshold of wb_domain in @dtc
900 static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc,  in __wb_calc_thresh()  argument
903 	struct wb_domain *dom = dtc_dom(dtc);  in __wb_calc_thresh()
904 	struct bdi_writeback *wb = dtc->wb;  in __wb_calc_thresh()
913 	fprop_fraction_percpu(&dom->completions, dtc->wb_completions,  in __wb_calc_thresh()
931 	if (thresh > dtc->dirty) {  in __wb_calc_thresh()
933 			wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 100);  in __wb_calc_thresh()
935 			wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 8);  in __wb_calc_thresh()
1071 static void wb_position_ratio(struct dirty_throttle_control *dtc)  in wb_position_ratio()  argument
1073 	struct bdi_writeback *wb = dtc->wb;  in wb_position_ratio()
1075 	unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);  in wb_position_ratio()
1076 	unsigned long limit = dtc->limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);  in wb_position_ratio()
1077 	unsigned long wb_thresh = dtc->wb_thresh;  in wb_position_ratio()
1085 	dtc->pos_ratio = 0;  in wb_position_ratio()
1087 	if (unlikely(dtc->dirty >= limit))  in wb_position_ratio()
1096 	pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);  in wb_position_ratio()
1126 		if (dtc->wb_dirty >= wb_thresh)  in wb_position_ratio()
1130 						    dtc->wb_bg_thresh);  in wb_position_ratio()
1135 		wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,  in wb_position_ratio()
1159 		dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);  in wb_position_ratio()
1194 	if (unlikely(wb_thresh > dtc->thresh))  in wb_position_ratio()
1195 		wb_thresh = dtc->thresh;  in wb_position_ratio()
1200 	x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);  in wb_position_ratio()
1210 	span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;  in wb_position_ratio()
1213 	if (dtc->wb_dirty < x_intercept - span / 4) {  in wb_position_ratio()
1214 		pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),  in wb_position_ratio()
1225 	if (dtc->wb_dirty < x_intercept) {  in wb_position_ratio()
1226 		if (dtc->wb_dirty > x_intercept / 8)  in wb_position_ratio()
1228 					    dtc->wb_dirty);  in wb_position_ratio()
1233 	dtc->pos_ratio = pos_ratio;  in wb_position_ratio()
1286 static void update_dirty_limit(struct dirty_throttle_control *dtc)  in update_dirty_limit()  argument
1288 	struct wb_domain *dom = dtc_dom(dtc);  in update_dirty_limit()
1289 	unsigned long thresh = dtc->thresh;  in update_dirty_limit()
1305 	thresh = max(thresh, dtc->dirty);  in update_dirty_limit()
1315 static void domain_update_dirty_limit(struct dirty_throttle_control *dtc,  in domain_update_dirty_limit()  argument
1318 	struct wb_domain *dom = dtc_dom(dtc);  in domain_update_dirty_limit()
1328 		update_dirty_limit(dtc);  in domain_update_dirty_limit()
1340 static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,  in wb_update_dirty_ratelimit()  argument
1344 	struct bdi_writeback *wb = dtc->wb;  in wb_update_dirty_ratelimit()
1345 	unsigned long dirty = dtc->dirty;  in wb_update_dirty_ratelimit()
1346 	unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);  in wb_update_dirty_ratelimit()
1347 	unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);  in wb_update_dirty_ratelimit()
1368 					dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;  in wb_update_dirty_ratelimit()
1453 		dirty = dtc->wb_dirty;  in wb_update_dirty_ratelimit()
1454 		setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;  in wb_update_dirty_ratelimit()
1670 static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)  in wb_dirty_limits()  argument
1672 	struct bdi_writeback *wb = dtc->wb;  in wb_dirty_limits()
1688 	dtc->wb_thresh = __wb_calc_thresh(dtc, dtc->thresh);  in wb_dirty_limits()
1689 	dtc->wb_bg_thresh = dtc->thresh ?  in wb_dirty_limits()
1690 		div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0;  in wb_dirty_limits()
1702 	if (dtc->wb_thresh < 2 * wb_stat_error()) {  in wb_dirty_limits()
1704 		dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);  in wb_dirty_limits()
1707 		dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);  in wb_dirty_limits()
1711 static unsigned long domain_poll_intv(struct dirty_throttle_control *dtc,  in domain_poll_intv()  argument
1717 		dirty = dtc->wb_dirty;  in domain_poll_intv()
1718 		thresh = dtc->wb_thresh;  in domain_poll_intv()
1720 		dirty = dtc->dirty;  in domain_poll_intv()
1721 		thresh = dtc->thresh;  in domain_poll_intv()
1736 static void domain_dirty_freerun(struct dirty_throttle_control *dtc,  in domain_dirty_freerun()  argument
1742 		wb_dirty_limits(dtc);  in domain_dirty_freerun()
1743 		dirty = dtc->wb_dirty;  in domain_dirty_freerun()
1744 		thresh = dtc->wb_thresh;  in domain_dirty_freerun()
1745 		bg_thresh = dtc->wb_bg_thresh;  in domain_dirty_freerun()
1747 		dirty = dtc->dirty;  in domain_dirty_freerun()
1748 		thresh = dtc->thresh;  in domain_dirty_freerun()
1749 		bg_thresh = dtc->bg_thresh;  in domain_dirty_freerun()
1751 	dtc->freerun = dirty <= dirty_freerun_ceiling(thresh, bg_thresh);  in domain_dirty_freerun()
1754 static void balance_domain_limits(struct dirty_throttle_control *dtc,  in balance_domain_limits()  argument
1757 	domain_dirty_avail(dtc, true);  in balance_domain_limits()
1758 	domain_dirty_limits(dtc);  in balance_domain_limits()
1759 	domain_dirty_freerun(dtc, strictlimit);  in balance_domain_limits()
1762 static void wb_dirty_freerun(struct dirty_throttle_control *dtc,  in wb_dirty_freerun()  argument
1765 	dtc->freerun = false;  in wb_dirty_freerun()
1771 	wb_dirty_limits(dtc);  in wb_dirty_freerun()
1779 	dtc->freerun = dtc->wb_dirty <  in wb_dirty_freerun()
1780 		       dirty_freerun_ceiling(dtc->wb_thresh, dtc->wb_bg_thresh);  in wb_dirty_freerun()
1783 static inline void wb_dirty_exceeded(struct dirty_throttle_control *dtc,  in wb_dirty_exceeded()  argument
1786 	dtc->dirty_exceeded = (dtc->wb_dirty > dtc->wb_thresh) &&  in wb_dirty_exceeded()
1787 		((dtc->dirty > dtc->thresh) || strictlimit);  in wb_dirty_exceeded()
1794 static void balance_wb_limits(struct dirty_throttle_control *dtc,  in balance_wb_limits()  argument
1797 	wb_dirty_freerun(dtc, strictlimit);  in balance_wb_limits()
1798 	if (dtc->freerun)  in balance_wb_limits()
1801 	wb_dirty_exceeded(dtc, strictlimit);  in balance_wb_limits()
1802 	wb_position_ratio(dtc);  in balance_wb_limits()
1889 		 * global dtc by default.  in balance_dirty_pages()
1916 		/* throttle according to the chosen dtc */  in balance_dirty_pages()
2128 static void wb_bg_dirty_limits(struct dirty_throttle_control *dtc)  in wb_bg_dirty_limits()  argument
2130 	struct bdi_writeback *wb = dtc->wb;  in wb_bg_dirty_limits()
2132 	dtc->wb_bg_thresh = __wb_calc_thresh(dtc, dtc->bg_thresh);  in wb_bg_dirty_limits()
2133 	if (dtc->wb_bg_thresh < 2 * wb_stat_error())  in wb_bg_dirty_limits()
2134 		dtc->wb_dirty = wb_stat_sum(wb, WB_RECLAIMABLE);  in wb_bg_dirty_limits()
2136 		dtc->wb_dirty = wb_stat(wb, WB_RECLAIMABLE);  in wb_bg_dirty_limits()
2139 static bool domain_over_bg_thresh(struct dirty_throttle_control *dtc)  in domain_over_bg_thresh()  argument
2141 	domain_dirty_avail(dtc, false);  in domain_over_bg_thresh()
2142 	domain_dirty_limits(dtc);  in domain_over_bg_thresh()
2143 	if (dtc->dirty > dtc->bg_thresh)  in domain_over_bg_thresh()
2146 	wb_bg_dirty_limits(dtc);  in domain_over_bg_thresh()
2147 	if (dtc->wb_dirty > dtc->wb_bg_thresh)  in domain_over_bg_thresh()