blk-iocost.c (54c52e10dc9b939084a7e6e3d32ce8fd8dee7898) blk-iocost.c (cd006509b0a93cb7ee9d9fd50ae274098997a460)
1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * IO cost model based controller.
4 *
5 * Copyright (C) 2019 Tejun Heo <tj@kernel.org>
6 * Copyright (C) 2019 Andy Newell <newella@fb.com>
7 * Copyright (C) 2019 Facebook
8 *

--- 246 unchanged lines hidden (view full) ---

255 * convenient during operation.
256 *
257 * 1s worth of vtime is 2^37. This gives us both sub-nanosecond
258 * granularity and days of wrap-around time even at extreme vrates.
259 */
260 VTIME_PER_SEC_SHIFT = 37,
261 VTIME_PER_SEC = 1LLU << VTIME_PER_SEC_SHIFT,
262 VTIME_PER_USEC = VTIME_PER_SEC / USEC_PER_SEC,
1/* SPDX-License-Identifier: GPL-2.0
2 *
3 * IO cost model based controller.
4 *
5 * Copyright (C) 2019 Tejun Heo <tj@kernel.org>
6 * Copyright (C) 2019 Andy Newell <newella@fb.com>
7 * Copyright (C) 2019 Facebook
8 *

--- 246 unchanged lines hidden (view full) ---

255 * convenient during operation.
256 *
257 * 1s worth of vtime is 2^37. This gives us both sub-nanosecond
258 * granularity and days of wrap-around time even at extreme vrates.
259 */
260 VTIME_PER_SEC_SHIFT = 37,
261 VTIME_PER_SEC = 1LLU << VTIME_PER_SEC_SHIFT,
262 VTIME_PER_USEC = VTIME_PER_SEC / USEC_PER_SEC,
263 VTIME_PER_NSEC = VTIME_PER_SEC / NSEC_PER_SEC,
263
264 /* bound vrate adjustments within two orders of magnitude */
265 VRATE_MIN_PPM = 10000, /* 1% */
266 VRATE_MAX_PPM = 100000000, /* 10000% */
267
268 VRATE_MIN = VTIME_PER_USEC * VRATE_MIN_PPM / MILLION,
269 VRATE_CLAMP_ADJ_PCT = 4,
270

--- 1392 unchanged lines hidden (view full) ---

1663static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
1664{
1665 u64 cost;
1666
1667 calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
1668 return cost;
1669}
1670
264
265 /* bound vrate adjustments within two orders of magnitude */
266 VRATE_MIN_PPM = 10000, /* 1% */
267 VRATE_MAX_PPM = 100000000, /* 10000% */
268
269 VRATE_MIN = VTIME_PER_USEC * VRATE_MIN_PPM / MILLION,
270 VRATE_CLAMP_ADJ_PCT = 4,
271

--- 1392 unchanged lines hidden (view full) ---

1664static u64 calc_vtime_cost(struct bio *bio, struct ioc_gq *iocg, bool is_merge)
1665{
1666 u64 cost;
1667
1668 calc_vtime_cost_builtin(bio, iocg, is_merge, &cost);
1669 return cost;
1670}
1671
1672static void calc_size_vtime_cost_builtin(struct request *rq, struct ioc *ioc,
1673 u64 *costp)
1674{
1675 unsigned int pages = blk_rq_stats_sectors(rq) >> IOC_SECT_TO_PAGE_SHIFT;
1676
1677 switch (req_op(rq)) {
1678 case REQ_OP_READ:
1679 *costp = pages * ioc->params.lcoefs[LCOEF_RPAGE];
1680 break;
1681 case REQ_OP_WRITE:
1682 *costp = pages * ioc->params.lcoefs[LCOEF_WPAGE];
1683 break;
1684 default:
1685 *costp = 0;
1686 }
1687}
1688
1689static u64 calc_size_vtime_cost(struct request *rq, struct ioc *ioc)
1690{
1691 u64 cost;
1692
1693 calc_size_vtime_cost_builtin(rq, ioc, &cost);
1694 return cost;
1695}
1696
1671static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
1672{
1673 struct blkcg_gq *blkg = bio->bi_blkg;
1674 struct ioc *ioc = rqos_to_ioc(rqos);
1675 struct ioc_gq *iocg = blkg_to_iocg(blkg);
1676 struct ioc_now now;
1677 struct iocg_wait wait;
1678 u32 hw_active, hw_inuse;

--- 153 unchanged lines hidden (view full) ---

1832
1833 if (iocg && bio->bi_iocost_cost)
1834 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
1835}
1836
1837static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
1838{
1839 struct ioc *ioc = rqos_to_ioc(rqos);
1697static void ioc_rqos_throttle(struct rq_qos *rqos, struct bio *bio)
1698{
1699 struct blkcg_gq *blkg = bio->bi_blkg;
1700 struct ioc *ioc = rqos_to_ioc(rqos);
1701 struct ioc_gq *iocg = blkg_to_iocg(blkg);
1702 struct ioc_now now;
1703 struct iocg_wait wait;
1704 u32 hw_active, hw_inuse;

--- 153 unchanged lines hidden (view full) ---

1858
1859 if (iocg && bio->bi_iocost_cost)
1860 atomic64_add(bio->bi_iocost_cost, &iocg->done_vtime);
1861}
1862
1863static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
1864{
1865 struct ioc *ioc = rqos_to_ioc(rqos);
1840 u64 on_q_ns, rq_wait_ns;
1866 u64 on_q_ns, rq_wait_ns, size_nsec;
1841 int pidx, rw;
1842
1843 if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
1844 return;
1845
1846 switch (req_op(rq) & REQ_OP_MASK) {
1847 case REQ_OP_READ:
1848 pidx = QOS_RLAT;

--- 4 unchanged lines hidden (view full) ---

1853 rw = WRITE;
1854 break;
1855 default:
1856 return;
1857 }
1858
1859 on_q_ns = ktime_get_ns() - rq->alloc_time_ns;
1860 rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
1867 int pidx, rw;
1868
1869 if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
1870 return;
1871
1872 switch (req_op(rq) & REQ_OP_MASK) {
1873 case REQ_OP_READ:
1874 pidx = QOS_RLAT;

--- 4 unchanged lines hidden (view full) ---

1879 rw = WRITE;
1880 break;
1881 default:
1882 return;
1883 }
1884
1885 on_q_ns = ktime_get_ns() - rq->alloc_time_ns;
1886 rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;
1887 size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);
1861
1888
1862 if (on_q_ns <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1889 if (on_q_ns <= size_nsec ||
1890 on_q_ns - size_nsec <= ioc->params.qos[pidx] * NSEC_PER_USEC)
1863 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
1864 else
1865 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
1866
1867 this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
1868}
1869
1870static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)

--- 391 unchanged lines hidden (view full) ---

2262 }
2263
2264 if (qos[QOS_MIN] > qos[QOS_MAX])
2265 goto einval;
2266
2267 spin_lock_irq(&ioc->lock);
2268
2269 if (enable) {
1891 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_met);
1892 else
1893 this_cpu_inc(ioc->pcpu_stat->missed[rw].nr_missed);
1894
1895 this_cpu_add(ioc->pcpu_stat->rq_wait_ns, rq_wait_ns);
1896}
1897
1898static void ioc_rqos_queue_depth_changed(struct rq_qos *rqos)

--- 391 unchanged lines hidden (view full) ---

2290 }
2291
2292 if (qos[QOS_MIN] > qos[QOS_MAX])
2293 goto einval;
2294
2295 spin_lock_irq(&ioc->lock);
2296
2297 if (enable) {
2298 blk_stat_enable_accounting(ioc->rqos.q);
2270 blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2271 ioc->enabled = true;
2272 } else {
2273 blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2274 ioc->enabled = false;
2275 }
2276
2277 if (user) {

--- 187 unchanged lines hidden --- 		2299 blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2300 ioc->enabled = true;
2301 } else {
2302 blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q);
2303 ioc->enabled = false;
2304 }
2305
2306 if (user) {

--- 187 unchanged lines hidden ---