1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block rq-qos policy for assigning an I/O priority class to requests. 4 * 5 * Using an rq-qos policy for assigning I/O priority class has two advantages 6 * over using the ioprio_set() system call: 7 * 8 * - This policy is cgroup based so it has all the advantages of cgroups. 9 * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos 10 * controller affects page cache writeback I/O for filesystems that support 11 * assiociating a cgroup with writeback I/O. See also 12 * Documentation/admin-guide/cgroup-v2.rst. 13 */ 14 15 #include <linux/blk-mq.h> 16 #include <linux/blk_types.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include "blk-cgroup.h" 20 #include "blk-ioprio.h" 21 #include "blk-rq-qos.h" 22 23 /** 24 * enum prio_policy - I/O priority class policy. 25 * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. 26 * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT. 27 * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into 28 * IOPRIO_CLASS_BE. 29 * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. 30 * 31 * See also <linux/ioprio.h>. 32 */ 33 enum prio_policy { 34 POLICY_NO_CHANGE = 0, 35 POLICY_NONE_TO_RT = 1, 36 POLICY_RESTRICT_TO_BE = 2, 37 POLICY_ALL_TO_IDLE = 3, 38 }; 39 40 static const char *policy_name[] = { 41 [POLICY_NO_CHANGE] = "no-change", 42 [POLICY_NONE_TO_RT] = "none-to-rt", 43 [POLICY_RESTRICT_TO_BE] = "restrict-to-be", 44 [POLICY_ALL_TO_IDLE] = "idle", 45 }; 46 47 static struct blkcg_policy ioprio_policy; 48 49 /** 50 * struct ioprio_blkg - Per (cgroup, request queue) data. 51 * @pd: blkg_policy_data structure. 52 */ 53 struct ioprio_blkg { 54 struct blkg_policy_data pd; 55 }; 56 57 /** 58 * struct ioprio_blkcg - Per cgroup data. 59 * @cpd: blkcg_policy_data structure. 60 * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. 61 */ 62 struct ioprio_blkcg { 63 struct blkcg_policy_data cpd; 64 enum prio_policy prio_policy; 65 }; 66 67 static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd) 68 { 69 return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL; 70 } 71 72 static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) 73 { 74 return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), 75 struct ioprio_blkcg, cpd); 76 } 77 78 static struct ioprio_blkcg * 79 ioprio_blkcg_from_css(struct cgroup_subsys_state *css) 80 { 81 return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); 82 } 83 84 static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio) 85 { 86 struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy); 87 88 if (!pd) 89 return NULL; 90 91 return blkcg_to_ioprio_blkcg(pd->blkg->blkcg); 92 } 93 94 static int ioprio_show_prio_policy(struct seq_file *sf, void *v) 95 { 96 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); 97 98 seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); 99 return 0; 100 } 101 102 static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, 103 size_t nbytes, loff_t off) 104 { 105 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); 106 int ret; 107 108 if (off != 0) 109 return -EIO; 110 /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ 111 ret = sysfs_match_string(policy_name, buf); 112 if (ret < 0) 113 return ret; 114 blkcg->prio_policy = ret; 115 return nbytes; 116 } 117 118 static struct blkg_policy_data * 119 ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg) 120 { 121 struct ioprio_blkg *ioprio_blkg; 122 123 ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp); 124 if (!ioprio_blkg) 125 return NULL; 126 127 return &ioprio_blkg->pd; 128 } 129 130 static void ioprio_free_pd(struct blkg_policy_data *pd) 131 { 132 struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd); 133 134 kfree(ioprio_blkg); 135 } 136 137 static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) 138 { 139 struct ioprio_blkcg *blkcg; 140 141 blkcg = kzalloc(sizeof(*blkcg), gfp); 142 if (!blkcg) 143 return NULL; 144 blkcg->prio_policy = POLICY_NO_CHANGE; 145 return &blkcg->cpd; 146 } 147 148 static void ioprio_free_cpd(struct blkcg_policy_data *cpd) 149 { 150 struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); 151 152 kfree(blkcg); 153 } 154 155 #define IOPRIO_ATTRS \ 156 { \ 157 .name = "prio.class", \ 158 .seq_show = ioprio_show_prio_policy, \ 159 .write = ioprio_set_prio_policy, \ 160 }, \ 161 { } /* sentinel */ 162 163 /* cgroup v2 attributes */ 164 static struct cftype ioprio_files[] = { 165 IOPRIO_ATTRS 166 }; 167 168 /* cgroup v1 attributes */ 169 static struct cftype ioprio_legacy_files[] = { 170 IOPRIO_ATTRS 171 }; 172 173 static struct blkcg_policy ioprio_policy = { 174 .dfl_cftypes = ioprio_files, 175 .legacy_cftypes = ioprio_legacy_files, 176 177 .cpd_alloc_fn = ioprio_alloc_cpd, 178 .cpd_free_fn = ioprio_free_cpd, 179 180 .pd_alloc_fn = ioprio_alloc_pd, 181 .pd_free_fn = ioprio_free_pd, 182 }; 183 184 void blkcg_set_ioprio(struct bio *bio) 185 { 186 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio); 187 u16 prio; 188 189 if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE) 190 return; 191 192 /* 193 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers 194 * correspond to a lower priority. Hence, the max_t() below selects 195 * the lower priority of bi_ioprio and the cgroup I/O priority class. 196 * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O 197 * priority is assigned to the bio. 198 */ 199 prio = max_t(u16, bio->bi_ioprio, 200 IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); 201 if (prio > bio->bi_ioprio) 202 bio->bi_ioprio = prio; 203 } 204 205 void blk_ioprio_exit(struct gendisk *disk) 206 { 207 blkcg_deactivate_policy(disk->queue, &ioprio_policy); 208 } 209 210 int blk_ioprio_init(struct gendisk *disk) 211 { 212 return blkcg_activate_policy(disk->queue, &ioprio_policy); 213 } 214 215 static int __init ioprio_init(void) 216 { 217 return blkcg_policy_register(&ioprio_policy); 218 } 219 220 static void __exit ioprio_exit(void) 221 { 222 blkcg_policy_unregister(&ioprio_policy); 223 } 224 225 module_init(ioprio_init); 226 module_exit(ioprio_exit); 227