1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block rq-qos policy for assigning an I/O priority class to requests. 4 * 5 * Using an rq-qos policy for assigning I/O priority class has two advantages 6 * over using the ioprio_set() system call: 7 * 8 * - This policy is cgroup based so it has all the advantages of cgroups. 9 * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos 10 * controller affects page cache writeback I/O for filesystems that support 11 * assiociating a cgroup with writeback I/O. See also 12 * Documentation/admin-guide/cgroup-v2.rst. 13 */ 14 15 #include <linux/blk-mq.h> 16 #include <linux/blk_types.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include "blk-cgroup.h" 20 #include "blk-ioprio.h" 21 #include "blk-rq-qos.h" 22 23 /** 24 * enum prio_policy - I/O priority class policy. 25 * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class. 26 * @POLICY_PROMOTE_TO_RT: modify no-IOPRIO_CLASS_RT to IOPRIO_CLASS_RT. 27 * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into 28 * IOPRIO_CLASS_BE. 29 * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE. 30 * @POLICY_NONE_TO_RT: an alias for POLICY_PROMOTE_TO_RT. 31 * 32 * See also <linux/ioprio.h>. 33 */ 34 enum prio_policy { 35 POLICY_NO_CHANGE = 0, 36 POLICY_PROMOTE_TO_RT = 1, 37 POLICY_RESTRICT_TO_BE = 2, 38 POLICY_ALL_TO_IDLE = 3, 39 POLICY_NONE_TO_RT = 4, 40 }; 41 42 static const char *policy_name[] = { 43 [POLICY_NO_CHANGE] = "no-change", 44 [POLICY_PROMOTE_TO_RT] = "promote-to-rt", 45 [POLICY_RESTRICT_TO_BE] = "restrict-to-be", 46 [POLICY_ALL_TO_IDLE] = "idle", 47 [POLICY_NONE_TO_RT] = "none-to-rt", 48 }; 49 50 static struct blkcg_policy ioprio_policy; 51 52 /** 53 * struct ioprio_blkcg - Per cgroup data. 54 * @cpd: blkcg_policy_data structure. 55 * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>. 56 */ 57 struct ioprio_blkcg { 58 struct blkcg_policy_data cpd; 59 enum prio_policy prio_policy; 60 }; 61 62 static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg) 63 { 64 return container_of(blkcg_to_cpd(blkcg, &ioprio_policy), 65 struct ioprio_blkcg, cpd); 66 } 67 68 static struct ioprio_blkcg * 69 ioprio_blkcg_from_css(struct cgroup_subsys_state *css) 70 { 71 return blkcg_to_ioprio_blkcg(css_to_blkcg(css)); 72 } 73 74 static int ioprio_show_prio_policy(struct seq_file *sf, void *v) 75 { 76 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf)); 77 78 seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]); 79 return 0; 80 } 81 82 static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf, 83 size_t nbytes, loff_t off) 84 { 85 struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of)); 86 int ret; 87 88 if (off != 0) 89 return -EIO; 90 /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */ 91 ret = sysfs_match_string(policy_name, buf); 92 if (ret < 0) 93 return ret; 94 blkcg->prio_policy = ret; 95 return nbytes; 96 } 97 98 static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp) 99 { 100 struct ioprio_blkcg *blkcg; 101 102 blkcg = kzalloc(sizeof(*blkcg), gfp); 103 if (!blkcg) 104 return NULL; 105 blkcg->prio_policy = POLICY_NO_CHANGE; 106 return &blkcg->cpd; 107 } 108 109 static void ioprio_free_cpd(struct blkcg_policy_data *cpd) 110 { 111 struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd); 112 113 kfree(blkcg); 114 } 115 116 #define IOPRIO_ATTRS \ 117 { \ 118 .name = "prio.class", \ 119 .seq_show = ioprio_show_prio_policy, \ 120 .write = ioprio_set_prio_policy, \ 121 }, \ 122 { } /* sentinel */ 123 124 /* cgroup v2 attributes */ 125 static struct cftype ioprio_files[] = { 126 IOPRIO_ATTRS 127 }; 128 129 /* cgroup v1 attributes */ 130 static struct cftype ioprio_legacy_files[] = { 131 IOPRIO_ATTRS 132 }; 133 134 static struct blkcg_policy ioprio_policy = { 135 .dfl_cftypes = ioprio_files, 136 .legacy_cftypes = ioprio_legacy_files, 137 138 .cpd_alloc_fn = ioprio_alloc_cpd, 139 .cpd_free_fn = ioprio_free_cpd, 140 }; 141 142 void blkcg_set_ioprio(struct bio *bio) 143 { 144 struct ioprio_blkcg *blkcg = blkcg_to_ioprio_blkcg(bio->bi_blkg->blkcg); 145 u16 prio; 146 147 if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE) 148 return; 149 150 if (blkcg->prio_policy == POLICY_PROMOTE_TO_RT || 151 blkcg->prio_policy == POLICY_NONE_TO_RT) { 152 /* 153 * For RT threads, the default priority level is 4 because 154 * task_nice is 0. By promoting non-RT io-priority to RT-class 155 * and default level 4, those requests that are already 156 * RT-class but need a higher io-priority can use ioprio_set() 157 * to achieve this. 158 */ 159 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) != IOPRIO_CLASS_RT) 160 bio->bi_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 4); 161 return; 162 } 163 164 /* 165 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers 166 * correspond to a lower priority. Hence, the max_t() below selects 167 * the lower priority of bi_ioprio and the cgroup I/O priority class. 168 * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O 169 * priority is assigned to the bio. 170 */ 171 prio = max_t(u16, bio->bi_ioprio, 172 IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0)); 173 if (prio > bio->bi_ioprio) 174 bio->bi_ioprio = prio; 175 } 176 177 static int __init ioprio_init(void) 178 { 179 return blkcg_policy_register(&ioprio_policy); 180 } 181 182 static void __exit ioprio_exit(void) 183 { 184 blkcg_policy_unregister(&ioprio_policy); 185 } 186 187 module_init(ioprio_init); 188 module_exit(ioprio_exit); 189