1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Guest ITS library, generously donated by drivers/irqchip/irq-gic-v3-its.c 4 * over in the kernel tree. 5 */ 6 7 #include <linux/kvm.h> 8 #include <linux/sizes.h> 9 #include <asm/kvm_para.h> 10 #include <asm/kvm.h> 11 12 #include "kvm_util.h" 13 #include "vgic.h" 14 #include "gic.h" 15 #include "gic_v3.h" 16 #include "processor.h" 17 18 #define GITS_COLLECTION_TARGET_SHIFT 16 19 20 static u64 its_read_u64(unsigned long offset) 21 { 22 return readq_relaxed(GITS_BASE_GVA + offset); 23 } 24 25 static void its_write_u64(unsigned long offset, u64 val) 26 { 27 writeq_relaxed(val, GITS_BASE_GVA + offset); 28 } 29 30 static u32 its_read_u32(unsigned long offset) 31 { 32 return readl_relaxed(GITS_BASE_GVA + offset); 33 } 34 35 static void its_write_u32(unsigned long offset, u32 val) 36 { 37 writel_relaxed(val, GITS_BASE_GVA + offset); 38 } 39 40 static unsigned long its_find_baser(unsigned int type) 41 { 42 int i; 43 44 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 45 u64 baser; 46 unsigned long offset = GITS_BASER + (i * sizeof(baser)); 47 48 baser = its_read_u64(offset); 49 if (GITS_BASER_TYPE(baser) == type) 50 return offset; 51 } 52 53 GUEST_FAIL("Couldn't find an ITS BASER of type %u", type); 54 return -1; 55 } 56 57 static void its_install_table(unsigned int type, gpa_t base, size_t size) 58 { 59 unsigned long offset = its_find_baser(type); 60 u64 baser; 61 62 baser = ((size / SZ_64K) - 1) | 63 GITS_BASER_PAGE_SIZE_64K | 64 GITS_BASER_InnerShareable | 65 base | 66 GITS_BASER_RaWaWb | 67 GITS_BASER_VALID; 68 69 its_write_u64(offset, baser); 70 } 71 72 static void its_install_cmdq(gpa_t base, size_t size) 73 { 74 u64 cbaser; 75 76 cbaser = ((size / SZ_4K) - 1) | 77 GITS_CBASER_InnerShareable | 78 base | 79 GITS_CBASER_RaWaWb | 80 GITS_CBASER_VALID; 81 82 its_write_u64(GITS_CBASER, cbaser); 83 } 84 85 void its_init(gpa_t coll_tbl, size_t coll_tbl_sz, gpa_t device_tbl, 86 size_t device_tbl_sz, gpa_t cmdq, size_t cmdq_size) 87 { 88 u32 ctlr; 89 90 its_install_table(GITS_BASER_TYPE_COLLECTION, coll_tbl, coll_tbl_sz); 91 its_install_table(GITS_BASER_TYPE_DEVICE, device_tbl, device_tbl_sz); 92 its_install_cmdq(cmdq, cmdq_size); 93 94 ctlr = its_read_u32(GITS_CTLR); 95 ctlr |= GITS_CTLR_ENABLE; 96 its_write_u32(GITS_CTLR, ctlr); 97 } 98 99 struct its_cmd_block { 100 union { 101 u64 raw_cmd[4]; 102 __le64 raw_cmd_le[4]; 103 }; 104 }; 105 106 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 107 { 108 /* Let's fixup BE commands */ 109 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); 110 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); 111 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); 112 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); 113 } 114 115 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 116 { 117 u64 mask = GENMASK_ULL(h, l); 118 *raw_cmd &= ~mask; 119 *raw_cmd |= (val << l) & mask; 120 } 121 122 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 123 { 124 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 125 } 126 127 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 128 { 129 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 130 } 131 132 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 133 { 134 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 135 } 136 137 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 138 { 139 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 140 } 141 142 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 143 { 144 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 145 } 146 147 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 148 { 149 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 150 } 151 152 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 153 { 154 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 155 } 156 157 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 158 { 159 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 160 } 161 162 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 163 { 164 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 165 } 166 167 static u64 procnum_to_rdbase(u32 vcpu_id) 168 { 169 return vcpu_id << GITS_COLLECTION_TARGET_SHIFT; 170 } 171 172 #define GITS_CMDQ_POLL_ITERATIONS 0 173 174 static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd) 175 { 176 u64 cwriter = its_read_u64(GITS_CWRITER); 177 struct its_cmd_block *dst = cmdq_base + cwriter; 178 u64 cbaser = its_read_u64(GITS_CBASER); 179 size_t cmdq_size; 180 u64 next; 181 int i; 182 183 cmdq_size = ((cbaser & 0xFF) + 1) * SZ_4K; 184 185 its_fixup_cmd(cmd); 186 187 WRITE_ONCE(*dst, *cmd); 188 dsb(ishst); 189 next = (cwriter + sizeof(*cmd)) % cmdq_size; 190 its_write_u64(GITS_CWRITER, next); 191 192 /* 193 * Polling isn't necessary considering KVM's ITS emulation at the time 194 * of writing this, as the CMDQ is processed synchronously after a write 195 * to CWRITER. 196 */ 197 for (i = 0; its_read_u64(GITS_CREADR) != next; i++) { 198 __GUEST_ASSERT(i < GITS_CMDQ_POLL_ITERATIONS, 199 "ITS didn't process command at offset %lu after %d iterations\n", 200 cwriter, i); 201 202 cpu_relax(); 203 } 204 } 205 206 void its_send_mapd_cmd(void *cmdq_base, u32 device_id, gpa_t itt_base, 207 size_t itt_size, bool valid) 208 { 209 struct its_cmd_block cmd = {}; 210 211 its_encode_cmd(&cmd, GITS_CMD_MAPD); 212 its_encode_devid(&cmd, device_id); 213 its_encode_size(&cmd, ilog2(itt_size) - 1); 214 its_encode_itt(&cmd, itt_base); 215 its_encode_valid(&cmd, valid); 216 217 its_send_cmd(cmdq_base, &cmd); 218 } 219 220 void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid) 221 { 222 struct its_cmd_block cmd = {}; 223 224 its_encode_cmd(&cmd, GITS_CMD_MAPC); 225 its_encode_collection(&cmd, collection_id); 226 its_encode_target(&cmd, procnum_to_rdbase(vcpu_id)); 227 its_encode_valid(&cmd, valid); 228 229 its_send_cmd(cmdq_base, &cmd); 230 } 231 232 void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id, 233 u32 collection_id, u32 intid) 234 { 235 struct its_cmd_block cmd = {}; 236 237 its_encode_cmd(&cmd, GITS_CMD_MAPTI); 238 its_encode_devid(&cmd, device_id); 239 its_encode_event_id(&cmd, event_id); 240 its_encode_phys_id(&cmd, intid); 241 its_encode_collection(&cmd, collection_id); 242 243 its_send_cmd(cmdq_base, &cmd); 244 } 245 246 void its_send_invall_cmd(void *cmdq_base, u32 collection_id) 247 { 248 struct its_cmd_block cmd = {}; 249 250 its_encode_cmd(&cmd, GITS_CMD_INVALL); 251 its_encode_collection(&cmd, collection_id); 252 253 its_send_cmd(cmdq_base, &cmd); 254 } 255 256 void its_send_sync_cmd(void *cmdq_base, u32 vcpu_id) 257 { 258 struct its_cmd_block cmd = {}; 259 260 its_encode_cmd(&cmd, GITS_CMD_SYNC); 261 its_encode_target(&cmd, procnum_to_rdbase(vcpu_id)); 262 263 its_send_cmd(cmdq_base, &cmd); 264 } 265