1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Guest ITS library, generously donated by drivers/irqchip/irq-gic-v3-its.c 4 * over in the kernel tree. 5 */ 6 7 #include <linux/kvm.h> 8 #include <linux/sizes.h> 9 #include <asm/kvm_para.h> 10 #include <asm/kvm.h> 11 12 #include "kvm_util.h" 13 #include "vgic.h" 14 #include "gic.h" 15 #include "gic_v3.h" 16 #include "processor.h" 17 18 static u64 its_read_u64(unsigned long offset) 19 { 20 return readq_relaxed(GITS_BASE_GVA + offset); 21 } 22 23 static void its_write_u64(unsigned long offset, u64 val) 24 { 25 writeq_relaxed(val, GITS_BASE_GVA + offset); 26 } 27 28 static u32 its_read_u32(unsigned long offset) 29 { 30 return readl_relaxed(GITS_BASE_GVA + offset); 31 } 32 33 static void its_write_u32(unsigned long offset, u32 val) 34 { 35 writel_relaxed(val, GITS_BASE_GVA + offset); 36 } 37 38 static unsigned long its_find_baser(unsigned int type) 39 { 40 int i; 41 42 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 43 u64 baser; 44 unsigned long offset = GITS_BASER + (i * sizeof(baser)); 45 46 baser = its_read_u64(offset); 47 if (GITS_BASER_TYPE(baser) == type) 48 return offset; 49 } 50 51 GUEST_FAIL("Couldn't find an ITS BASER of type %u", type); 52 return -1; 53 } 54 55 static void its_install_table(unsigned int type, vm_paddr_t base, size_t size) 56 { 57 unsigned long offset = its_find_baser(type); 58 u64 baser; 59 60 baser = ((size / SZ_64K) - 1) | 61 GITS_BASER_PAGE_SIZE_64K | 62 GITS_BASER_InnerShareable | 63 base | 64 GITS_BASER_RaWaWb | 65 GITS_BASER_VALID; 66 67 its_write_u64(offset, baser); 68 } 69 70 static void its_install_cmdq(vm_paddr_t base, size_t size) 71 { 72 u64 cbaser; 73 74 cbaser = ((size / SZ_4K) - 1) | 75 GITS_CBASER_InnerShareable | 76 base | 77 GITS_CBASER_RaWaWb | 78 GITS_CBASER_VALID; 79 80 its_write_u64(GITS_CBASER, cbaser); 81 } 82 83 void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz, 84 vm_paddr_t device_tbl, size_t device_tbl_sz, 85 vm_paddr_t cmdq, size_t cmdq_size) 86 { 87 u32 ctlr; 88 89 its_install_table(GITS_BASER_TYPE_COLLECTION, coll_tbl, coll_tbl_sz); 90 its_install_table(GITS_BASER_TYPE_DEVICE, device_tbl, device_tbl_sz); 91 its_install_cmdq(cmdq, cmdq_size); 92 93 ctlr = its_read_u32(GITS_CTLR); 94 ctlr |= GITS_CTLR_ENABLE; 95 its_write_u32(GITS_CTLR, ctlr); 96 } 97 98 struct its_cmd_block { 99 union { 100 u64 raw_cmd[4]; 101 __le64 raw_cmd_le[4]; 102 }; 103 }; 104 105 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 106 { 107 /* Let's fixup BE commands */ 108 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); 109 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); 110 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); 111 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); 112 } 113 114 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 115 { 116 u64 mask = GENMASK_ULL(h, l); 117 *raw_cmd &= ~mask; 118 *raw_cmd |= (val << l) & mask; 119 } 120 121 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 122 { 123 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 124 } 125 126 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 127 { 128 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 129 } 130 131 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 132 { 133 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 134 } 135 136 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 137 { 138 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 139 } 140 141 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 142 { 143 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 144 } 145 146 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 147 { 148 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 149 } 150 151 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 152 { 153 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 154 } 155 156 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 157 { 158 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 159 } 160 161 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 162 { 163 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 164 } 165 166 #define GITS_CMDQ_POLL_ITERATIONS 0 167 168 static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd) 169 { 170 u64 cwriter = its_read_u64(GITS_CWRITER); 171 struct its_cmd_block *dst = cmdq_base + cwriter; 172 u64 cbaser = its_read_u64(GITS_CBASER); 173 size_t cmdq_size; 174 u64 next; 175 int i; 176 177 cmdq_size = ((cbaser & 0xFF) + 1) * SZ_4K; 178 179 its_fixup_cmd(cmd); 180 181 WRITE_ONCE(*dst, *cmd); 182 dsb(ishst); 183 next = (cwriter + sizeof(*cmd)) % cmdq_size; 184 its_write_u64(GITS_CWRITER, next); 185 186 /* 187 * Polling isn't necessary considering KVM's ITS emulation at the time 188 * of writing this, as the CMDQ is processed synchronously after a write 189 * to CWRITER. 190 */ 191 for (i = 0; its_read_u64(GITS_CREADR) != next; i++) { 192 __GUEST_ASSERT(i < GITS_CMDQ_POLL_ITERATIONS, 193 "ITS didn't process command at offset %lu after %d iterations\n", 194 cwriter, i); 195 196 cpu_relax(); 197 } 198 } 199 200 void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base, 201 size_t itt_size, bool valid) 202 { 203 struct its_cmd_block cmd = {}; 204 205 its_encode_cmd(&cmd, GITS_CMD_MAPD); 206 its_encode_devid(&cmd, device_id); 207 its_encode_size(&cmd, ilog2(itt_size) - 1); 208 its_encode_itt(&cmd, itt_base); 209 its_encode_valid(&cmd, valid); 210 211 its_send_cmd(cmdq_base, &cmd); 212 } 213 214 void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid) 215 { 216 struct its_cmd_block cmd = {}; 217 218 its_encode_cmd(&cmd, GITS_CMD_MAPC); 219 its_encode_collection(&cmd, collection_id); 220 its_encode_target(&cmd, vcpu_id); 221 its_encode_valid(&cmd, valid); 222 223 its_send_cmd(cmdq_base, &cmd); 224 } 225 226 void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id, 227 u32 collection_id, u32 intid) 228 { 229 struct its_cmd_block cmd = {}; 230 231 its_encode_cmd(&cmd, GITS_CMD_MAPTI); 232 its_encode_devid(&cmd, device_id); 233 its_encode_event_id(&cmd, event_id); 234 its_encode_phys_id(&cmd, intid); 235 its_encode_collection(&cmd, collection_id); 236 237 its_send_cmd(cmdq_base, &cmd); 238 } 239 240 void its_send_invall_cmd(void *cmdq_base, u32 collection_id) 241 { 242 struct its_cmd_block cmd = {}; 243 244 its_encode_cmd(&cmd, GITS_CMD_INVALL); 245 its_encode_collection(&cmd, collection_id); 246 247 its_send_cmd(cmdq_base, &cmd); 248 } 249