Lines Matching full:its
3 * GICv3 ITS emulation
28 static int vgic_its_save_tables_v0(struct vgic_its *its);
29 static int vgic_its_restore_tables_v0(struct vgic_its *its);
30 static int vgic_its_commit_v0(struct vgic_its *its);
71 * If this LPI is already mapped on another ITS, we increase its refcount
158 * struct vgic_its_abi - ITS abi ops and settings
162 * @save_tables: save the ITS tables into guest RAM
163 * @restore_tables: restore the ITS internal structs from tables
172 int (*save_tables)(struct vgic_its *its);
173 int (*restore_tables)(struct vgic_its *its);
174 int (*commit)(struct vgic_its *its);
193 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its) in vgic_its_get_abi() argument
195 return &its_table_abi_versions[its->abi_rev]; in vgic_its_get_abi()
198 static int vgic_its_set_abi(struct vgic_its *its, u32 rev) in vgic_its_set_abi() argument
202 its->abi_rev = rev; in vgic_its_set_abi()
203 abi = vgic_its_get_abi(its); in vgic_its_set_abi()
204 return abi->commit(its); in vgic_its_set_abi()
208 * Find and returns a device in the device table for an ITS.
211 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id) in find_its_device() argument
215 list_for_each_entry(device, &its->device_list, dev_list) in find_its_device()
224 * Device ID/Event ID pair on an ITS.
227 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id, in find_ite() argument
233 device = find_its_device(its, device_id); in find_ite()
245 #define for_each_lpi_its(dev, ite, its) \ argument
246 list_for_each_entry(dev, &(its)->device_list, dev_list) \
259 * Finds and returns a collection in the ITS collection table.
262 static struct its_collection *find_collection(struct vgic_its *its, int coll_id) in find_collection() argument
266 list_for_each_entry(collection, &its->collection_list, coll_list) { in find_collection()
346 * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
366 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its, in update_affinity_collection() argument
372 for_each_lpi_its(device, ite, its) { in update_affinity_collection()
437 struct vgic_its *its, in vgic_mmio_read_its_typer() argument
440 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_mmio_read_its_typer()
459 struct vgic_its *its, in vgic_mmio_read_its_iidr() argument
464 val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK; in vgic_mmio_read_its_iidr()
470 struct vgic_its *its, in vgic_mmio_uaccess_write_its_iidr() argument
478 return vgic_its_set_abi(its, rev); in vgic_mmio_uaccess_write_its_iidr()
482 struct vgic_its *its, in vgic_mmio_read_its_idregs() argument
524 return iodev->its; in __vgic_doorbell_to_its()
537 struct vgic_its *its; in vgic_its_check_cache() local
543 its = __vgic_doorbell_to_its(kvm, db); in vgic_its_check_cache()
544 if (IS_ERR(its)) in vgic_its_check_cache()
549 irq = xa_load(&its->translation_cache, cache_key); in vgic_its_check_cache()
558 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its, in vgic_its_cache_translation() argument
573 lockdep_assert_held(&its->its_lock); in vgic_its_cache_translation()
576 old = xa_store(&its->translation_cache, cache_key, irq, GFP_KERNEL_ACCOUNT); in vgic_its_cache_translation()
596 static void vgic_its_invalidate_cache(struct vgic_its *its) in vgic_its_invalidate_cache() argument
598 struct kvm *kvm = its->dev->kvm; in vgic_its_invalidate_cache()
602 xa_for_each(&its->translation_cache, idx, irq) { in vgic_its_invalidate_cache()
603 xa_erase(&its->translation_cache, idx); in vgic_its_invalidate_cache()
611 struct vgic_its *its; in vgic_its_invalidate_all_caches() local
619 its = dev->private; in vgic_its_invalidate_all_caches()
620 vgic_its_invalidate_cache(its); in vgic_its_invalidate_all_caches()
626 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, in vgic_its_resolve_lpi() argument
632 if (!its->enabled) in vgic_its_resolve_lpi()
635 ite = find_ite(its, devid, eventid); in vgic_its_resolve_lpi()
646 vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq); in vgic_its_resolve_lpi()
671 * Returns 0 on success, a positive error value for any ITS mapping
674 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its, in vgic_its_trigger_msi() argument
681 err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq); in vgic_its_trigger_msi()
716 * Queries the KVM IO bus framework to get the ITS pointer from the given
723 struct vgic_its *its; in vgic_its_inject_msi() local
729 its = vgic_msi_to_its(kvm, msi); in vgic_its_inject_msi()
730 if (IS_ERR(its)) in vgic_its_inject_msi()
731 return PTR_ERR(its); in vgic_its_inject_msi()
733 mutex_lock(&its->its_lock); in vgic_its_inject_msi()
734 ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data); in vgic_its_inject_msi()
735 mutex_unlock(&its->its_lock); in vgic_its_inject_msi()
791 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_discard() argument
798 ite = find_ite(its, device_id, event_id); in vgic_its_cmd_handle_discard()
800 struct its_device *device = find_its_device(its, device_id); in vgic_its_cmd_handle_discard()
801 int ite_esz = vgic_its_get_abi(its)->ite_esz; in vgic_its_cmd_handle_discard()
808 vgic_its_invalidate_cache(its); in vgic_its_cmd_handle_discard()
812 return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); in vgic_its_cmd_handle_discard()
822 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_movi() argument
832 ite = find_ite(its, device_id, event_id); in vgic_its_cmd_handle_movi()
839 collection = find_collection(its, coll_id); in vgic_its_cmd_handle_movi()
846 vgic_its_invalidate_cache(its); in vgic_its_cmd_handle_movi()
851 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa) in __is_visible_gfn_locked() argument
857 idx = srcu_read_lock(&its->dev->kvm->srcu); in __is_visible_gfn_locked()
858 ret = kvm_is_visible_gfn(its->dev->kvm, gfn); in __is_visible_gfn_locked()
859 srcu_read_unlock(&its->dev->kvm->srcu, idx); in __is_visible_gfn_locked()
870 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, in vgic_its_check_id() argument
885 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */ in vgic_its_check_id()
904 return __is_visible_gfn_locked(its, addr); in vgic_its_check_id()
913 if (kvm_read_guest_lock(its->dev->kvm, in vgic_its_check_id()
934 return __is_visible_gfn_locked(its, indirect_ptr); in vgic_its_check_id()
941 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device, in vgic_its_check_event_id() argument
944 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_check_event_id()
953 return __is_visible_gfn_locked(its, gpa); in vgic_its_check_event_id()
957 * Add a new collection into the ITS collection table.
960 static int vgic_its_alloc_collection(struct vgic_its *its, in vgic_its_alloc_collection() argument
973 list_add_tail(&collection->coll_list, &its->collection_list); in vgic_its_alloc_collection()
979 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id) in vgic_its_free_collection() argument
990 collection = find_collection(its, coll_id); in vgic_its_free_collection()
994 for_each_lpi_its(device, ite, its) in vgic_its_free_collection()
1025 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_mapi() argument
1038 device = find_its_device(its, device_id); in vgic_its_cmd_handle_mapi()
1042 if (!vgic_its_check_event_id(its, device, event_id)) in vgic_its_cmd_handle_mapi()
1054 if (find_ite(its, device_id, event_id)) in vgic_its_cmd_handle_mapi()
1057 collection = find_collection(its, coll_id); in vgic_its_cmd_handle_mapi()
1061 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) in vgic_its_cmd_handle_mapi()
1064 ret = vgic_its_alloc_collection(its, &collection, coll_id); in vgic_its_cmd_handle_mapi()
1073 vgic_its_free_collection(its, coll_id); in vgic_its_cmd_handle_mapi()
1083 vgic_its_free_collection(its, coll_id); in vgic_its_cmd_handle_mapi()
1093 static void vgic_its_free_device(struct kvm *kvm, struct vgic_its *its, in vgic_its_free_device() argument
1106 vgic_its_invalidate_cache(its); in vgic_its_free_device()
1112 /* its lock must be held */
1113 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its) in vgic_its_free_device_list() argument
1117 list_for_each_entry_safe(cur, temp, &its->device_list, dev_list) in vgic_its_free_device_list()
1118 vgic_its_free_device(kvm, its, cur); in vgic_its_free_device_list()
1121 /* its lock must be held */
1122 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its) in vgic_its_free_collection_list() argument
1126 list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list) in vgic_its_free_collection_list()
1127 vgic_its_free_collection(its, cur->collection_id); in vgic_its_free_collection_list()
1131 static struct its_device *vgic_its_alloc_device(struct vgic_its *its, in vgic_its_alloc_device() argument
1146 list_add_tail(&device->dev_list, &its->device_list); in vgic_its_alloc_device()
1154 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_mapd() argument
1164 if (!vgic_its_check_id(its, its->baser_device_table, device_id, &gpa)) in vgic_its_cmd_handle_mapd()
1170 device = find_its_device(its, device_id); in vgic_its_cmd_handle_mapd()
1178 vgic_its_free_device(kvm, its, device); in vgic_its_cmd_handle_mapd()
1185 return vgic_its_write_entry_lock(its, gpa, 0ULL, dte); in vgic_its_cmd_handle_mapd()
1187 device = vgic_its_alloc_device(its, device_id, itt_addr, in vgic_its_cmd_handle_mapd()
1197 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_mapc() argument
1208 vgic_its_free_collection(its, coll_id); in vgic_its_cmd_handle_mapc()
1209 vgic_its_invalidate_cache(its); in vgic_its_cmd_handle_mapc()
1217 collection = find_collection(its, coll_id); in vgic_its_cmd_handle_mapc()
1222 if (!vgic_its_check_id(its, its->baser_coll_table, in vgic_its_cmd_handle_mapc()
1226 ret = vgic_its_alloc_collection(its, &collection, in vgic_its_cmd_handle_mapc()
1233 update_affinity_collection(kvm, its, collection); in vgic_its_cmd_handle_mapc()
1244 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_clear() argument
1252 ite = find_ite(its, device_id, event_id); in vgic_its_cmd_handle_clear()
1274 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_inv() argument
1282 ite = find_ite(its, device_id, event_id); in vgic_its_cmd_handle_inv()
1294 * collection, and we don't need to hold the its_lock, since no ITS is
1327 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_invall() argument
1334 collection = find_collection(its, coll_id); in vgic_its_cmd_handle_invall()
1352 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_movall() argument
1380 vgic_its_invalidate_cache(its); in vgic_its_cmd_handle_movall()
1389 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its, in vgic_its_cmd_handle_int() argument
1395 return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data); in vgic_its_cmd_handle_int()
1399 * This function is called with the its_cmd lock held, but the ITS data
1402 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its, in vgic_its_handle_command() argument
1407 mutex_lock(&its->its_lock); in vgic_its_handle_command()
1410 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd); in vgic_its_handle_command()
1413 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd); in vgic_its_handle_command()
1416 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd); in vgic_its_handle_command()
1419 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd); in vgic_its_handle_command()
1422 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd); in vgic_its_handle_command()
1425 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd); in vgic_its_handle_command()
1428 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd); in vgic_its_handle_command()
1431 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd); in vgic_its_handle_command()
1434 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd); in vgic_its_handle_command()
1437 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd); in vgic_its_handle_command()
1440 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd); in vgic_its_handle_command()
1447 mutex_unlock(&its->its_lock); in vgic_its_handle_command()
1464 /* We support only one (ITS) page size: 64K */ in vgic_sanitise_its_baser()
1489 struct vgic_its *its, in vgic_mmio_read_its_cbaser() argument
1492 return extract_bytes(its->cbaser, addr & 7, len); in vgic_mmio_read_its_cbaser()
1495 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its, in vgic_mmio_write_its_cbaser() argument
1500 if (its->enabled) in vgic_mmio_write_its_cbaser()
1503 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_cbaser()
1504 its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val); in vgic_mmio_write_its_cbaser()
1505 its->cbaser = vgic_sanitise_its_cbaser(its->cbaser); in vgic_mmio_write_its_cbaser()
1506 its->creadr = 0; in vgic_mmio_write_its_cbaser()
1511 its->cwriter = its->creadr; in vgic_mmio_write_its_cbaser()
1512 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cbaser()
1520 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its) in vgic_its_process_commands() argument
1525 /* Commands are only processed when the ITS is enabled. */ in vgic_its_process_commands()
1526 if (!its->enabled) in vgic_its_process_commands()
1529 cbaser = GITS_CBASER_ADDRESS(its->cbaser); in vgic_its_process_commands()
1531 while (its->cwriter != its->creadr) { in vgic_its_process_commands()
1532 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr, in vgic_its_process_commands()
1542 vgic_its_handle_command(kvm, its, cmd_buf); in vgic_its_process_commands()
1544 its->creadr += ITS_CMD_SIZE; in vgic_its_process_commands()
1545 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser)) in vgic_its_process_commands()
1546 its->creadr = 0; in vgic_its_process_commands()
1554 * per ITS handling commands at a given time.
1556 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its, in vgic_mmio_write_its_cwriter() argument
1562 if (!its) in vgic_mmio_write_its_cwriter()
1565 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1567 reg = update_64bit_reg(its->cwriter, addr & 7, len, val); in vgic_mmio_write_its_cwriter()
1569 if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) { in vgic_mmio_write_its_cwriter()
1570 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1573 its->cwriter = reg; in vgic_mmio_write_its_cwriter()
1575 vgic_its_process_commands(kvm, its); in vgic_mmio_write_its_cwriter()
1577 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_cwriter()
1581 struct vgic_its *its, in vgic_mmio_read_its_cwriter() argument
1584 return extract_bytes(its->cwriter, addr & 0x7, len); in vgic_mmio_read_its_cwriter()
1588 struct vgic_its *its, in vgic_mmio_read_its_creadr() argument
1591 return extract_bytes(its->creadr, addr & 0x7, len); in vgic_mmio_read_its_creadr()
1595 struct vgic_its *its, in vgic_mmio_uaccess_write_its_creadr() argument
1602 mutex_lock(&its->cmd_lock); in vgic_mmio_uaccess_write_its_creadr()
1604 if (its->enabled) { in vgic_mmio_uaccess_write_its_creadr()
1610 if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) { in vgic_mmio_uaccess_write_its_creadr()
1615 its->creadr = cmd_offset; in vgic_mmio_uaccess_write_its_creadr()
1617 mutex_unlock(&its->cmd_lock); in vgic_mmio_uaccess_write_its_creadr()
1623 struct vgic_its *its, in vgic_mmio_read_its_baser() argument
1630 reg = its->baser_device_table; in vgic_mmio_read_its_baser()
1633 reg = its->baser_coll_table; in vgic_mmio_read_its_baser()
1645 struct vgic_its *its, in vgic_mmio_write_its_baser() argument
1649 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_mmio_write_its_baser()
1654 if (its->enabled) in vgic_mmio_write_its_baser()
1659 regptr = &its->baser_device_table; in vgic_mmio_write_its_baser()
1664 regptr = &its->baser_coll_table; in vgic_mmio_write_its_baser()
1685 mutex_lock(&its->its_lock); in vgic_mmio_write_its_baser()
1688 vgic_its_free_device_list(kvm, its); in vgic_mmio_write_its_baser()
1691 vgic_its_free_collection_list(kvm, its); in vgic_mmio_write_its_baser()
1694 mutex_unlock(&its->its_lock); in vgic_mmio_write_its_baser()
1699 struct vgic_its *its, in vgic_mmio_read_its_ctlr() argument
1704 mutex_lock(&its->cmd_lock); in vgic_mmio_read_its_ctlr()
1705 if (its->creadr == its->cwriter) in vgic_mmio_read_its_ctlr()
1707 if (its->enabled) in vgic_mmio_read_its_ctlr()
1709 mutex_unlock(&its->cmd_lock); in vgic_mmio_read_its_ctlr()
1714 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its, in vgic_mmio_write_its_ctlr() argument
1718 mutex_lock(&its->cmd_lock); in vgic_mmio_write_its_ctlr()
1721 * It is UNPREDICTABLE to enable the ITS if any of the CBASER or in vgic_mmio_write_its_ctlr()
1724 if (!its->enabled && (val & GITS_CTLR_ENABLE) && in vgic_mmio_write_its_ctlr()
1725 (!(its->baser_device_table & GITS_BASER_VALID) || in vgic_mmio_write_its_ctlr()
1726 !(its->baser_coll_table & GITS_BASER_VALID) || in vgic_mmio_write_its_ctlr()
1727 !(its->cbaser & GITS_CBASER_VALID))) in vgic_mmio_write_its_ctlr()
1730 its->enabled = !!(val & GITS_CTLR_ENABLE); in vgic_mmio_write_its_ctlr()
1731 if (!its->enabled) in vgic_mmio_write_its_ctlr()
1732 vgic_its_invalidate_cache(its); in vgic_mmio_write_its_ctlr()
1736 * if the ITS is disabled or no commands have been queued. in vgic_mmio_write_its_ctlr()
1738 vgic_its_process_commands(kvm, its); in vgic_mmio_write_its_ctlr()
1741 mutex_unlock(&its->cmd_lock); in vgic_mmio_write_its_ctlr()
1763 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its, in its_mmio_write_wi() argument
1805 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its, in vgic_register_its_iodev() argument
1808 struct vgic_io_device *iodev = &its->iodev; in vgic_register_its_iodev()
1812 if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) { in vgic_register_its_iodev()
1817 its->vgic_its_base = addr; in vgic_register_its_iodev()
1822 iodev->base_addr = its->vgic_its_base; in vgic_register_its_iodev()
1824 iodev->its = its; in vgic_register_its_iodev()
1847 struct vgic_its *its; in vgic_its_create() local
1852 its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT); in vgic_its_create()
1853 if (!its) in vgic_its_create()
1862 kfree(its); in vgic_its_create()
1867 mutex_init(&its->its_lock); in vgic_its_create()
1868 mutex_init(&its->cmd_lock); in vgic_its_create()
1872 mutex_lock(&its->cmd_lock); in vgic_its_create()
1873 mutex_lock(&its->its_lock); in vgic_its_create()
1874 mutex_unlock(&its->its_lock); in vgic_its_create()
1875 mutex_unlock(&its->cmd_lock); in vgic_its_create()
1878 its->vgic_its_base = VGIC_ADDR_UNDEF; in vgic_its_create()
1880 INIT_LIST_HEAD(&its->device_list); in vgic_its_create()
1881 INIT_LIST_HEAD(&its->collection_list); in vgic_its_create()
1882 xa_init(&its->translation_cache); in vgic_its_create()
1886 its->enabled = false; in vgic_its_create()
1887 its->dev = dev; in vgic_its_create()
1889 its->baser_device_table = INITIAL_BASER_VALUE | in vgic_its_create()
1891 its->baser_coll_table = INITIAL_BASER_VALUE | in vgic_its_create()
1895 dev->private = its; in vgic_its_create()
1897 ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1); in vgic_its_create()
1907 struct vgic_its *its = kvm_dev->private; in vgic_its_destroy() local
1909 mutex_lock(&its->its_lock); in vgic_its_destroy()
1913 vgic_its_free_device_list(kvm, its); in vgic_its_destroy()
1914 vgic_its_free_collection_list(kvm, its); in vgic_its_destroy()
1915 vgic_its_invalidate_cache(its); in vgic_its_destroy()
1916 xa_destroy(&its->translation_cache); in vgic_its_destroy()
1918 mutex_unlock(&its->its_lock); in vgic_its_destroy()
1919 kfree(its); in vgic_its_destroy()
1949 struct vgic_its *its; in vgic_its_attr_regs_access() local
1954 its = dev->private; in vgic_its_attr_regs_access()
1959 * 64-bit ITS registers, the userspace ABI requires 64-bit in vgic_its_attr_regs_access()
1981 if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) { in vgic_its_attr_regs_access()
1994 addr = its->vgic_its_base + offset; in vgic_its_attr_regs_access()
2000 ret = region->uaccess_its_write(dev->kvm, its, addr, in vgic_its_attr_regs_access()
2003 region->its_write(dev->kvm, its, addr, len, *reg); in vgic_its_attr_regs_access()
2005 *reg = region->its_read(dev->kvm, its, addr, len); in vgic_its_attr_regs_access()
2043 * @its: its handle
2051 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2058 * @its: its handle
2070 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz, in scan_its_table() argument
2073 struct kvm *kvm = its->dev->kvm; in scan_its_table()
2090 next_offset = fn(its, id, entry, opaque); in scan_its_table()
2108 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, in vgic_its_save_ite() argument
2120 return vgic_its_write_entry_lock(its, gpa, val, ite); in vgic_its_save_ite()
2126 * @its: its handle
2131 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id, in vgic_its_restore_ite() argument
2136 struct kvm *kvm = its->dev->kvm; in vgic_its_restore_ite()
2162 collection = find_collection(its, coll_id); in vgic_its_restore_ite()
2166 if (!vgic_its_check_event_id(its, dev, event_id)) in vgic_its_restore_ite()
2198 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device) in vgic_its_save_itt() argument
2200 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_save_itt()
2220 ret = vgic_its_save_ite(its, device, ite, gpa); in vgic_its_save_itt()
2230 * @its: its handle
2235 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev) in vgic_its_restore_itt() argument
2237 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_restore_itt()
2243 ret = scan_its_table(its, base, max_size, ite_esz, 0, in vgic_its_restore_itt()
2256 * @its: ITS handle
2257 * @dev: ITS device
2260 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev, in vgic_its_save_dte() argument
2267 next_offset = compute_next_devid_offset(&its->device_list, dev); in vgic_its_save_dte()
2274 return vgic_its_write_entry_lock(its, ptr, val, dte); in vgic_its_save_dte()
2280 * @its: its handle
2288 static int vgic_its_restore_dte(struct vgic_its *its, u32 id, in vgic_its_restore_dte() argument
2292 u64 baser = its->baser_device_table; in vgic_its_restore_dte()
2313 if (!vgic_its_check_id(its, baser, id, NULL)) in vgic_its_restore_dte()
2316 dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits); in vgic_its_restore_dte()
2320 ret = vgic_its_restore_itt(its, dev); in vgic_its_restore_dte()
2322 vgic_its_free_device(its->dev->kvm, its, dev); in vgic_its_restore_dte()
2348 static int vgic_its_save_device_tables(struct vgic_its *its) in vgic_its_save_device_tables() argument
2350 u64 baser = its->baser_device_table; in vgic_its_save_device_tables()
2356 list_sort(NULL, &its->device_list, vgic_its_device_cmp); in vgic_its_save_device_tables()
2358 list_for_each_entry(dev, &its->device_list, dev_list) { in vgic_its_save_device_tables()
2362 if (!vgic_its_check_id(its, baser, in vgic_its_save_device_tables()
2366 ret = vgic_its_save_itt(its, dev); in vgic_its_save_device_tables()
2370 ret = vgic_its_save_dte(its, dev, eaddr); in vgic_its_save_device_tables()
2380 * @its: its handle
2389 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr, in handle_l1_dte() argument
2392 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in handle_l1_dte()
2406 ret = scan_its_table(its, gpa, SZ_64K, dte_esz, in handle_l1_dte()
2416 static int vgic_its_restore_device_tables(struct vgic_its *its) in vgic_its_restore_device_tables() argument
2418 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_restore_device_tables()
2419 u64 baser = its->baser_device_table; in vgic_its_restore_device_tables()
2431 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0, in vgic_its_restore_device_tables()
2435 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0, in vgic_its_restore_device_tables()
2444 vgic_its_free_device_list(its->dev->kvm, its); in vgic_its_restore_device_tables()
2449 static int vgic_its_save_cte(struct vgic_its *its, in vgic_its_save_cte() argument
2460 return vgic_its_write_entry_lock(its, gpa, val, cte); in vgic_its_save_cte()
2464 * Restore a collection entry into the ITS collection table.
2468 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa) in vgic_its_restore_cte() argument
2471 struct kvm *kvm = its->dev->kvm; in vgic_its_restore_cte()
2476 ret = vgic_its_read_entry_lock(its, gpa, &val, cte); in vgic_its_restore_cte()
2490 collection = find_collection(its, coll_id); in vgic_its_restore_cte()
2494 if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) in vgic_its_restore_cte()
2497 ret = vgic_its_alloc_collection(its, &collection, coll_id); in vgic_its_restore_cte()
2508 static int vgic_its_save_collection_table(struct vgic_its *its) in vgic_its_save_collection_table() argument
2510 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_save_collection_table()
2511 u64 baser = its->baser_coll_table; in vgic_its_save_collection_table()
2522 list_for_each_entry(collection, &its->collection_list, coll_list) { in vgic_its_save_collection_table()
2523 ret = vgic_its_save_cte(its, collection, gpa); in vgic_its_save_collection_table()
2537 return vgic_its_write_entry_lock(its, gpa, 0ULL, cte); in vgic_its_save_collection_table()
2542 * in guest memory and restores the ITS internal state. Requires the
2545 static int vgic_its_restore_collection_table(struct vgic_its *its) in vgic_its_restore_collection_table() argument
2547 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_restore_collection_table()
2548 u64 baser = its->baser_coll_table; in vgic_its_restore_collection_table()
2562 ret = vgic_its_restore_cte(its, gpa); in vgic_its_restore_collection_table()
2573 vgic_its_free_collection_list(its->dev->kvm, its); in vgic_its_restore_collection_table()
2579 * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2582 static int vgic_its_save_tables_v0(struct vgic_its *its) in vgic_its_save_tables_v0() argument
2586 ret = vgic_its_save_device_tables(its); in vgic_its_save_tables_v0()
2590 return vgic_its_save_collection_table(its); in vgic_its_save_tables_v0()
2594 * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2598 static int vgic_its_restore_tables_v0(struct vgic_its *its) in vgic_its_restore_tables_v0() argument
2602 ret = vgic_its_restore_collection_table(its); in vgic_its_restore_tables_v0()
2606 ret = vgic_its_restore_device_tables(its); in vgic_its_restore_tables_v0()
2608 vgic_its_free_collection_list(its->dev->kvm, its); in vgic_its_restore_tables_v0()
2612 static int vgic_its_commit_v0(struct vgic_its *its) in vgic_its_commit_v0() argument
2616 abi = vgic_its_get_abi(its); in vgic_its_commit_v0()
2617 its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK; in vgic_its_commit_v0()
2618 its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK; in vgic_its_commit_v0()
2620 its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5) in vgic_its_commit_v0()
2623 its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5) in vgic_its_commit_v0()
2628 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its) in vgic_its_reset() argument
2631 its->baser_coll_table &= ~GITS_BASER_VALID; in vgic_its_reset()
2632 its->baser_device_table &= ~GITS_BASER_VALID; in vgic_its_reset()
2633 its->cbaser = 0; in vgic_its_reset()
2634 its->creadr = 0; in vgic_its_reset()
2635 its->cwriter = 0; in vgic_its_reset()
2636 its->enabled = 0; in vgic_its_reset()
2637 vgic_its_free_device_list(kvm, its); in vgic_its_reset()
2638 vgic_its_free_collection_list(kvm, its); in vgic_its_reset()
2669 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr) in vgic_its_ctrl() argument
2671 const struct vgic_its_abi *abi = vgic_its_get_abi(its); in vgic_its_ctrl()
2685 mutex_lock(&its->its_lock); in vgic_its_ctrl()
2689 vgic_its_reset(kvm, its); in vgic_its_ctrl()
2692 ret = abi->save_tables(its); in vgic_its_ctrl()
2695 ret = abi->restore_tables(its); in vgic_its_ctrl()
2702 mutex_unlock(&its->its_lock); in vgic_its_ctrl()
2715 * dirty guest pages. When vgic/its tables are being saved, the backup
2729 struct vgic_its *its = dev->private; in vgic_its_set_attr() local
2744 ret = vgic_check_iorange(dev->kvm, its->vgic_its_base, in vgic_its_set_attr()
2749 ret = vgic_register_its_iodev(dev->kvm, its, addr); in vgic_its_set_attr()
2757 return vgic_its_ctrl(dev->kvm, its, attr->attr); in vgic_its_set_attr()
2776 struct vgic_its *its = dev->private; in vgic_its_get_attr() local
2777 u64 addr = its->vgic_its_base; in vgic_its_get_attr()
2806 .name = "kvm-arm-vgic-its",