1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
37 #include <linux/irqchip/arm-gic-v3.h>
38 #include <linux/irqchip/arm-gic-v4.h>
43 #include "irq-gic-common.h"
44 #include "irq-gic-its-msi-parent.h"
45 #include <linux/irqchip/irq-msi-lib.h>
62  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
72  * Collection structure - just an ID, and a redistributor address to
82  * The ITS_BASER structure - contains memory information, cached
95  * The ITS structure - contains most of the infrastructure, with the
96  * top-level MSI domain, the command queue, the collections, and the
131 #define is_v4(its)		(!!((its)->typer & GITS_TYPER_VLPIS))
132 #define is_v4_1(its)		(!!((its)->typer & GITS_TYPER_VMAPP))
133 #define device_ids(its)		(FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
137 /* The maximum number of VPEID bits supported by VLPI commands */
141 		if (gic_rdists->has_rvpeid &&				\
142 		    gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)		\
143 			nvpeid = 1 + (gic_rdists->gicd_typer2 &		\
165  * The ITS view of a device - belongs to an ITS, owns an interrupt
206 #define gic_data_rdist()		(raw_cpu_ptr(gic_rdists->rdist))
207 #define gic_data_rdist_cpu(cpu)		(per_cpu_ptr(gic_rdists->rdist, cpu))
208 #define gic_data_rdist_rd_base()	(gic_data_rdist()->rd_base)
300 	return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);  in require_its_list_vmovp()
305 	return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE);  in rdists_support_shareable()
318 			__set_bit(its->list_nr, &its_list);  in get_its_list()
327 	return d->hwirq - its_dev->event_map.lpi_base;  in its_get_event_id()
333 	struct its_node *its = its_dev->its;  in dev_event_to_col()
335 	return its->collections + its_dev->event_map.col_map[event];  in dev_event_to_col()
341 	if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))  in dev_event_to_vlpi_map()
344 	return &its_dev->event_map.vlpi_maps[event];  in dev_event_to_vlpi_map()
361 	raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);  in vpe_to_cpuid_lock()
362 	return vpe->col_idx;  in vpe_to_cpuid_lock()
367 	raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);  in vpe_to_cpuid_unlock()
377 	if (d->chip == &its_vpe_irq_chip) {  in irq_to_cpuid_lock()
382 			vpe = map->vpe;  in irq_to_cpuid_lock()
390 		cpu = its_dev->event_map.col_map[its_get_event_id(d)];  in irq_to_cpuid_lock()
402 	if (d->chip == &its_vpe_irq_chip) {  in irq_to_cpuid_unlock()
407 			vpe = map->vpe;  in irq_to_cpuid_unlock()
416 	if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))  in valid_col()
424 	if (valid_col(its->collections + vpe->col_idx))  in valid_vpe()
431  * ITS command descriptors - parameters to be encoded in a command
559 	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);  in its_encode_cmd()
564 	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);  in its_encode_devid()
569 	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);  in its_encode_event_id()
574 	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);  in its_encode_phys_id()
579 	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);  in its_encode_size()
584 	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);  in its_encode_itt()
589 	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);  in its_encode_valid()
594 	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);  in its_encode_target()
599 	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);  in its_encode_collection()
604 	its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);  in its_encode_vpeid()
609 	its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);  in its_encode_virt_id()
614 	its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);  in its_encode_db_phys_id()
619 	its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);  in its_encode_db_valid()
624 	its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);  in its_encode_seq_num()
629 	its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);  in its_encode_its_list()
634 	its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);  in its_encode_vpt_addr()
639 	its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);  in its_encode_vpt_size()
644 	its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);  in its_encode_vconf_addr()
649 	its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);  in its_encode_alloc()
654 	its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);  in its_encode_ptz()
660 	its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);  in its_encode_vmapp_default_db()
666 	its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);  in its_encode_vmovp_default_db()
671 	its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);  in its_encode_db()
676 	its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);  in its_encode_sgi_intid()
681 	its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);  in its_encode_sgi_priority()
686 	its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);  in its_encode_sgi_group()
691 	its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);  in its_encode_sgi_clear()
696 	its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);  in its_encode_sgi_enable()
702 	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);  in its_fixup_cmd()
703 	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);  in its_fixup_cmd()
704 	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);  in its_fixup_cmd()
705 	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);  in its_fixup_cmd()
713 	u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);  in its_build_mapd_cmd()
715 	itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);  in its_build_mapd_cmd()
718 	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);  in its_build_mapd_cmd()
719 	its_encode_size(cmd, size - 1);  in its_build_mapd_cmd()
721 	its_encode_valid(cmd, desc->its_mapd_cmd.valid);  in its_build_mapd_cmd()
733 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);  in its_build_mapc_cmd()
734 	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);  in its_build_mapc_cmd()
735 	its_encode_valid(cmd, desc->its_mapc_cmd.valid);  in its_build_mapc_cmd()
739 	return desc->its_mapc_cmd.col;  in its_build_mapc_cmd()
748 	col = dev_event_to_col(desc->its_mapti_cmd.dev,  in its_build_mapti_cmd()
749 			       desc->its_mapti_cmd.event_id);  in its_build_mapti_cmd()
752 	its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);  in its_build_mapti_cmd()
753 	its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);  in its_build_mapti_cmd()
754 	its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);  in its_build_mapti_cmd()
755 	its_encode_collection(cmd, col->col_id);  in its_build_mapti_cmd()
768 	col = dev_event_to_col(desc->its_movi_cmd.dev,  in its_build_movi_cmd()
769 			       desc->its_movi_cmd.event_id);  in its_build_movi_cmd()
772 	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);  in its_build_movi_cmd()
773 	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);  in its_build_movi_cmd()
774 	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);  in its_build_movi_cmd()
787 	col = dev_event_to_col(desc->its_discard_cmd.dev,  in its_build_discard_cmd()
788 			       desc->its_discard_cmd.event_id);  in its_build_discard_cmd()
791 	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);  in its_build_discard_cmd()
792 	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);  in its_build_discard_cmd()
805 	col = dev_event_to_col(desc->its_inv_cmd.dev,  in its_build_inv_cmd()
806 			       desc->its_inv_cmd.event_id);  in its_build_inv_cmd()
809 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);  in its_build_inv_cmd()
810 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);  in its_build_inv_cmd()
823 	col = dev_event_to_col(desc->its_int_cmd.dev,  in its_build_int_cmd()
824 			       desc->its_int_cmd.event_id);  in its_build_int_cmd()
827 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);  in its_build_int_cmd()
828 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);  in its_build_int_cmd()
841 	col = dev_event_to_col(desc->its_clear_cmd.dev,  in its_build_clear_cmd()
842 			       desc->its_clear_cmd.event_id);  in its_build_clear_cmd()
845 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);  in its_build_clear_cmd()
846 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);  in its_build_clear_cmd()
858 	its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);  in its_build_invall_cmd()
862 	return desc->its_invall_cmd.col;  in its_build_invall_cmd()
870 	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);  in its_build_vinvall_cmd()
874 	return valid_vpe(its, desc->its_vinvall_cmd.vpe);  in its_build_vinvall_cmd()
881 	struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);  in its_build_vmapp_cmd()
887 	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);  in its_build_vmapp_cmd()
888 	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);  in its_build_vmapp_cmd()
890 	if (!desc->its_vmapp_cmd.valid) {  in its_build_vmapp_cmd()
891 		alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);  in its_build_vmapp_cmd()
895 			 * Unmapping a VPE is self-synchronizing on GICv4.1,  in its_build_vmapp_cmd()
904 	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));  in its_build_vmapp_cmd()
905 	target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;  in its_build_vmapp_cmd()
909 	its_encode_vpt_size(cmd, LPI_NRBITS - 1);  in its_build_vmapp_cmd()
911 	alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);  in its_build_vmapp_cmd()
916 	vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));  in its_build_vmapp_cmd()
928 	its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);  in its_build_vmapp_cmd()
942 	if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)  in its_build_vmapti_cmd()
943 		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;  in its_build_vmapti_cmd()
948 	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);  in its_build_vmapti_cmd()
949 	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);  in its_build_vmapti_cmd()
950 	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);  in its_build_vmapti_cmd()
952 	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);  in its_build_vmapti_cmd()
956 	return valid_vpe(its, desc->its_vmapti_cmd.vpe);  in its_build_vmapti_cmd()
965 	if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)  in its_build_vmovi_cmd()
966 		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;  in its_build_vmovi_cmd()
971 	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);  in its_build_vmovi_cmd()
972 	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);  in its_build_vmovi_cmd()
973 	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);  in its_build_vmovi_cmd()
979 	return valid_vpe(its, desc->its_vmovi_cmd.vpe);  in its_build_vmovi_cmd()
988 	target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;  in its_build_vmovp_cmd()
990 	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);  in its_build_vmovp_cmd()
991 	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);  in its_build_vmovp_cmd()
992 	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);  in its_build_vmovp_cmd()
997 		its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);  in its_build_vmovp_cmd()
1002 	return valid_vpe(its, desc->its_vmovp_cmd.vpe);  in its_build_vmovp_cmd()
1011 	map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,  in its_build_vinv_cmd()
1012 				    desc->its_inv_cmd.event_id);  in its_build_vinv_cmd()
1015 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);  in its_build_vinv_cmd()
1016 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);  in its_build_vinv_cmd()
1020 	return valid_vpe(its, map->vpe);  in its_build_vinv_cmd()
1029 	map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,  in its_build_vint_cmd()
1030 				    desc->its_int_cmd.event_id);  in its_build_vint_cmd()
1033 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);  in its_build_vint_cmd()
1034 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);  in its_build_vint_cmd()
1038 	return valid_vpe(its, map->vpe);  in its_build_vint_cmd()
1047 	map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,  in its_build_vclear_cmd()
1048 				    desc->its_clear_cmd.event_id);  in its_build_vclear_cmd()
1051 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);  in its_build_vclear_cmd()
1052 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);  in its_build_vclear_cmd()
1056 	return valid_vpe(its, map->vpe);  in its_build_vclear_cmd()
1067 	its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);  in its_build_invdb_cmd()
1071 	return valid_vpe(its, desc->its_invdb_cmd.vpe);  in its_build_invdb_cmd()
1082 	its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);  in its_build_vsgi_cmd()
1083 	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);  in its_build_vsgi_cmd()
1084 	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);  in its_build_vsgi_cmd()
1085 	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);  in its_build_vsgi_cmd()
1086 	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);  in its_build_vsgi_cmd()
1087 	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);  in its_build_vsgi_cmd()
1091 	return valid_vpe(its, desc->its_vsgi_cmd.vpe);  in its_build_vsgi_cmd()
1097 	return (ptr - its->cmd_base) * sizeof(*ptr);  in its_cmd_ptr_to_offset()
1105 	widx = its->cmd_write - its->cmd_base;  in its_queue_full()
1106 	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);  in its_queue_full()
1121 		count--;  in its_allocate_entry()
1130 	cmd = its->cmd_write++;  in its_allocate_entry()
1133 	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))  in its_allocate_entry()
1134 		its->cmd_write = its->cmd_base;  in its_allocate_entry()
1137 	cmd->raw_cmd[0] = 0;  in its_allocate_entry()
1138 	cmd->raw_cmd[1] = 0;  in its_allocate_entry()
1139 	cmd->raw_cmd[2] = 0;  in its_allocate_entry()
1140 	cmd->raw_cmd[3] = 0;  in its_allocate_entry()
1147 	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);  in its_post_commands()
1149 	writel_relaxed(wr, its->base + GITS_CWRITER);  in its_post_commands()
1151 	return its->cmd_write;  in its_post_commands()
1160 	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)  in its_flush_cmd()
1163 		dsb(ishst);  in its_flush_cmd()
1183 		rd_idx = readl_relaxed(its->base + GITS_CREADR);  in its_wait_for_range_completion()
1187 		 * potential wrap-around into account.  in its_wait_for_range_completion()
1189 		delta = rd_idx - prev_idx;  in its_wait_for_range_completion()
1197 		count--;  in its_wait_for_range_completion()
1201 			return -1;  in its_wait_for_range_completion()
1222 	raw_spin_lock_irqsave(&its->lock, flags);			\
1226 		raw_spin_unlock_irqrestore(&its->lock, flags);		\
1242 	rd_idx = readl_relaxed(its->base + GITS_CREADR);		\
1244 	raw_spin_unlock_irqrestore(&its->lock, flags);			\
1255 	its_encode_target(sync_cmd, sync_col->target_address);  in its_build_sync_cmd()
1268 	its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);  in BUILD_SINGLE_CMD_FUNC()
1283 	its_send_single_command(dev->its, its_build_int_cmd, &desc);  in BUILD_SINGLE_CMD_FUNC()
1293 	its_send_single_command(dev->its, its_build_clear_cmd, &desc);  in its_send_clear()
1303 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);  in its_send_inv()
1313 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);  in its_send_mapd()
1335 	its_send_single_command(dev->its, its_build_mapti_cmd, &desc);  in its_send_mapti()
1347 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);  in its_send_movi()
1357 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);  in its_send_discard()
1374 	desc.its_vmapti_cmd.vpe = map->vpe;  in its_send_vmapti()
1376 	desc.its_vmapti_cmd.virt_id = map->vintid;  in its_send_vmapti()
1378 	desc.its_vmapti_cmd.db_enabled = map->db_enabled;  in its_send_vmapti()
1380 	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);  in its_send_vmapti()
1388 	desc.its_vmovi_cmd.vpe = map->vpe;  in its_send_vmovi()
1391 	desc.its_vmovi_cmd.db_enabled = map->db_enabled;  in its_send_vmovi()
1393 	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);  in its_send_vmovi()
1403 	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];  in its_send_vmapp()
1412 	int col_id = vpe->col_idx;  in its_send_vmovp()
1418 		desc.its_vmovp_cmd.col = &its->collections[col_id];  in its_send_vmovp()
1429 	 * Wall <-- Head.  in its_send_vmovp()
1433 	desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);  in its_send_vmovp()
1440 		if (!require_its_list_vmovp(vpe->its_vm, its))  in its_send_vmovp()
1443 		desc.its_vmovp_cmd.col = &its->collections[col_id];  in its_send_vmovp()
1467 	its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);  in its_send_vinv()
1481 	its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);  in its_send_vint()
1495 	its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);  in its_send_vclear()
1507  * irqchip functions - assumes MSI, mostly.
1517 		va = page_address(map->vm->vprop_page);  in lpi_write_config()
1518 		hwirq = map->vintid;  in lpi_write_config()
1521 		map->properties &= ~clr;  in lpi_write_config()
1522 		map->properties |= set | LPI_PROP_GROUP1;  in lpi_write_config()
1524 		va = gic_rdists->prop_table_va;  in lpi_write_config()
1525 		hwirq = d->hwirq;  in lpi_write_config()
1528 	cfg = va + hwirq - 8192;  in lpi_write_config()
1537 	if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)  in lpi_write_config()
1540 		dsb(ishst);  in lpi_write_config()
1557 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);  in __direct_lpi_inv()
1559 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;  in __direct_lpi_inv()
1563 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);  in __direct_lpi_inv()
1575 		WARN_ON(!is_v4_1(its_dev->its));  in direct_lpi_inv()
1578 		val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);  in direct_lpi_inv()
1579 		val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);  in direct_lpi_inv()
1581 		val = d->hwirq;  in direct_lpi_inv()
1592 	if (gic_rdists->has_direct_lpi &&  in lpi_update_config()
1593 	    (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))  in lpi_update_config()
1608 	 * GICv4.1 does away with the per-LPI nonsense, nothing to do  in its_vlpi_set_doorbell()
1611 	if (is_v4_1(its_dev->its))  in its_vlpi_set_doorbell()
1616 	if (map->db_enabled == enable)  in its_vlpi_set_doorbell()
1619 	map->db_enabled = enable;  in its_vlpi_set_doorbell()
1653 		return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);  in its_read_lpi_count()
1655 	return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);  in its_read_lpi_count()
1661 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);  in its_inc_lpi_count()
1663 		atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);  in its_inc_lpi_count()
1669 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);  in its_dec_lpi_count()
1671 		atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);  in its_dec_lpi_count()
1704 	node = its_dev->its->numa_node;  in its_select_cpu()
1736 			if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))  in its_select_cpu()
1754 		if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&  in its_select_cpu()
1763 	pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);  in its_select_cpu()
1777 		return -EINVAL;  in its_set_affinity()
1779 	prev_cpu = its_dev->event_map.col_map[id];  in its_set_affinity()
1792 		target_col = &its_dev->its->collections[cpu];  in its_set_affinity()
1794 		its_dev->event_map.col_map[id] = cpu;  in its_set_affinity()
1804 	return -EINVAL;  in its_set_affinity()
1809 	struct its_node *its = its_dev->its;  in its_irq_get_msi_base()
1811 	return its->phys_base + GITS_TRANSLATER;  in its_irq_get_msi_base()
1818 	msg->data = its_get_event_id(d);  in its_irq_compose_msi_msg()
1820 			 its_dev->its->get_msi_base(its_dev));  in its_irq_compose_msi_msg()
1831 		return -EINVAL;  in its_irq_set_irqchip_state()
1867 	if (!its_list_map || gic_rdists->has_rvpeid)  in gic_requires_eager_mapping()
1878 	guard(raw_spinlock_irqsave)(&vm->vmapp_lock);  in its_map_vm()
1884 	vm->vlpi_count[its->list_nr]++;  in its_map_vm()
1886 	if (vm->vlpi_count[its->list_nr] == 1) {  in its_map_vm()
1889 		for (i = 0; i < vm->nr_vpes; i++) {  in its_map_vm()
1890 			struct its_vpe *vpe = vm->vpes[i];  in its_map_vm()
1892 			scoped_guard(raw_spinlock, &vpe->vpe_lock)  in its_map_vm()
1906 	guard(raw_spinlock_irqsave)(&vm->vmapp_lock);  in its_unmap_vm()
1908 	if (!--vm->vlpi_count[its->list_nr]) {  in its_unmap_vm()
1911 		for (i = 0; i < vm->nr_vpes; i++) {  in its_unmap_vm()
1912 			guard(raw_spinlock)(&vm->vpes[i]->vpe_lock);  in its_unmap_vm()
1913 			its_send_vmapp(its, vm->vpes[i], false);  in its_unmap_vm()
1923 	if (!info->map)  in its_vlpi_map()
1924 		return -EINVAL;  in its_vlpi_map()
1926 	if (!its_dev->event_map.vm) {  in its_vlpi_map()
1929 		maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),  in its_vlpi_map()
1932 			return -ENOMEM;  in its_vlpi_map()
1934 		its_dev->event_map.vm = info->map->vm;  in its_vlpi_map()
1935 		its_dev->event_map.vlpi_maps = maps;  in its_vlpi_map()
1936 	} else if (its_dev->event_map.vm != info->map->vm) {  in its_vlpi_map()
1937 		return -EINVAL;  in its_vlpi_map()
1941 	its_dev->event_map.vlpi_maps[event] = *info->map;  in its_vlpi_map()
1948 		its_map_vm(its_dev->its, info->map->vm);  in its_vlpi_map()
1957 		lpi_write_config(d, 0xff, info->map->properties);  in its_vlpi_map()
1966 		its_dev->event_map.nr_vlpis++;  in its_vlpi_map()
1979 	if (!its_dev->event_map.vm || !map)  in its_vlpi_get()
1980 		return -EINVAL;  in its_vlpi_get()
1983 	*info->map = *map;  in its_vlpi_get()
1993 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))  in its_vlpi_unmap()
1994 		return -EINVAL;  in its_vlpi_unmap()
2001 	its_send_mapti(its_dev, d->hwirq, event);  in its_vlpi_unmap()
2007 	its_unmap_vm(its_dev->its, its_dev->event_map.vm);  in its_vlpi_unmap()
2013 	if (!--its_dev->event_map.nr_vlpis) {  in its_vlpi_unmap()
2014 		its_dev->event_map.vm = NULL;  in its_vlpi_unmap()
2015 		kfree(its_dev->event_map.vlpi_maps);  in its_vlpi_unmap()
2025 	if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))  in its_vlpi_prop_update()
2026 		return -EINVAL;  in its_vlpi_prop_update()
2028 	if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)  in its_vlpi_prop_update()
2029 		lpi_update_config(d, 0xff, info->config);  in its_vlpi_prop_update()
2031 		lpi_write_config(d, 0xff, info->config);  in its_vlpi_prop_update()
2032 	its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));  in its_vlpi_prop_update()
2043 	if (!is_v4(its_dev->its))  in its_irq_set_vcpu_affinity()
2044 		return -EINVAL;  in its_irq_set_vcpu_affinity()
2046 	guard(raw_spinlock)(&its_dev->event_map.vlpi_lock);  in its_irq_set_vcpu_affinity()
2052 	switch (info->cmd_type) {  in its_irq_set_vcpu_affinity()
2064 		return -EINVAL;  in its_irq_set_vcpu_affinity()
2113 		range->base_id = base;  in mk_lpi_range()
2114 		range->span = span;  in mk_lpi_range()
2123 	int err = -ENOSPC;  in alloc_lpi_range()
2128 		if (range->span >= nr_lpis) {  in alloc_lpi_range()
2129 			*base = range->base_id;  in alloc_lpi_range()
2130 			range->base_id += nr_lpis;  in alloc_lpi_range()
2131 			range->span -= nr_lpis;  in alloc_lpi_range()
2133 			if (range->span == 0) {  in alloc_lpi_range()
2134 				list_del(&range->entry);  in alloc_lpi_range()
2151 	if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)  in merge_lpi_ranges()
2153 	if (a->base_id + a->span != b->base_id)  in merge_lpi_ranges()
2155 	b->base_id = a->base_id;  in merge_lpi_ranges()
2156 	b->span += a->span;  in merge_lpi_ranges()
2157 	list_del(&a->entry);  in merge_lpi_ranges()
2167 		return -ENOMEM;  in free_lpi_range()
2172 		if (old->base_id < base)  in free_lpi_range()
2176 	 * old is the last element with ->base_id smaller than base,  in free_lpi_range()
2178 	 * ->base_id smaller than base, &old->entry ends up pointing  in free_lpi_range()
2182 	list_add(&new->entry, &old->entry);  in free_lpi_range()
2196 	u32 lpis = (1UL << id_bits) - 8192;  in its_lpi_init()
2200 	numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);  in its_lpi_init()
2231 		err = -ENOSPC;  in its_lpi_alloc()
2257 	/* Regular IRQ priority, Group-1, disabled */  in gic_reset_prop_table()
2296 	addr_end = addr + size - 1;  in gic_check_reserved_range()
2320 	if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {  in its_setup_lpi_prop_table()
2326 		gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);  in its_setup_lpi_prop_table()
2327 		gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,  in its_setup_lpi_prop_table()
2330 		gic_reset_prop_table(gic_rdists->prop_table_va);  in its_setup_lpi_prop_table()
2335 				    GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),  in its_setup_lpi_prop_table()
2340 			return -ENOMEM;  in its_setup_lpi_prop_table()
2343 		gic_rdists->prop_table_pa = page_to_phys(page);  in its_setup_lpi_prop_table()
2344 		gic_rdists->prop_table_va = page_address(page);  in its_setup_lpi_prop_table()
2345 		WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,  in its_setup_lpi_prop_table()
2350 		&gic_rdists->prop_table_pa);  in its_setup_lpi_prop_table()
2367 	u32 idx = baser - its->tables;  in its_read_baser()
2369 	return gits_read_baser(its->base + GITS_BASER + (idx << 3));  in its_read_baser()
2375 	u32 idx = baser - its->tables;  in its_write_baser()
2377 	gits_write_baser(val, its->base + GITS_BASER + (idx << 3));  in its_write_baser()
2378 	baser->val = its_read_baser(its, baser);  in its_write_baser()
2392 	psz = baser->psz;  in its_setup_baser()
2395 		pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",  in its_setup_baser()
2396 			&its->phys_base, its_base_type_string[type],  in its_setup_baser()
2402 	page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);  in its_setup_baser()
2404 		return -ENOMEM;  in its_setup_baser()
2409 	/* Check if the physical address of the memory is above 48bits */  in its_setup_baser()
2416 			return -ENXIO;  in its_setup_baser()
2426 		((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)	 |  in its_setup_baser()
2427 		((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)	 |  in its_setup_baser()
2450 	tmp = baser->val;  in its_setup_baser()
2458 		 * non-cacheable as well.  in its_setup_baser()
2469 		       &its->phys_base, its_base_type_string[type],  in its_setup_baser()
2472 		return -ENXIO;  in its_setup_baser()
2475 	baser->order = order;  in its_setup_baser()
2476 	baser->base = base;  in its_setup_baser()
2477 	baser->psz = psz;  in its_setup_baser()
2481 		&its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),  in its_setup_baser()
2499 	u32 psz = baser->psz;  in its_parse_indirect_baser()
2505 		 * Find out whether hw supports a single or two-level table by  in its_parse_indirect_baser()
2509 		indirect = !!(baser->val & GITS_BASER_INDIRECT);  in its_parse_indirect_baser()
2515 			 * subtract ID bits that sparse lvl2 table from 'ids'  in its_parse_indirect_baser()
2519 			ids -= ilog2(psz / (int)esz);  in its_parse_indirect_baser()
2528 	 * massive waste of memory if two-level device table  in its_parse_indirect_baser()
2535 		pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",  in its_parse_indirect_baser()
2536 			&its->phys_base, its_base_type_string[type],  in its_parse_indirect_baser()
2565 	svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);  in compute_its_aff()
2567 	val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);  in compute_its_aff()
2576 	if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))  in find_sibling_its()
2587 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))  in find_sibling_its()
2594 		baser = its->tables[2].val;  in find_sibling_its()
2609 		if (its->tables[i].base) {  in its_free_tables()
2610 			its_free_pages(its->tables[i].base, its->tables[i].order);  in its_free_tables()
2611 			its->tables[i].base = NULL;  in its_free_tables()
2644 		if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)  in its_probe_baser_psz()
2656 			return -1;  in its_probe_baser_psz()
2660 	baser->psz = psz;  in its_probe_baser_psz()
2670 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)  in its_alloc_tables()
2674 	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) {  in its_alloc_tables()
2680 		struct its_baser *baser = its->tables + i;  in its_alloc_tables()
2691 			return -ENXIO;  in its_alloc_tables()
2694 		order = get_order(baser->psz);  in its_alloc_tables()
2708 					*baser = sibling->tables[2];  in its_alloc_tables()
2709 					its_write_baser(its, baser, baser->val);  in its_alloc_tables()
2726 		cache = baser->val & GITS_BASER_CACHEABILITY_MASK;  in its_alloc_tables()
2727 		shr = baser->val & GITS_BASER_SHAREABILITY_MASK;  in its_alloc_tables()
2748 		if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))  in inherit_vpe_l1_table_from_its()
2755 		baser = its->tables[2].val;  in inherit_vpe_l1_table_from_its()
2760 		gic_data_rdist()->vpe_l1_base = its->tables[2].base;  in inherit_vpe_l1_table_from_its()
2782 		val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);  in inherit_vpe_l1_table_from_its()
2801 		void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;  in inherit_vpe_l1_table_from_rd()
2819 		gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;  in inherit_vpe_l1_table_from_rd()
2820 		*mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;  in inherit_vpe_l1_table_from_rd()
2831 	void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;  in allocate_vpe_l2_table()
2837 	if (!gic_rdists->has_rvpeid)  in allocate_vpe_l2_table()
2840 	/* Skip non-present CPUs */  in allocate_vpe_l2_table()
2874 	table = gic_data_rdist_cpu(cpu)->vpe_l1_base;  in allocate_vpe_l2_table()
2893 		dsb(sy);  in allocate_vpe_l2_table()
2907 	if (!gic_rdists->has_rvpeid)  in allocate_vpe_l1_table()
2926 	val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);  in allocate_vpe_l1_table()
2930 	gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);  in allocate_vpe_l1_table()
2931 	if (!gic_data_rdist()->vpe_table_mask)  in allocate_vpe_l1_table()
2932 		return -ENOMEM;  in allocate_vpe_l1_table()
2990 	val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);  in allocate_vpe_l1_table()
2999 		return -ENOMEM;  in allocate_vpe_l1_table()
3001 	gic_data_rdist()->vpe_l1_base = page_address(page);  in allocate_vpe_l1_table()
3015 	cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);  in allocate_vpe_l1_table()
3019 		 cpumask_pr_args(gic_data_rdist()->vpe_table_mask));  in allocate_vpe_l1_table()
3028 	its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),  in its_alloc_collections()
3030 	if (!its->collections)  in its_alloc_collections()
3031 		return -ENOMEM;  in its_alloc_collections()
3034 		its->collections[i].target_address = ~0ULL;  in its_alloc_collections()
3047 	/* Make sure the GIC will observe the zero-ed page */  in its_allocate_pending_table()
3081 	 * flag the RD tables as pre-allocated if the stars do align.  in allocate_lpi_tables()
3085 		gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |  in allocate_lpi_tables()
3105 			return -ENOMEM;  in allocate_lpi_tables()
3108 		gic_data_rdist_cpu(cpu)->pend_page = pend_page;  in allocate_lpi_tables()
3124 			count--;  in read_vpend_dirty_clear()
3161 	if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)  in its_cpu_init_lpis()
3165 	if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&  in its_cpu_init_lpis()
3173 		if (WARN_ON(gic_rdists->prop_table_pa != paddr))  in its_cpu_init_lpis()
3180 		gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;  in its_cpu_init_lpis()
3185 	pend_page = gic_data_rdist()->pend_page;  in its_cpu_init_lpis()
3189 	val = (gic_rdists->prop_table_pa |  in its_cpu_init_lpis()
3192 	       ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));  in its_cpu_init_lpis()
3203 			 * The HW reports non-shareable, we must  in its_cpu_init_lpis()
3213 		gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;  in its_cpu_init_lpis()
3229 		 * The HW reports non-shareable, we must remove the  in its_cpu_init_lpis()
3244 	if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {  in its_cpu_init_lpis()
3254 		val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;  in its_cpu_init_lpis()
3273 		gic_rdists->has_rvpeid = false;  in its_cpu_init_lpis()
3274 		gic_rdists->has_vlpis = false;  in its_cpu_init_lpis()
3278 	dsb(sy);  in its_cpu_init_lpis()
3279 	gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;  in its_cpu_init_lpis()
3282 		gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?  in its_cpu_init_lpis()
3293 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {  in its_cpu_init_collection()
3297 		if (its->numa_node != NUMA_NO_NODE &&  in its_cpu_init_collection()
3298 			its->numa_node != of_node_to_nid(cpu_node))  in its_cpu_init_collection()
3306 	if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {  in its_cpu_init_collection()
3311 		target = gic_data_rdist()->phys_base;  in its_cpu_init_collection()
3319 	its->collections[cpu].target_address = target;  in its_cpu_init_collection()
3320 	its->collections[cpu].col_id = cpu;  in its_cpu_init_collection()
3322 	its_send_mapc(its, &its->collections[cpu], 1);  in its_cpu_init_collection()
3323 	its_send_invall(its, &its->collections[cpu]);  in its_cpu_init_collection()
3343 	raw_spin_lock_irqsave(&its->lock, flags);  in its_find_device()
3345 	list_for_each_entry(tmp, &its->its_device_list, entry) {  in its_find_device()
3346 		if (tmp->device_id == dev_id) {  in its_find_device()
3352 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_find_device()
3362 		if (GITS_BASER_TYPE(its->tables[i].val) == type)  in its_get_baser()
3363 			return &its->tables[i];  in its_get_baser()
3377 	esz = GITS_BASER_ENTRY_SIZE(baser->val);  in its_alloc_table_entry()
3378 	if (!(baser->val & GITS_BASER_INDIRECT))  in its_alloc_table_entry()
3379 		return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));  in its_alloc_table_entry()
3382 	idx = id >> ilog2(baser->psz / esz);  in its_alloc_table_entry()
3383 	if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))  in its_alloc_table_entry()
3386 	table = baser->base;  in its_alloc_table_entry()
3390 		page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,  in its_alloc_table_entry()
3391 					    get_order(baser->psz));  in its_alloc_table_entry()
3396 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))  in its_alloc_table_entry()
3397 			gic_flush_dcache_to_poc(page_address(page), baser->psz);  in its_alloc_table_entry()
3402 		if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))  in its_alloc_table_entry()
3406 		dsb(sy);  in its_alloc_table_entry()
3452 	if (!gic_rdists->has_rvpeid)  in its_alloc_vpe_table()
3491 	sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);  in its_create_device()
3494 	itt = itt_alloc_pool(its->numa_node, sz);  in its_create_device()
3519 	dev->its = its;  in its_create_device()
3520 	dev->itt = itt;  in its_create_device()
3521 	dev->itt_sz = sz;  in its_create_device()
3522 	dev->nr_ites = nr_ites;  in its_create_device()
3523 	dev->event_map.lpi_map = lpi_map;  in its_create_device()
3524 	dev->event_map.col_map = col_map;  in its_create_device()
3525 	dev->event_map.lpi_base = lpi_base;  in its_create_device()
3526 	dev->event_map.nr_lpis = nr_lpis;  in its_create_device()
3527 	raw_spin_lock_init(&dev->event_map.vlpi_lock);  in its_create_device()
3528 	dev->device_id = dev_id;  in its_create_device()
3529 	INIT_LIST_HEAD(&dev->entry);  in its_create_device()
3531 	raw_spin_lock_irqsave(&its->lock, flags);  in its_create_device()
3532 	list_add(&dev->entry, &its->its_device_list);  in its_create_device()
3533 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_create_device()
3545 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);  in its_free_device()
3546 	list_del(&its_dev->entry);  in its_free_device()
3547 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);  in its_free_device()
3548 	kfree(its_dev->event_map.col_map);  in its_free_device()
3549 	itt_free_pool(its_dev->itt, its_dev->itt_sz);  in its_free_device()
3558 	idx = bitmap_find_free_region(dev->event_map.lpi_map,  in its_alloc_device_irq()
3559 				      dev->event_map.nr_lpis,  in its_alloc_device_irq()
3562 		return -ENOSPC;  in its_alloc_device_irq()
3564 	*hwirq = dev->event_map.lpi_base + idx;  in its_alloc_device_irq()
3584 	dev_id = info->scratchpad[0].ul;  in its_msi_prepare()
3587 	its = msi_info->data;  in its_msi_prepare()
3589 	if (!gic_rdists->has_direct_lpi &&  in its_msi_prepare()
3591 	    vpe_proxy.dev->its == its &&  in its_msi_prepare()
3592 	    dev_id == vpe_proxy.dev->device_id) {  in its_msi_prepare()
3596 		return -EINVAL;  in its_msi_prepare()
3599 	mutex_lock(&its->dev_alloc_lock);  in its_msi_prepare()
3607 		its_dev->shared = true;  in its_msi_prepare()
3614 		err = -ENOMEM;  in its_msi_prepare()
3618 	if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)  in its_msi_prepare()
3619 		its_dev->shared = true;  in its_msi_prepare()
3621 	pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));  in its_msi_prepare()
3623 	mutex_unlock(&its->dev_alloc_lock);  in its_msi_prepare()
3624 	info->scratchpad[0].ptr = its_dev;  in its_msi_prepare()
3630 	struct its_device *its_dev = info->scratchpad[0].ptr;  in its_msi_teardown()
3632 	guard(mutex)(&its_dev->its->dev_alloc_lock);  in its_msi_teardown()
3635 	if (its_dev->shared)  in its_msi_teardown()
3639 	if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map.lpi_map,  in its_msi_teardown()
3640 				       its_dev->event_map.nr_lpis)))  in its_msi_teardown()
3643 	its_lpi_free(its_dev->event_map.lpi_map,  in its_msi_teardown()
3644 		     its_dev->event_map.lpi_base,  in its_msi_teardown()
3645 		     its_dev->event_map.nr_lpis);  in its_msi_teardown()
3663 	if (irq_domain_get_of_node(domain->parent)) {  in its_irq_gic_domain_alloc()
3664 		fwspec.fwnode = domain->parent->fwnode;  in its_irq_gic_domain_alloc()
3669 	} else if (is_fwnode_irqchip(domain->parent->fwnode)) {  in its_irq_gic_domain_alloc()
3670 		fwspec.fwnode = domain->parent->fwnode;  in its_irq_gic_domain_alloc()
3675 		return -EINVAL;  in its_irq_gic_domain_alloc()
3685 	struct its_device *its_dev = info->scratchpad[0].ptr;  in its_irq_domain_alloc()
3686 	struct its_node *its = its_dev->its;  in its_irq_domain_alloc()
3696 	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));  in its_irq_domain_alloc()
3712 			 (int)(hwirq + i - its_dev->event_map.lpi_base),  in its_irq_domain_alloc()
3728 		return -EINVAL;  in its_irq_domain_activate()
3731 	its_dev->event_map.col_map[event] = cpu;  in its_irq_domain_activate()
3735 	its_send_mapti(its_dev, d->hwirq, event);  in its_irq_domain_activate()
3745 	its_dec_lpi_count(d, its_dev->event_map.col_map[event]);  in its_irq_domain_deactivate()
3757 	bitmap_release_region(its_dev->event_map.lpi_map,  in its_irq_domain_free()
3801 	if (gic_rdists->has_rvpeid)  in its_vpe_db_proxy_unmap_locked()
3805 	if (vpe->vpe_proxy_event == -1)  in its_vpe_db_proxy_unmap_locked()
3808 	its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);  in its_vpe_db_proxy_unmap_locked()
3809 	vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;  in its_vpe_db_proxy_unmap_locked()
3819 		vpe_proxy.next_victim = vpe->vpe_proxy_event;  in its_vpe_db_proxy_unmap_locked()
3821 	vpe->vpe_proxy_event = -1;  in its_vpe_db_proxy_unmap_locked()
3827 	if (gic_rdists->has_rvpeid)  in its_vpe_db_proxy_unmap()
3830 	if (!gic_rdists->has_direct_lpi) {  in its_vpe_db_proxy_unmap()
3842 	if (gic_rdists->has_rvpeid)  in its_vpe_db_proxy_map_locked()
3846 	if (vpe->vpe_proxy_event != -1)  in its_vpe_db_proxy_map_locked()
3855 	vpe->vpe_proxy_event = vpe_proxy.next_victim;  in its_vpe_db_proxy_map_locked()
3856 	vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;  in its_vpe_db_proxy_map_locked()
3858 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;  in its_vpe_db_proxy_map_locked()
3859 	its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);  in its_vpe_db_proxy_map_locked()
3868 	if (gic_rdists->has_rvpeid)  in its_vpe_db_proxy_move()
3871 	if (gic_rdists->has_direct_lpi) {  in its_vpe_db_proxy_move()
3874 		rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;  in its_vpe_db_proxy_move()
3875 		gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);  in its_vpe_db_proxy_move()
3885 	target_col = &vpe_proxy.dev->its->collections[to];  in its_vpe_db_proxy_move()
3886 	its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);  in its_vpe_db_proxy_move()
3887 	vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;  in its_vpe_db_proxy_move()
3898 	val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);  in its_vpe_4_1_invall_locked()
3900 	guard(raw_spinlock)(&gic_data_rdist_cpu(cpu)->rd_lock);  in its_vpe_4_1_invall_locked()
3901 	rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;  in its_vpe_4_1_invall_locked()
3920 	if (!atomic_read(&vpe->vmapp_count)) {  in its_vpe_set_affinity()
3922 			return -EINVAL;  in its_vpe_set_affinity()
3942 	 * protect us, and that we must ensure nobody samples vpe->col_idx  in its_vpe_set_affinity()
3944 	 * taken on any vLPI handling path that evaluates vpe->col_idx.  in its_vpe_set_affinity()
3951 		raw_spin_lock(&vpe->its_vm->vmapp_lock);  in its_vpe_set_affinity()
3954 	table_mask = gic_data_rdist_cpu(from)->vpe_table_mask;  in its_vpe_set_affinity()
3973 	vpe->col_idx = cpu;  in its_vpe_set_affinity()
3978 	if (its && its->flags & ITS_FLAGS_WORKAROUND_HISILICON_162100801)  in its_vpe_set_affinity()
3988 		raw_spin_unlock(&vpe->its_vm->vmapp_lock);  in its_vpe_set_affinity()
3998 	if (!gic_rdists->has_vpend_valid_dirty)  in its_wait_vpt_parse_complete()
4013 	val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &  in its_vpe_schedule()
4015 	val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;  in its_vpe_schedule()
4022 	val  = virt_to_phys(page_address(vpe->vpt_page)) &  in its_vpe_schedule()
4031 	 * easily. So in the end, vpe->pending_last is only an  in its_vpe_schedule()
4038 	val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;  in its_vpe_schedule()
4050 	vpe->idai = !!(val & GICR_VPENDBASER_IDAI);  in its_vpe_deschedule()
4051 	vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);  in its_vpe_deschedule()
4058 	guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock);  in its_vpe_invall()
4064 		if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])  in its_vpe_invall()
4081 	switch (info->cmd_type) {  in its_vpe_set_vcpu_affinity()
4099 		return -EINVAL;  in its_vpe_set_vcpu_affinity()
4111 	cmd(vpe_proxy.dev, vpe->vpe_proxy_event);  in its_vpe_send_cmd()
4120 	if (gic_rdists->has_direct_lpi)  in its_vpe_send_inv()
4121 		__direct_lpi_inv(d, d->parent_data->hwirq);  in its_vpe_send_inv()
4134 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);  in its_vpe_mask_irq()
4141 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);  in its_vpe_unmask_irq()
4152 		return -EINVAL;  in its_vpe_set_irqchip_state()
4154 	if (gic_rdists->has_direct_lpi) {  in its_vpe_set_irqchip_state()
4157 		rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;  in its_vpe_set_irqchip_state()
4159 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);  in its_vpe_set_irqchip_state()
4161 			gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);  in its_vpe_set_irqchip_state()
4180 	.name			= "GICv4-vpe",
4224 	lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);  in its_vpe_4_1_mask_irq()
4230 	lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);  in its_vpe_4_1_unmask_irq()
4242 	val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;  in its_vpe_4_1_schedule()
4243 	val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;  in its_vpe_4_1_schedule()
4244 	val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);  in its_vpe_4_1_schedule()
4255 	if (info->req_db) {  in its_vpe_4_1_deschedule()
4259 		 * vPE is going to block: make the vPE non-resident with  in its_vpe_4_1_deschedule()
4261 		 * we read-back PendingLast clear, then a doorbell will be  in its_vpe_4_1_deschedule()
4268 		raw_spin_lock_irqsave(&vpe->vpe_lock, flags);  in its_vpe_4_1_deschedule()
4272 		vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);  in its_vpe_4_1_deschedule()
4273 		raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);  in its_vpe_4_1_deschedule()
4276 		 * We're not blocking, so just make the vPE non-resident  in its_vpe_4_1_deschedule()
4282 		vpe->pending_last = true;  in its_vpe_4_1_deschedule()
4302 	switch (info->cmd_type) {  in its_vpe_4_1_set_vcpu_affinity()
4320 		return -EINVAL;  in its_vpe_4_1_set_vcpu_affinity()
4325 	.name			= "GICv4.1-vpe",
4339 	desc.its_vsgi_cmd.sgi = d->hwirq;  in its_configure_sgi()
4340 	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;  in its_configure_sgi()
4341 	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;  in its_configure_sgi()
4342 	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;  in its_configure_sgi()
4357 	vpe->sgi_config[d->hwirq].enabled = false;  in its_sgi_mask_irq()
4365 	vpe->sgi_config[d->hwirq].enabled = true;  in its_sgi_unmask_irq()
4387 		return -EINVAL;  in its_sgi_set_irqchip_state()
4394 		val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);  in its_sgi_set_irqchip_state()
4395 		val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);  in its_sgi_set_irqchip_state()
4396 		writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);  in its_sgi_set_irqchip_state()
4415 		return -EINVAL;  in its_sgi_get_irqchip_state()
4420 	 * - Concurrent vPE affinity change: we must make sure it cannot  in its_sgi_get_irqchip_state()
4424 	 * - Concurrent VSGIPENDR access: As it involves accessing two  in its_sgi_get_irqchip_state()
4428 	raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);  in its_sgi_get_irqchip_state()
4429 	base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;  in its_sgi_get_irqchip_state()
4430 	writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);  in its_sgi_get_irqchip_state()
4436 		count--;  in its_sgi_get_irqchip_state()
4446 	raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);  in its_sgi_get_irqchip_state()
4450 		return -ENXIO;  in its_sgi_get_irqchip_state()
4452 	*val = !!(status & (1 << d->hwirq));  in its_sgi_get_irqchip_state()
4462 	switch (info->cmd_type) {  in its_sgi_set_vcpu_affinity()
4464 		vpe->sgi_config[d->hwirq].priority = info->priority;  in its_sgi_set_vcpu_affinity()
4465 		vpe->sgi_config[d->hwirq].group = info->group;  in its_sgi_set_vcpu_affinity()
4470 		return -EINVAL;  in its_sgi_set_vcpu_affinity()
4475 	.name			= "GICv4.1-sgi",
4495 		vpe->sgi_config[i].priority = 0;  in its_sgi_irq_domain_alloc()
4496 		vpe->sgi_config[i].enabled = false;  in its_sgi_irq_domain_alloc()
4497 		vpe->sgi_config[i].group = false;  in its_sgi_irq_domain_alloc()
4530 	 * - To change the configuration, CLEAR must be set to false,  in its_sgi_irq_domain_deactivate()
4532 	 * - To clear the pending bit, CLEAR must be set to true, leaving  in its_sgi_irq_domain_deactivate()
4537 	vpe->sgi_config[d->hwirq].enabled = false;  in its_sgi_irq_domain_deactivate()
4551 	return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL);  in its_vpe_id_alloc()
4573 		return -ENOMEM;  in its_vpe_init()
4579 		return -ENOMEM;  in its_vpe_init()
4582 	raw_spin_lock_init(&vpe->vpe_lock);  in its_vpe_init()
4583 	vpe->vpe_id = vpe_id;  in its_vpe_init()
4584 	vpe->vpt_page = vpt_page;  in its_vpe_init()
4585 	atomic_set(&vpe->vmapp_count, 0);  in its_vpe_init()
4586 	if (!gic_rdists->has_rvpeid)  in its_vpe_init()
4587 		vpe->vpe_proxy_event = -1;  in its_vpe_init()
4595 	its_vpe_id_free(vpe->vpe_id);  in its_vpe_teardown()
4596 	its_free_pending_table(vpe->vpt_page);  in its_vpe_teardown()
4603 	struct its_vm *vm = domain->host_data;  in its_vpe_irq_domain_free()
4613 		BUG_ON(vm != vpe->its_vm);  in its_vpe_irq_domain_free()
4615 		clear_bit(data->hwirq, vm->db_bitmap);  in its_vpe_irq_domain_free()
4620 	if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {  in its_vpe_irq_domain_free()
4621 		its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);  in its_vpe_irq_domain_free()
4622 		its_free_prop_table(vm->vprop_page);  in its_vpe_irq_domain_free()
4637 		return -ENOMEM;  in its_vpe_irq_domain_alloc()
4641 		return -ENOMEM;  in its_vpe_irq_domain_alloc()
4647 		return -ENOMEM;  in its_vpe_irq_domain_alloc()
4650 	vm->db_bitmap = bitmap;  in its_vpe_irq_domain_alloc()
4651 	vm->db_lpi_base = base;  in its_vpe_irq_domain_alloc()
4652 	vm->nr_db_lpis = nr_ids;  in its_vpe_irq_domain_alloc()
4653 	vm->vprop_page = vprop_page;  in its_vpe_irq_domain_alloc()
4654 	raw_spin_lock_init(&vm->vmapp_lock);  in its_vpe_irq_domain_alloc()
4656 	if (gic_rdists->has_rvpeid)  in its_vpe_irq_domain_alloc()
4660 		vm->vpes[i]->vpe_db_lpi = base + i;  in its_vpe_irq_domain_alloc()
4661 		err = its_vpe_init(vm->vpes[i]);  in its_vpe_irq_domain_alloc()
4665 					       vm->vpes[i]->vpe_db_lpi);  in its_vpe_irq_domain_alloc()
4669 					      irqchip, vm->vpes[i]);  in its_vpe_irq_domain_alloc()
4687 	vpe->col_idx = cpumask_first(cpu_online_mask);  in its_vpe_irq_domain_activate()
4688 	irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));  in its_vpe_irq_domain_activate()
4734 	if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))  in its_vpe_irq_domain_deactivate()
4735 		gic_flush_dcache_to_poc(page_address(vpe->vpt_page),  in its_vpe_irq_domain_deactivate()
4770 		count--;  in its_force_quiescent()
4772 			return -EBUSY;  in its_force_quiescent()
4783 	/* erratum 22375: only alloc 8MB table size (20 bits) */  in its_enable_quirk_cavium_22375()
4784 	its->typer &= ~GITS_TYPER_DEVBITS;  in its_enable_quirk_cavium_22375()
4785 	its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);  in its_enable_quirk_cavium_22375()
4786 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;  in its_enable_quirk_cavium_22375()
4795 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;  in its_enable_quirk_cavium_23144()
4805 	its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;  in its_enable_quirk_qdf2400_e0065()
4806 	its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);  in its_enable_quirk_qdf2400_e0065()
4813 	struct its_node *its = its_dev->its;  in its_irq_get_msi_base_pre_its()
4816 	 * The Socionext Synquacer SoC has a so-called 'pre-ITS',  in its_irq_get_msi_base_pre_its()
4817 	 * which maps 32-bit writes targeted at a separate window of  in its_irq_get_msi_base_pre_its()
4819 	 * with device ID taken from bits [device_id_bits + 1:2] of  in its_irq_get_msi_base_pre_its()
4822 	return its->pre_its_base + (its_dev->device_id << 2);  in its_irq_get_msi_base_pre_its()
4831 	if (!fwnode_property_read_u32_array(its->fwnode_handle,  in its_enable_quirk_socionext_synquacer()
4832 					   "socionext,synquacer-pre-its",  in its_enable_quirk_socionext_synquacer()
4836 		its->pre_its_base = pre_its_window[0];  in its_enable_quirk_socionext_synquacer()
4837 		its->get_msi_base = its_irq_get_msi_base_pre_its;  in its_enable_quirk_socionext_synquacer()
4839 		ids = ilog2(pre_its_window[1]) - 2;  in its_enable_quirk_socionext_synquacer()
4841 			its->typer &= ~GITS_TYPER_DEVBITS;  in its_enable_quirk_socionext_synquacer()
4842 			its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);  in its_enable_quirk_socionext_synquacer()
4845 		/* the pre-ITS breaks isolation, so disable MSI remapping */  in its_enable_quirk_socionext_synquacer()
4846 		its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;  in its_enable_quirk_socionext_synquacer()
4860 	its->vlpi_redist_offset = SZ_128K;  in its_enable_quirk_hip07_161600802()
4872 	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;  in its_enable_rk3588001()
4873 	gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;  in its_enable_rk3588001()
4882 	its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;  in its_set_non_coherent()
4890 	its->flags |= ITS_FLAGS_WORKAROUND_HISILICON_162100801;  in its_enable_quirk_hip09_162100801()
4933 		 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4934 		 * implementation, but with a 'pre-ITS' added that requires
4937 		.desc	= "ITS: Socionext Synquacer pre-ITS",
4968 		.desc   = "ITS: non-coherent attribute",
4969 		.property = "dma-noncoherent",
4986 	u32 iidr = readl_relaxed(its->base + GITS_IIDR);  in its_enable_quirks()
4990 	if (is_of_node(its->fwnode_handle))  in its_enable_quirks()
4991 		gic_enable_of_quirks(to_of_node(its->fwnode_handle),  in its_enable_quirks()
5004 		base = its->base;  in its_save_disable()
5005 		its->ctlr_save = readl_relaxed(base + GITS_CTLR);  in its_save_disable()
5009 			       &its->phys_base, err);  in its_save_disable()
5010 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);  in its_save_disable()
5014 		its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);  in its_save_disable()
5022 			base = its->base;  in its_save_disable()
5023 			writel_relaxed(its->ctlr_save, base + GITS_CTLR);  in its_save_disable()
5041 		base = its->base;  in its_restore_enable()
5055 			       &its->phys_base, ret);  in its_restore_enable()
5059 		gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);  in its_restore_enable()
5065 		its->cmd_write = its->cmd_base;  in its_restore_enable()
5070 			struct its_baser *baser = &its->tables[i];  in its_restore_enable()
5072 			if (!(baser->val & GITS_BASER_VALID))  in its_restore_enable()
5075 			its_write_baser(its, baser, baser->val);  in its_restore_enable()
5077 		writel_relaxed(its->ctlr_save, base + GITS_CTLR);  in its_restore_enable()
5084 		if (its->collections[smp_processor_id()].col_id <  in its_restore_enable()
5101 	its_base = ioremap(res->start, SZ_64K);  in its_map_one()
5103 		pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);  in its_map_one()
5104 		*err = -ENOMEM;  in its_map_one()
5110 		pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);  in its_map_one()
5111 		*err = -ENODEV;  in its_map_one()
5117 		pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);  in its_map_one()
5131 		.fwnode		= its->fwnode_handle,  in its_init_domain()
5133 		.domain_flags	= its->msi_domain_flags,  in its_init_domain()
5140 		return -ENOMEM;  in its_init_domain()
5142 	info->ops = &its_msi_domain_ops;  in its_init_domain()
5143 	info->data = its;  in its_init_domain()
5148 		return -ENOMEM;  in its_init_domain()
5159 	if (gic_rdists->has_direct_lpi) {  in its_init_vpe_domain()
5171 		return -ENOMEM;  in its_init_vpe_domain()
5174 	devid = GENMASK(device_ids(its) - 1, 0);  in its_init_vpe_domain()
5179 		return -ENOMEM;  in its_init_vpe_domain()
5182 	BUG_ON(entries > vpe_proxy.dev->nr_ites);  in its_init_vpe_domain()
5187 		devid, vpe_proxy.dev->nr_ites);  in its_init_vpe_domain()
5199 	 * guaranteed to be single-threaded, hence no  in its_compute_its_list_map()
5206 		       &its->phys_base);  in its_compute_its_list_map()
5207 		return -EINVAL;  in its_compute_its_list_map()
5210 	ctlr = readl_relaxed(its->base + GITS_CTLR);  in its_compute_its_list_map()
5213 	writel_relaxed(ctlr, its->base + GITS_CTLR);  in its_compute_its_list_map()
5214 	ctlr = readl_relaxed(its->base + GITS_CTLR);  in its_compute_its_list_map()
5222 		       &its->phys_base, its_number);  in its_compute_its_list_map()
5223 		return -EINVAL;  in its_compute_its_list_map()
5239 		if (!(its->typer & GITS_TYPER_VMOVP)) {  in its_probe_one()
5244 			its->list_nr = err;  in its_probe_one()
5247 				&its->phys_base, err);  in its_probe_one()
5249 			pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);  in its_probe_one()
5253 			u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);  in its_probe_one()
5255 			its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);  in its_probe_one()
5256 			if (!its->sgir_base) {  in its_probe_one()
5257 				err = -ENOMEM;  in its_probe_one()
5261 			its->mpidr = readl_relaxed(its->base + GITS_MPIDR);  in its_probe_one()
5264 				&its->phys_base, its->mpidr, svpet);  in its_probe_one()
5268 	page = its_alloc_pages_node(its->numa_node,  in its_probe_one()
5272 		err = -ENOMEM;  in its_probe_one()
5275 	its->cmd_base = (void *)page_address(page);  in its_probe_one()
5276 	its->cmd_write = its->cmd_base;  in its_probe_one()
5286 	baser = (virt_to_phys(its->cmd_base)	|  in its_probe_one()
5289 		 (ITS_CMD_QUEUE_SZ / SZ_4K - 1)	|  in its_probe_one()
5292 	gits_write_cbaser(baser, its->base + GITS_CBASER);  in its_probe_one()
5293 	tmp = gits_read_cbaser(its->base + GITS_CBASER);  in its_probe_one()
5295 	if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)  in its_probe_one()
5301 			 * The HW reports non-shareable, we must  in its_probe_one()
5308 			gits_write_cbaser(baser, its->base + GITS_CBASER);  in its_probe_one()
5311 		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;  in its_probe_one()
5314 	gits_write_cwriter(0, its->base + GITS_CWRITER);  in its_probe_one()
5315 	ctlr = readl_relaxed(its->base + GITS_CTLR);  in its_probe_one()
5319 	writel_relaxed(ctlr, its->base + GITS_CTLR);  in its_probe_one()
5326 	list_add(&its->entry, &its_nodes);  in its_probe_one()
5334 	its_free_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));  in its_probe_one()
5336 	if (its->sgir_base)  in its_probe_one()
5337 		iounmap(its->sgir_base);  in its_probe_one()
5339 	pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);  in its_probe_one()
5356 		return -ENXIO;  in redist_disable_lpis()
5365 	 * LPIs before trying to re-enable them. They are already  in redist_disable_lpis()
5370 	if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||  in redist_disable_lpis()
5371 	    (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))  in redist_disable_lpis()
5386 	dsb(sy);  in redist_disable_lpis()
5397 			return -ETIMEDOUT;  in redist_disable_lpis()
5400 		timeout--;  in redist_disable_lpis()
5410 		return -EBUSY;  in redist_disable_lpis()
5434 	cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);  in rdist_memreserve_cpuhp_cleanup_workfn()
5435 	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;  in rdist_memreserve_cpuhp_cleanup_workfn()
5447 	if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)  in its_cpu_memreserve_lpi()
5450 	pend_page = gic_data_rdist()->pend_page;  in its_cpu_memreserve_lpi()
5452 		ret = -ENOMEM;  in its_cpu_memreserve_lpi()
5456 	 * If the pending table was pre-programmed, free the memory we  in its_cpu_memreserve_lpi()
5460 	if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {  in its_cpu_memreserve_lpi()
5462 		gic_data_rdist()->pend_page = NULL;  in its_cpu_memreserve_lpi()
5474 	gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;  in its_cpu_memreserve_lpi()
5496 	{	.compatible	= "arm,gic-v3-its",	},
5517 	raw_spin_lock_init(&its->lock);  in its_node_init()
5518 	mutex_init(&its->dev_alloc_lock);  in its_node_init()
5519 	INIT_LIST_HEAD(&its->entry);  in its_node_init()
5520 	INIT_LIST_HEAD(&its->its_device_list);  in its_node_init()
5522 	its->typer = gic_read_typer(its_base + GITS_TYPER);  in its_node_init()
5523 	its->base = its_base;  in its_node_init()
5524 	its->phys_base = res->start;  in its_node_init()
5525 	its->get_msi_base = its_irq_get_msi_base;  in its_node_init()
5526 	its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI | IRQ_DOMAIN_FLAG_MSI_IMMUTABLE;  in its_node_init()
5528 	its->numa_node = numa_node;  in its_node_init()
5529 	its->fwnode_handle = handle;  in its_node_init()
5540 	iounmap(its->base);  in its_node_destroy()
5559 		    !of_property_read_bool(np, "msi-controller") ||  in its_of_probe()
5574 		if (!of_property_read_bool(np, "msi-controller")) {  in its_of_probe()
5575 			pr_warn("%pOF: no msi-controller property, ITS ignored\n",  in its_of_probe()
5586 		its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));  in its_of_probe()
5588 			return -ENOMEM;  in its_of_probe()
5639 		return -EINVAL;  in gic_acpi_parse_srat_its()
5641 	if (its_affinity->header.length < sizeof(*its_affinity)) {  in gic_acpi_parse_srat_its()
5643 			its_affinity->header.length);  in gic_acpi_parse_srat_its()
5644 		return -EINVAL;  in gic_acpi_parse_srat_its()
5652 	node = pxm_to_node(its_affinity->proximity_domain);  in gic_acpi_parse_srat_its()
5660 	its_srat_maps[its_in_srat].its_id = its_affinity->its_id;  in gic_acpi_parse_srat_its()
5662 	pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",  in gic_acpi_parse_srat_its()
5663 		its_affinity->proximity_domain, its_affinity->its_id, node);  in gic_acpi_parse_srat_its()
5712 	res.start = its_entry->base_address;  in gic_acpi_parse_madt_its()
5713 	res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;  in gic_acpi_parse_madt_its()
5720 		return -ENOMEM;  in gic_acpi_parse_madt_its()
5723 	err = iort_register_domain_token(its_entry->translation_id, res.start,  in gic_acpi_parse_madt_its()
5727 		       &res.start, its_entry->translation_id);  in gic_acpi_parse_madt_its()
5732 			    acpi_get_its_numa_node(its_entry->translation_id));  in gic_acpi_parse_madt_its()
5734 		err = -ENOMEM;  in gic_acpi_parse_madt_its()
5739 	    (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT))  in gic_acpi_parse_madt_its()
5740 		its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;  in gic_acpi_parse_madt_its()
5747 	iort_deregister_domain_token(its_entry->translation_id);  in gic_acpi_parse_madt_its()
5761 		.start	= its_entry->base_address,  in its_acpi_reset()
5762 		.end	= its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,  in its_acpi_reset()
5798 	gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;  in its_lpi_memreserve_init()
5806 	gic_rdists->cpuhp_memreserve_state = state;  in its_lpi_memreserve_init()
5820 	itt_pool = gen_pool_create(get_order(ITS_ITT_ALIGN), -1);  in its_init()
5822 		return -ENOMEM;  in its_init()
5836 		return -ENXIO;  in its_init()
5849 	if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))  in its_init()
5850 		rdists->has_rvpeid = false;  in its_init()
5852 	if (has_v4 & rdists->has_vlpis) {  in its_init()
5862 			rdists->has_vlpis = false;  in its_init()