6 #define pr_fmt(fmt)	"GICv5 ITS: " fmt
22 #include "irq-gic-its-msi-parent.h"
62 static void gicv5_its_dcache_clean(struct gicv5_its_chip_data *its, void *start,  in gicv5_its_dcache_clean()  argument
67 	if (its->flags & ITS_FLAGS_NON_COHERENT)  in gicv5_its_dcache_clean()
73 static void its_write_table_entry(struct gicv5_its_chip_data *its, __le64 *entry,  in its_write_table_entry()  argument
77 	gicv5_its_dcache_clean(its, entry, sizeof(*entry));  in its_write_table_entry()
80 #define devtab_cfgr_field(its, f)	\  argument
81 	FIELD_GET(GICV5_ITS_DT_CFGR_##f, (its)->devtab_cfgr.cfgr)
83 static int gicv5_its_cache_sync(struct gicv5_its_chip_data *its)  in gicv5_its_cache_sync()  argument
85 	return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_STATUSR,  in gicv5_its_cache_sync()
89 static void gicv5_its_syncr(struct gicv5_its_chip_data *its,  in gicv5_its_syncr()  argument
97 	its_writeq_relaxed(its, syncr, GICV5_ITS_SYNCR);  in gicv5_its_syncr()
99 	gicv5_wait_for_op(its->its_base, GICV5_ITS_SYNC_STATUSR, GICV5_ITS_SYNC_STATUSR_IDLE);  in gicv5_its_syncr()
120 static int gicv5_its_itt_cache_inv(struct gicv5_its_chip_data *its, u32 device_id,  in gicv5_its_itt_cache_inv()  argument
130 	its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);  in gicv5_its_itt_cache_inv()
131 	its_writel_relaxed(its, eidr, GICV5_ITS_EIDR);  in gicv5_its_itt_cache_inv()
132 	its_writel_relaxed(its, eventr, GICV5_ITS_INV_EVENTR);  in gicv5_its_itt_cache_inv()
134 	return gicv5_its_cache_sync(its);  in gicv5_its_itt_cache_inv()
161 static int gicv5_its_create_itt_linear(struct gicv5_its_chip_data *its,  in gicv5_its_create_itt_linear()  argument
177 	gicv5_its_dcache_clean(its, itt, num_ents * sizeof(*itt));  in gicv5_its_create_itt_linear()
187 static int gicv5_its_create_itt_two_level(struct gicv5_its_chip_data *its,  in gicv5_its_create_itt_two_level()  argument
253 		gicv5_its_dcache_clean(its, itt_l2, l2sz);  in gicv5_its_create_itt_two_level()
262 	gicv5_its_dcache_clean(its, itt_l1, num_ents * sizeof(*itt_l1));  in gicv5_its_create_itt_two_level()
361 static int gicv5_its_device_cache_inv(struct gicv5_its_chip_data *its,  in gicv5_its_device_cache_inv()  argument
372 	its_writeq_relaxed(its, didr, GICV5_ITS_DIDR);  in gicv5_its_device_cache_inv()
373 	its_writel_relaxed(its, devicer, GICV5_ITS_INV_DEVICER);  in gicv5_its_device_cache_inv()
375 	return gicv5_its_cache_sync(its);  in gicv5_its_device_cache_inv()
382 static int gicv5_its_alloc_l2_devtab(struct gicv5_its_chip_data *its,  in gicv5_its_alloc_l2_devtab()  argument
385 	__le64 *l2devtab, *l1devtab = its->devtab_cfgr.l2.l1devtab;  in gicv5_its_alloc_l2_devtab()
393 	l2sz = devtab_cfgr_field(its, L2SZ);  in gicv5_its_alloc_l2_devtab()
408 	its->devtab_cfgr.l2.l2ptrs[l1_index] = l2devtab;  in gicv5_its_alloc_l2_devtab()
413 	its_write_table_entry(its, &l1devtab[l1_index], l1dte);  in gicv5_its_alloc_l2_devtab()
418 static __le64 *gicv5_its_devtab_get_dte_ref(struct gicv5_its_chip_data *its,  in gicv5_its_devtab_get_dte_ref()  argument
421 	u8 str = devtab_cfgr_field(its, STRUCTURE);  in gicv5_its_devtab_get_dte_ref()
427 		l2devtab = its->devtab_cfgr.linear.devtab;  in gicv5_its_devtab_get_dte_ref()
431 	l2sz = devtab_cfgr_field(its, L2SZ);  in gicv5_its_devtab_get_dte_ref()
443 		ret = gicv5_its_alloc_l2_devtab(its, l1_idx);  in gicv5_its_devtab_get_dte_ref()
448 	l2devtab = its->devtab_cfgr.l2.l2ptrs[l1_idx];  in gicv5_its_devtab_get_dte_ref()
457 static int gicv5_its_device_register(struct gicv5_its_chip_data *its,  in gicv5_its_device_register()  argument
468 	device_id_bits = devtab_cfgr_field(its, DEVICEID_BITS);  in gicv5_its_device_register()
476 	dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, true);  in gicv5_its_device_register()
489 	idr2 = its_readl_relaxed(its, GICV5_ITS_IDR2);  in gicv5_its_device_register()
498 	idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);  in gicv5_its_device_register()
509 		ret = gicv5_its_create_itt_two_level(its, its_dev, event_id_bits,  in gicv5_its_device_register()
513 		ret = gicv5_its_create_itt_linear(its, its_dev, event_id_bits);  in gicv5_its_device_register()
529 	its_write_table_entry(its, dte, val);  in gicv5_its_device_register()
531 	ret = gicv5_its_device_cache_inv(its, its_dev);  in gicv5_its_device_register()
533 		its_write_table_entry(its, dte, 0);  in gicv5_its_device_register()
545 static int gicv5_its_device_unregister(struct gicv5_its_chip_data *its,  in gicv5_its_device_unregister()  argument
550 	dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, false);  in gicv5_its_device_unregister()
559 	its_write_table_entry(its, dte, 0);  in gicv5_its_device_unregister()
563 	return gicv5_its_device_cache_inv(its, its_dev);  in gicv5_its_device_unregister()
570 static int gicv5_its_alloc_devtab_linear(struct gicv5_its_chip_data *its,  in gicv5_its_alloc_devtab_linear()  argument
598 	gicv5_its_dcache_clean(its, devtab, sz);  in gicv5_its_alloc_devtab_linear()
604 	its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);  in gicv5_its_alloc_devtab_linear()
607 	its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);  in gicv5_its_alloc_devtab_linear()
609 	its->devtab_cfgr.cfgr = cfgr;  in gicv5_its_alloc_devtab_linear()
610 	its->devtab_cfgr.linear.devtab = devtab;  in gicv5_its_alloc_devtab_linear()
619 static int gicv5_its_alloc_devtab_two_level(struct gicv5_its_chip_data *its,  in gicv5_its_alloc_devtab_two_level()  argument
663 	gicv5_its_dcache_clean(its, l1devtab, l1_sz);  in gicv5_its_alloc_devtab_two_level()
669 	its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR);  in gicv5_its_alloc_devtab_two_level()
672 	its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER);  in gicv5_its_alloc_devtab_two_level()
674 	its->devtab_cfgr.cfgr = cfgr;  in gicv5_its_alloc_devtab_two_level()
675 	its->devtab_cfgr.l2.l1devtab = l1devtab;  in gicv5_its_alloc_devtab_two_level()
676 	its->devtab_cfgr.l2.l2ptrs = l2ptrs;  in gicv5_its_alloc_devtab_two_level()
685 static int gicv5_its_init_devtab(struct gicv5_its_chip_data *its)  in gicv5_its_init_devtab()  argument
691 	idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1);  in gicv5_its_init_devtab()
697 		return gicv5_its_alloc_devtab_two_level(its, device_id_bits,  in gicv5_its_init_devtab()
700 		return gicv5_its_alloc_devtab_linear(its, device_id_bits);  in gicv5_its_init_devtab()
703 static void gicv5_its_deinit_devtab(struct gicv5_its_chip_data *its)  in gicv5_its_deinit_devtab()  argument
705 	u8 str = devtab_cfgr_field(its, STRUCTURE);  in gicv5_its_deinit_devtab()
708 		kfree(its->devtab_cfgr.linear.devtab);  in gicv5_its_deinit_devtab()
710 		kfree(its->devtab_cfgr.l2.l1devtab);  in gicv5_its_deinit_devtab()
711 		kfree(its->devtab_cfgr.l2.l2ptrs);  in gicv5_its_deinit_devtab()
725 	.name			= "GICv5-ITS-MSI",
735 static struct gicv5_its_dev *gicv5_its_find_device(struct gicv5_its_chip_data *its,  in gicv5_its_find_device()  argument
738 	struct gicv5_its_dev *dev = xa_load(&its->its_devices, device_id);  in gicv5_its_find_device()
743 static struct gicv5_its_dev *gicv5_its_alloc_device(struct gicv5_its_chip_data *its, int nvec,  in gicv5_its_alloc_device()  argument
750 	its_dev = gicv5_its_find_device(its, dev_id);  in gicv5_its_alloc_device()
765 	ret = gicv5_its_device_register(its, its_dev);  in gicv5_its_alloc_device()
771 	gicv5_its_device_cache_inv(its, its_dev);  in gicv5_its_alloc_device()
773 	its_dev->its_node = its;  in gicv5_its_alloc_device()
781 	entry = xa_store(&its->its_devices, dev_id, its_dev, GFP_KERNEL);  in gicv5_its_alloc_device()
792 	gicv5_its_device_unregister(its, its_dev);  in gicv5_its_alloc_device()
803 	struct gicv5_its_chip_data *its;  in gicv5_its_msi_prepare()  local
807 	its = msi_info->data;  in gicv5_its_msi_prepare()
809 	guard(mutex)(&its->dev_alloc_lock);  in gicv5_its_msi_prepare()
811 	its_dev = gicv5_its_alloc_device(its, nvec, dev_id);  in gicv5_its_msi_prepare()
825 	struct gicv5_its_chip_data *its;  in gicv5_its_msi_teardown()  local
828 	its = msi_info->data;  in gicv5_its_msi_teardown()
830 	guard(mutex)(&its->dev_alloc_lock);  in gicv5_its_msi_teardown()
835 	xa_erase(&its->its_devices, its_dev->device_id);  in gicv5_its_msi_teardown()
837 	gicv5_its_device_unregister(its, its_dev);  in gicv5_its_msi_teardown()
848 	struct gicv5_its_chip_data *its = its_dev->its_node;  in gicv5_its_map_event()  local
860 	its_write_table_entry(its, itte, itt_entry);  in gicv5_its_map_event()
862 	gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);  in gicv5_its_map_event()
869 	struct gicv5_its_chip_data *its = its_dev->its_node;  in gicv5_its_unmap_event()  local
878 	its_write_table_entry(its, itte, itte_val);  in gicv5_its_unmap_event()
880 	gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id);  in gicv5_its_unmap_event()
991 	struct gicv5_its_chip_data *its;  in gicv5_its_irq_domain_free()  local
997 	its = its_dev->its_node;  in gicv5_its_irq_domain_free()
1013 	gicv5_its_syncr(its, its_dev);  in gicv5_its_irq_domain_free()
1049 static int gicv5_its_write_cr0(struct gicv5_its_chip_data *its, bool enable)  in gicv5_its_write_cr0()  argument
1053 	its_writel_relaxed(its, cr0, GICV5_ITS_CR0);  in gicv5_its_write_cr0()
1054 	return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_CR0,  in gicv5_its_write_cr0()
1058 static int gicv5_its_enable(struct gicv5_its_chip_data *its)  in gicv5_its_enable()  argument
1060 	return gicv5_its_write_cr0(its, true);  in gicv5_its_enable()
1063 static int gicv5_its_disable(struct gicv5_its_chip_data *its)  in gicv5_its_disable()  argument
1065 	return gicv5_its_write_cr0(its, false);  in gicv5_its_disable()
1079 	pr_info("ITS %s enabled using %s device table device_id_bits %u\n",  in gicv5_its_print_info()
1085 static int gicv5_its_init_domain(struct gicv5_its_chip_data *its, struct irq_domain *parent)  in gicv5_its_init_domain()  argument
1088 		.fwnode		= its->fwnode,  in gicv5_its_init_domain()
1090 		.domain_flags	= its->msi_domain_flags,  in gicv5_its_init_domain()
1100 	info->data = its;  in gicv5_its_init_domain()
1133 	if (WARN(enabled, "ITS %s enabled, disabling it before proceeding\n", np->full_name)) {  in gicv5_its_init_bases()
1141 		 * A non-coherent ITS implies that some cache levels cannot be  in gicv5_its_init_bases()
1221 		if (!of_device_is_compatible(np, "arm,gic-v5-its"))  in gicv5_its_of_probe()
1225 			pr_err("Failed to init ITS %s\n", np->full_name);  in gicv5_its_of_probe()