Lines Matching +full:synquacer +full:- +full:spi
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
34 #include <linux/irqchip/arm-gic-v3.h>
35 #include <linux/irqchip/arm-gic-v4.h>
40 #include "irq-gic-common.h"
41 #include "irq-msi-lib.h"
57 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
66 * Collection structure - just an ID, and a redistributor address to
76 * The ITS_BASER structure - contains memory information, cached
89 * The ITS structure - contains most of the infrastructure, with the
90 * top-level MSI domain, the command queue, the collections, and the
119 u32 pre_its_base; /* for Socionext Synquacer */
123 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS))
124 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP))
125 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
133 if (gic_rdists->has_rvpeid && \
134 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
135 nvpeid = 1 + (gic_rdists->gicd_typer2 & \
157 * The ITS view of a device - belongs to an ITS, owns an interrupt
197 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
198 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
199 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
208 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); in require_its_list_vmovp()
213 return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE); in rdists_support_shareable()
226 __set_bit(its->list_nr, &its_list); in get_its_list()
235 return d->hwirq - its_dev->event_map.lpi_base; in its_get_event_id()
241 struct its_node *its = its_dev->its; in dev_event_to_col()
243 return its->collections + its_dev->event_map.col_map[event]; in dev_event_to_col()
249 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis)) in dev_event_to_vlpi_map()
252 return &its_dev->event_map.vlpi_maps[event]; in dev_event_to_vlpi_map()
269 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags); in vpe_to_cpuid_lock()
270 return vpe->col_idx; in vpe_to_cpuid_lock()
275 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); in vpe_to_cpuid_unlock()
285 if (d->chip == &its_vpe_irq_chip) { in irq_to_cpuid_lock()
290 vpe = map->vpe; in irq_to_cpuid_lock()
298 cpu = its_dev->event_map.col_map[its_get_event_id(d)]; in irq_to_cpuid_lock()
310 if (d->chip == &its_vpe_irq_chip) { in irq_to_cpuid_unlock()
315 vpe = map->vpe; in irq_to_cpuid_unlock()
324 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0))) in valid_col()
332 if (valid_col(its->collections + vpe->col_idx)) in valid_vpe()
339 * ITS command descriptors - parameters to be encoded in a command
467 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); in its_encode_cmd()
472 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); in its_encode_devid()
477 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); in its_encode_event_id()
482 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); in its_encode_phys_id()
487 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); in its_encode_size()
492 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); in its_encode_itt()
497 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); in its_encode_valid()
502 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); in its_encode_target()
507 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); in its_encode_collection()
512 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); in its_encode_vpeid()
517 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); in its_encode_virt_id()
522 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); in its_encode_db_phys_id()
527 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); in its_encode_db_valid()
532 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); in its_encode_seq_num()
537 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); in its_encode_its_list()
542 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); in its_encode_vpt_addr()
547 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); in its_encode_vpt_size()
552 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); in its_encode_vconf_addr()
557 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); in its_encode_alloc()
562 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); in its_encode_ptz()
568 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); in its_encode_vmapp_default_db()
574 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0); in its_encode_vmovp_default_db()
579 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); in its_encode_db()
584 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); in its_encode_sgi_intid()
589 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); in its_encode_sgi_priority()
594 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10); in its_encode_sgi_group()
599 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9); in its_encode_sgi_clear()
604 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8); in its_encode_sgi_enable()
610 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); in its_fixup_cmd()
611 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); in its_fixup_cmd()
612 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); in its_fixup_cmd()
613 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); in its_fixup_cmd()
621 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); in its_build_mapd_cmd()
623 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); in its_build_mapd_cmd()
627 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); in its_build_mapd_cmd()
628 its_encode_size(cmd, size - 1); in its_build_mapd_cmd()
630 its_encode_valid(cmd, desc->its_mapd_cmd.valid); in its_build_mapd_cmd()
642 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); in its_build_mapc_cmd()
643 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); in its_build_mapc_cmd()
644 its_encode_valid(cmd, desc->its_mapc_cmd.valid); in its_build_mapc_cmd()
648 return desc->its_mapc_cmd.col; in its_build_mapc_cmd()
657 col = dev_event_to_col(desc->its_mapti_cmd.dev, in its_build_mapti_cmd()
658 desc->its_mapti_cmd.event_id); in its_build_mapti_cmd()
661 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); in its_build_mapti_cmd()
662 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); in its_build_mapti_cmd()
663 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); in its_build_mapti_cmd()
664 its_encode_collection(cmd, col->col_id); in its_build_mapti_cmd()
677 col = dev_event_to_col(desc->its_movi_cmd.dev, in its_build_movi_cmd()
678 desc->its_movi_cmd.event_id); in its_build_movi_cmd()
681 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); in its_build_movi_cmd()
682 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); in its_build_movi_cmd()
683 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); in its_build_movi_cmd()
696 col = dev_event_to_col(desc->its_discard_cmd.dev, in its_build_discard_cmd()
697 desc->its_discard_cmd.event_id); in its_build_discard_cmd()
700 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); in its_build_discard_cmd()
701 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); in its_build_discard_cmd()
714 col = dev_event_to_col(desc->its_inv_cmd.dev, in its_build_inv_cmd()
715 desc->its_inv_cmd.event_id); in its_build_inv_cmd()
718 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); in its_build_inv_cmd()
719 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); in its_build_inv_cmd()
732 col = dev_event_to_col(desc->its_int_cmd.dev, in its_build_int_cmd()
733 desc->its_int_cmd.event_id); in its_build_int_cmd()
736 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); in its_build_int_cmd()
737 its_encode_event_id(cmd, desc->its_int_cmd.event_id); in its_build_int_cmd()
750 col = dev_event_to_col(desc->its_clear_cmd.dev, in its_build_clear_cmd()
751 desc->its_clear_cmd.event_id); in its_build_clear_cmd()
754 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); in its_build_clear_cmd()
755 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); in its_build_clear_cmd()
767 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); in its_build_invall_cmd()
771 return desc->its_invall_cmd.col; in its_build_invall_cmd()
779 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); in its_build_vinvall_cmd()
783 return valid_vpe(its, desc->its_vinvall_cmd.vpe); in its_build_vinvall_cmd()
790 struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe); in its_build_vmapp_cmd()
796 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); in its_build_vmapp_cmd()
797 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); in its_build_vmapp_cmd()
799 if (!desc->its_vmapp_cmd.valid) { in its_build_vmapp_cmd()
801 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); in its_build_vmapp_cmd()
804 * Unmapping a VPE is self-synchronizing on GICv4.1, in its_build_vmapp_cmd()
813 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); in its_build_vmapp_cmd()
814 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmapp_cmd()
818 its_encode_vpt_size(cmd, LPI_NRBITS - 1); in its_build_vmapp_cmd()
823 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); in its_build_vmapp_cmd()
825 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); in its_build_vmapp_cmd()
837 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); in its_build_vmapp_cmd()
851 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) in its_build_vmapti_cmd()
852 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; in its_build_vmapti_cmd()
857 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); in its_build_vmapti_cmd()
858 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); in its_build_vmapti_cmd()
859 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); in its_build_vmapti_cmd()
861 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); in its_build_vmapti_cmd()
865 return valid_vpe(its, desc->its_vmapti_cmd.vpe); in its_build_vmapti_cmd()
874 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) in its_build_vmovi_cmd()
875 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; in its_build_vmovi_cmd()
880 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); in its_build_vmovi_cmd()
881 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); in its_build_vmovi_cmd()
882 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); in its_build_vmovi_cmd()
888 return valid_vpe(its, desc->its_vmovi_cmd.vpe); in its_build_vmovi_cmd()
897 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmovp_cmd()
899 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); in its_build_vmovp_cmd()
900 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); in its_build_vmovp_cmd()
901 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); in its_build_vmovp_cmd()
906 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi); in its_build_vmovp_cmd()
911 return valid_vpe(its, desc->its_vmovp_cmd.vpe); in its_build_vmovp_cmd()
920 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev, in its_build_vinv_cmd()
921 desc->its_inv_cmd.event_id); in its_build_vinv_cmd()
924 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); in its_build_vinv_cmd()
925 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); in its_build_vinv_cmd()
929 return valid_vpe(its, map->vpe); in its_build_vinv_cmd()
938 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev, in its_build_vint_cmd()
939 desc->its_int_cmd.event_id); in its_build_vint_cmd()
942 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); in its_build_vint_cmd()
943 its_encode_event_id(cmd, desc->its_int_cmd.event_id); in its_build_vint_cmd()
947 return valid_vpe(its, map->vpe); in its_build_vint_cmd()
956 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev, in its_build_vclear_cmd()
957 desc->its_clear_cmd.event_id); in its_build_vclear_cmd()
960 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); in its_build_vclear_cmd()
961 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); in its_build_vclear_cmd()
965 return valid_vpe(its, map->vpe); in its_build_vclear_cmd()
976 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); in its_build_invdb_cmd()
980 return valid_vpe(its, desc->its_invdb_cmd.vpe); in its_build_invdb_cmd()
991 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); in its_build_vsgi_cmd()
992 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); in its_build_vsgi_cmd()
993 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); in its_build_vsgi_cmd()
994 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); in its_build_vsgi_cmd()
995 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); in its_build_vsgi_cmd()
996 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable); in its_build_vsgi_cmd()
1000 return valid_vpe(its, desc->its_vsgi_cmd.vpe); in its_build_vsgi_cmd()
1006 return (ptr - its->cmd_base) * sizeof(*ptr); in its_cmd_ptr_to_offset()
1014 widx = its->cmd_write - its->cmd_base; in its_queue_full()
1015 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); in its_queue_full()
1030 count--; in its_allocate_entry()
1039 cmd = its->cmd_write++; in its_allocate_entry()
1042 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) in its_allocate_entry()
1043 its->cmd_write = its->cmd_base; in its_allocate_entry()
1046 cmd->raw_cmd[0] = 0; in its_allocate_entry()
1047 cmd->raw_cmd[1] = 0; in its_allocate_entry()
1048 cmd->raw_cmd[2] = 0; in its_allocate_entry()
1049 cmd->raw_cmd[3] = 0; in its_allocate_entry()
1056 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); in its_post_commands()
1058 writel_relaxed(wr, its->base + GITS_CWRITER); in its_post_commands()
1060 return its->cmd_write; in its_post_commands()
1069 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) in its_flush_cmd()
1092 rd_idx = readl_relaxed(its->base + GITS_CREADR); in its_wait_for_range_completion()
1096 * potential wrap-around into account. in its_wait_for_range_completion()
1098 delta = rd_idx - prev_idx; in its_wait_for_range_completion()
1106 count--; in its_wait_for_range_completion()
1110 return -1; in its_wait_for_range_completion()
1131 raw_spin_lock_irqsave(&its->lock, flags); \
1135 raw_spin_unlock_irqrestore(&its->lock, flags); \
1151 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1153 raw_spin_unlock_irqrestore(&its->lock, flags); \
1164 its_encode_target(sync_cmd, sync_col->target_address); in its_build_sync_cmd()
1177 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); in BUILD_SINGLE_CMD_FUNC()
1192 its_send_single_command(dev->its, its_build_int_cmd, &desc); in BUILD_SINGLE_CMD_FUNC()
1202 its_send_single_command(dev->its, its_build_clear_cmd, &desc); in its_send_clear()
1212 its_send_single_command(dev->its, its_build_inv_cmd, &desc); in its_send_inv()
1222 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); in its_send_mapd()
1244 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); in its_send_mapti()
1256 its_send_single_command(dev->its, its_build_movi_cmd, &desc); in its_send_movi()
1266 its_send_single_command(dev->its, its_build_discard_cmd, &desc); in its_send_discard()
1283 desc.its_vmapti_cmd.vpe = map->vpe; in its_send_vmapti()
1285 desc.its_vmapti_cmd.virt_id = map->vintid; in its_send_vmapti()
1287 desc.its_vmapti_cmd.db_enabled = map->db_enabled; in its_send_vmapti()
1289 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); in its_send_vmapti()
1297 desc.its_vmovi_cmd.vpe = map->vpe; in its_send_vmovi()
1300 desc.its_vmovi_cmd.db_enabled = map->db_enabled; in its_send_vmovi()
1302 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); in its_send_vmovi()
1312 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; in its_send_vmapp()
1321 int col_id = vpe->col_idx; in its_send_vmovp()
1327 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1338 * Wall <-- Head. in its_send_vmovp()
1342 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm); in its_send_vmovp()
1349 if (!require_its_list_vmovp(vpe->its_vm, its)) in its_send_vmovp()
1352 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1376 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); in its_send_vinv()
1390 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); in its_send_vint()
1404 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); in its_send_vclear()
1416 * irqchip functions - assumes MSI, mostly.
1426 va = page_address(map->vm->vprop_page); in lpi_write_config()
1427 hwirq = map->vintid; in lpi_write_config()
1430 map->properties &= ~clr; in lpi_write_config()
1431 map->properties |= set | LPI_PROP_GROUP1; in lpi_write_config()
1433 va = gic_rdists->prop_table_va; in lpi_write_config()
1434 hwirq = d->hwirq; in lpi_write_config()
1437 cfg = va + hwirq - 8192; in lpi_write_config()
1446 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) in lpi_write_config()
1466 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); in __direct_lpi_inv()
1468 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; in __direct_lpi_inv()
1472 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); in __direct_lpi_inv()
1484 WARN_ON(!is_v4_1(its_dev->its)); in direct_lpi_inv()
1487 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); in direct_lpi_inv()
1488 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); in direct_lpi_inv()
1490 val = d->hwirq; in direct_lpi_inv()
1501 if (gic_rdists->has_direct_lpi && in lpi_update_config()
1502 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) in lpi_update_config()
1517 * GICv4.1 does away with the per-LPI nonsense, nothing to do in its_vlpi_set_doorbell()
1520 if (is_v4_1(its_dev->its)) in its_vlpi_set_doorbell()
1525 if (map->db_enabled == enable) in its_vlpi_set_doorbell()
1528 map->db_enabled = enable; in its_vlpi_set_doorbell()
1562 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_read_lpi_count()
1564 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_read_lpi_count()
1570 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_inc_lpi_count()
1572 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_inc_lpi_count()
1578 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_dec_lpi_count()
1580 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_dec_lpi_count()
1613 node = its_dev->its->numa_node; in its_select_cpu()
1645 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) in its_select_cpu()
1663 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && in its_select_cpu()
1672 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); in its_select_cpu()
1686 return -EINVAL; in its_set_affinity()
1688 prev_cpu = its_dev->event_map.col_map[id]; in its_set_affinity()
1701 target_col = &its_dev->its->collections[cpu]; in its_set_affinity()
1703 its_dev->event_map.col_map[id] = cpu; in its_set_affinity()
1713 return -EINVAL; in its_set_affinity()
1718 struct its_node *its = its_dev->its; in its_irq_get_msi_base()
1720 return its->phys_base + GITS_TRANSLATER; in its_irq_get_msi_base()
1729 its = its_dev->its; in its_irq_compose_msi_msg()
1730 addr = its->get_msi_base(its_dev); in its_irq_compose_msi_msg()
1732 msg->address_lo = lower_32_bits(addr); in its_irq_compose_msi_msg()
1733 msg->address_hi = upper_32_bits(addr); in its_irq_compose_msi_msg()
1734 msg->data = its_get_event_id(d); in its_irq_compose_msi_msg()
1747 return -EINVAL; in its_irq_set_irqchip_state()
1783 if (!its_list_map || gic_rdists->has_rvpeid) in gic_requires_eager_mapping()
1794 guard(raw_spinlock_irqsave)(&vm->vmapp_lock); in its_map_vm()
1800 vm->vlpi_count[its->list_nr]++; in its_map_vm()
1802 if (vm->vlpi_count[its->list_nr] == 1) { in its_map_vm()
1805 for (i = 0; i < vm->nr_vpes; i++) { in its_map_vm()
1806 struct its_vpe *vpe = vm->vpes[i]; in its_map_vm()
1808 scoped_guard(raw_spinlock, &vpe->vpe_lock) in its_map_vm()
1822 guard(raw_spinlock_irqsave)(&vm->vmapp_lock); in its_unmap_vm()
1824 if (!--vm->vlpi_count[its->list_nr]) { in its_unmap_vm()
1827 for (i = 0; i < vm->nr_vpes; i++) { in its_unmap_vm()
1828 guard(raw_spinlock)(&vm->vpes[i]->vpe_lock); in its_unmap_vm()
1829 its_send_vmapp(its, vm->vpes[i], false); in its_unmap_vm()
1839 if (!info->map) in its_vlpi_map()
1840 return -EINVAL; in its_vlpi_map()
1842 if (!its_dev->event_map.vm) { in its_vlpi_map()
1845 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), in its_vlpi_map()
1848 return -ENOMEM; in its_vlpi_map()
1850 its_dev->event_map.vm = info->map->vm; in its_vlpi_map()
1851 its_dev->event_map.vlpi_maps = maps; in its_vlpi_map()
1852 } else if (its_dev->event_map.vm != info->map->vm) { in its_vlpi_map()
1853 return -EINVAL; in its_vlpi_map()
1857 its_dev->event_map.vlpi_maps[event] = *info->map; in its_vlpi_map()
1864 its_map_vm(its_dev->its, info->map->vm); in its_vlpi_map()
1873 lpi_write_config(d, 0xff, info->map->properties); in its_vlpi_map()
1882 its_dev->event_map.nr_vlpis++; in its_vlpi_map()
1895 if (!its_dev->event_map.vm || !map) in its_vlpi_get()
1896 return -EINVAL; in its_vlpi_get()
1899 *info->map = *map; in its_vlpi_get()
1909 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) in its_vlpi_unmap()
1910 return -EINVAL; in its_vlpi_unmap()
1917 its_send_mapti(its_dev, d->hwirq, event); in its_vlpi_unmap()
1923 its_unmap_vm(its_dev->its, its_dev->event_map.vm); in its_vlpi_unmap()
1929 if (!--its_dev->event_map.nr_vlpis) { in its_vlpi_unmap()
1930 its_dev->event_map.vm = NULL; in its_vlpi_unmap()
1931 kfree(its_dev->event_map.vlpi_maps); in its_vlpi_unmap()
1941 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) in its_vlpi_prop_update()
1942 return -EINVAL; in its_vlpi_prop_update()
1944 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) in its_vlpi_prop_update()
1945 lpi_update_config(d, 0xff, info->config); in its_vlpi_prop_update()
1947 lpi_write_config(d, 0xff, info->config); in its_vlpi_prop_update()
1948 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); in its_vlpi_prop_update()
1959 if (!is_v4(its_dev->its)) in its_irq_set_vcpu_affinity()
1960 return -EINVAL; in its_irq_set_vcpu_affinity()
1962 guard(raw_spinlock_irq)(&its_dev->event_map.vlpi_lock); in its_irq_set_vcpu_affinity()
1968 switch (info->cmd_type) { in its_irq_set_vcpu_affinity()
1980 return -EINVAL; in its_irq_set_vcpu_affinity()
2029 range->base_id = base; in mk_lpi_range()
2030 range->span = span; in mk_lpi_range()
2039 int err = -ENOSPC; in alloc_lpi_range()
2044 if (range->span >= nr_lpis) { in alloc_lpi_range()
2045 *base = range->base_id; in alloc_lpi_range()
2046 range->base_id += nr_lpis; in alloc_lpi_range()
2047 range->span -= nr_lpis; in alloc_lpi_range()
2049 if (range->span == 0) { in alloc_lpi_range()
2050 list_del(&range->entry); in alloc_lpi_range()
2067 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list) in merge_lpi_ranges()
2069 if (a->base_id + a->span != b->base_id) in merge_lpi_ranges()
2071 b->base_id = a->base_id; in merge_lpi_ranges()
2072 b->span += a->span; in merge_lpi_ranges()
2073 list_del(&a->entry); in merge_lpi_ranges()
2083 return -ENOMEM; in free_lpi_range()
2088 if (old->base_id < base) in free_lpi_range()
2092 * old is the last element with ->base_id smaller than base, in free_lpi_range()
2094 * ->base_id smaller than base, &old->entry ends up pointing in free_lpi_range()
2098 list_add(&new->entry, &old->entry); in free_lpi_range()
2112 u32 lpis = (1UL << id_bits) - 8192; in its_lpi_init()
2116 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer); in its_lpi_init()
2147 err = -ENOSPC; in its_lpi_alloc()
2173 /* Regular IRQ priority, Group-1, disabled */ in gic_reset_prop_table()
2212 addr_end = addr + size - 1; in gic_check_reserved_range()
2236 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) { in its_setup_lpi_prop_table()
2242 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12); in its_setup_lpi_prop_table()
2243 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa, in its_setup_lpi_prop_table()
2246 gic_reset_prop_table(gic_rdists->prop_table_va); in its_setup_lpi_prop_table()
2251 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer), in its_setup_lpi_prop_table()
2256 return -ENOMEM; in its_setup_lpi_prop_table()
2259 gic_rdists->prop_table_pa = page_to_phys(page); in its_setup_lpi_prop_table()
2260 gic_rdists->prop_table_va = page_address(page); in its_setup_lpi_prop_table()
2261 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa, in its_setup_lpi_prop_table()
2266 &gic_rdists->prop_table_pa); in its_setup_lpi_prop_table()
2283 u32 idx = baser - its->tables; in its_read_baser()
2285 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); in its_read_baser()
2291 u32 idx = baser - its->tables; in its_write_baser()
2293 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); in its_write_baser()
2294 baser->val = its_read_baser(its, baser); in its_write_baser()
2308 psz = baser->psz; in its_setup_baser()
2311 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", in its_setup_baser()
2312 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2318 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); in its_setup_baser()
2320 return -ENOMEM; in its_setup_baser()
2332 return -ENXIO; in its_setup_baser()
2342 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | in its_setup_baser()
2343 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | in its_setup_baser()
2366 tmp = baser->val; in its_setup_baser()
2374 * non-cacheable as well. in its_setup_baser()
2385 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2388 return -ENXIO; in its_setup_baser()
2391 baser->order = order; in its_setup_baser()
2392 baser->base = base; in its_setup_baser()
2393 baser->psz = psz; in its_setup_baser()
2397 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), in its_setup_baser()
2415 u32 psz = baser->psz; in its_parse_indirect_baser()
2421 * Find out whether hw supports a single or two-level table by in its_parse_indirect_baser()
2425 indirect = !!(baser->val & GITS_BASER_INDIRECT); in its_parse_indirect_baser()
2435 ids -= ilog2(psz / (int)esz); in its_parse_indirect_baser()
2444 * massive waste of memory if two-level device table in its_parse_indirect_baser()
2451 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", in its_parse_indirect_baser()
2452 &its->phys_base, its_base_type_string[type], in its_parse_indirect_baser()
2481 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); in compute_its_aff()
2483 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); in compute_its_aff()
2492 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) in find_sibling_its()
2503 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in find_sibling_its()
2510 baser = its->tables[2].val; in find_sibling_its()
2525 if (its->tables[i].base) { in its_free_tables()
2526 free_pages((unsigned long)its->tables[i].base, in its_free_tables()
2527 its->tables[i].order); in its_free_tables()
2528 its->tables[i].base = NULL; in its_free_tables()
2561 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz) in its_probe_baser_psz()
2573 return -1; in its_probe_baser_psz()
2577 baser->psz = psz; in its_probe_baser_psz()
2587 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) in its_alloc_tables()
2591 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) { in its_alloc_tables()
2597 struct its_baser *baser = its->tables + i; in its_alloc_tables()
2608 return -ENXIO; in its_alloc_tables()
2611 order = get_order(baser->psz); in its_alloc_tables()
2625 *baser = sibling->tables[2]; in its_alloc_tables()
2626 its_write_baser(its, baser, baser->val); in its_alloc_tables()
2643 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; in its_alloc_tables()
2644 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; in its_alloc_tables()
2665 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in inherit_vpe_l1_table_from_its()
2672 baser = its->tables[2].val; in inherit_vpe_l1_table_from_its()
2677 gic_data_rdist()->vpe_l1_base = its->tables[2].base; in inherit_vpe_l1_table_from_its()
2699 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); in inherit_vpe_l1_table_from_its()
2717 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; in inherit_vpe_l1_table_from_rd()
2735 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; in inherit_vpe_l1_table_from_rd()
2736 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; in inherit_vpe_l1_table_from_rd()
2746 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; in allocate_vpe_l2_table()
2752 if (!gic_rdists->has_rvpeid) in allocate_vpe_l2_table()
2755 /* Skip non-present CPUs */ in allocate_vpe_l2_table()
2789 table = gic_data_rdist_cpu(cpu)->vpe_l1_base; in allocate_vpe_l2_table()
2822 if (!gic_rdists->has_rvpeid) in allocate_vpe_l1_table()
2841 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); in allocate_vpe_l1_table()
2845 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); in allocate_vpe_l1_table()
2846 if (!gic_data_rdist()->vpe_table_mask) in allocate_vpe_l1_table()
2847 return -ENOMEM; in allocate_vpe_l1_table()
2905 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); in allocate_vpe_l1_table()
2914 return -ENOMEM; in allocate_vpe_l1_table()
2916 gic_data_rdist()->vpe_l1_base = page_address(page); in allocate_vpe_l1_table()
2930 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); in allocate_vpe_l1_table()
2934 cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); in allocate_vpe_l1_table()
2943 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), in its_alloc_collections()
2945 if (!its->collections) in its_alloc_collections()
2946 return -ENOMEM; in its_alloc_collections()
2949 its->collections[i].target_address = ~0ULL; in its_alloc_collections()
2963 /* Make sure the GIC will observe the zero-ed page */ in its_allocate_pending_table()
2997 * flag the RD tables as pre-allocated if the stars do align. in allocate_lpi_tables()
3001 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED | in allocate_lpi_tables()
3021 return -ENOMEM; in allocate_lpi_tables()
3024 gic_data_rdist_cpu(cpu)->pend_page = pend_page; in allocate_lpi_tables()
3040 count--; in read_vpend_dirty_clear()
3077 if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) in its_cpu_init_lpis()
3081 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && in its_cpu_init_lpis()
3089 if (WARN_ON(gic_rdists->prop_table_pa != paddr)) in its_cpu_init_lpis()
3096 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED; in its_cpu_init_lpis()
3101 pend_page = gic_data_rdist()->pend_page; in its_cpu_init_lpis()
3105 val = (gic_rdists->prop_table_pa | in its_cpu_init_lpis()
3108 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); in its_cpu_init_lpis()
3119 * The HW reports non-shareable, we must in its_cpu_init_lpis()
3129 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; in its_cpu_init_lpis()
3145 * The HW reports non-shareable, we must remove the in its_cpu_init_lpis()
3160 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { in its_cpu_init_lpis()
3170 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; in its_cpu_init_lpis()
3189 gic_rdists->has_rvpeid = false; in its_cpu_init_lpis()
3190 gic_rdists->has_vlpis = false; in its_cpu_init_lpis()
3195 gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; in its_cpu_init_lpis()
3198 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ? in its_cpu_init_lpis()
3209 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { in its_cpu_init_collection()
3213 if (its->numa_node != NUMA_NO_NODE && in its_cpu_init_collection()
3214 its->numa_node != of_node_to_nid(cpu_node)) in its_cpu_init_collection()
3222 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { in its_cpu_init_collection()
3227 target = gic_data_rdist()->phys_base; in its_cpu_init_collection()
3235 its->collections[cpu].target_address = target; in its_cpu_init_collection()
3236 its->collections[cpu].col_id = cpu; in its_cpu_init_collection()
3238 its_send_mapc(its, &its->collections[cpu], 1); in its_cpu_init_collection()
3239 its_send_invall(its, &its->collections[cpu]); in its_cpu_init_collection()
3259 raw_spin_lock_irqsave(&its->lock, flags); in its_find_device()
3261 list_for_each_entry(tmp, &its->its_device_list, entry) { in its_find_device()
3262 if (tmp->device_id == dev_id) { in its_find_device()
3268 raw_spin_unlock_irqrestore(&its->lock, flags); in its_find_device()
3278 if (GITS_BASER_TYPE(its->tables[i].val) == type) in its_get_baser()
3279 return &its->tables[i]; in its_get_baser()
3293 esz = GITS_BASER_ENTRY_SIZE(baser->val); in its_alloc_table_entry()
3294 if (!(baser->val & GITS_BASER_INDIRECT)) in its_alloc_table_entry()
3295 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); in its_alloc_table_entry()
3298 idx = id >> ilog2(baser->psz / esz); in its_alloc_table_entry()
3299 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) in its_alloc_table_entry()
3302 table = baser->base; in its_alloc_table_entry()
3306 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, in its_alloc_table_entry()
3307 get_order(baser->psz)); in its_alloc_table_entry()
3312 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) in its_alloc_table_entry()
3313 gic_flush_dcache_to_poc(page_address(page), baser->psz); in its_alloc_table_entry()
3318 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) in its_alloc_table_entry()
3368 if (!gic_rdists->has_rvpeid) in its_alloc_vpe_table()
3408 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); in its_create_device()
3409 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; in its_create_device()
3410 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); in its_create_device()
3432 dev->its = its; in its_create_device()
3433 dev->itt = itt; in its_create_device()
3434 dev->nr_ites = nr_ites; in its_create_device()
3435 dev->event_map.lpi_map = lpi_map; in its_create_device()
3436 dev->event_map.col_map = col_map; in its_create_device()
3437 dev->event_map.lpi_base = lpi_base; in its_create_device()
3438 dev->event_map.nr_lpis = nr_lpis; in its_create_device()
3439 raw_spin_lock_init(&dev->event_map.vlpi_lock); in its_create_device()
3440 dev->device_id = dev_id; in its_create_device()
3441 INIT_LIST_HEAD(&dev->entry); in its_create_device()
3443 raw_spin_lock_irqsave(&its->lock, flags); in its_create_device()
3444 list_add(&dev->entry, &its->its_device_list); in its_create_device()
3445 raw_spin_unlock_irqrestore(&its->lock, flags); in its_create_device()
3457 raw_spin_lock_irqsave(&its_dev->its->lock, flags); in its_free_device()
3458 list_del(&its_dev->entry); in its_free_device()
3459 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); in its_free_device()
3460 kfree(its_dev->event_map.col_map); in its_free_device()
3461 kfree(its_dev->itt); in its_free_device()
3470 idx = bitmap_find_free_region(dev->event_map.lpi_map, in its_alloc_device_irq()
3471 dev->event_map.nr_lpis, in its_alloc_device_irq()
3474 return -ENOSPC; in its_alloc_device_irq()
3476 *hwirq = dev->event_map.lpi_base + idx; in its_alloc_device_irq()
3496 dev_id = info->scratchpad[0].ul; in its_msi_prepare()
3499 its = msi_info->data; in its_msi_prepare()
3501 if (!gic_rdists->has_direct_lpi && in its_msi_prepare()
3503 vpe_proxy.dev->its == its && in its_msi_prepare()
3504 dev_id == vpe_proxy.dev->device_id) { in its_msi_prepare()
3508 return -EINVAL; in its_msi_prepare()
3511 mutex_lock(&its->dev_alloc_lock); in its_msi_prepare()
3519 its_dev->shared = true; in its_msi_prepare()
3526 err = -ENOMEM; in its_msi_prepare()
3530 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE) in its_msi_prepare()
3531 its_dev->shared = true; in its_msi_prepare()
3535 mutex_unlock(&its->dev_alloc_lock); in its_msi_prepare()
3536 info->scratchpad[0].ptr = its_dev; in its_msi_prepare()
3550 if (irq_domain_get_of_node(domain->parent)) { in its_irq_gic_domain_alloc()
3551 fwspec.fwnode = domain->parent->fwnode; in its_irq_gic_domain_alloc()
3556 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { in its_irq_gic_domain_alloc()
3557 fwspec.fwnode = domain->parent->fwnode; in its_irq_gic_domain_alloc()
3562 return -EINVAL; in its_irq_gic_domain_alloc()
3572 struct its_device *its_dev = info->scratchpad[0].ptr; in its_irq_domain_alloc()
3573 struct its_node *its = its_dev->its; in its_irq_domain_alloc()
3583 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); in its_irq_domain_alloc()
3599 (int)(hwirq + i - its_dev->event_map.lpi_base), in its_irq_domain_alloc()
3615 return -EINVAL; in its_irq_domain_activate()
3618 its_dev->event_map.col_map[event] = cpu; in its_irq_domain_activate()
3622 its_send_mapti(its_dev, d->hwirq, event); in its_irq_domain_activate()
3632 its_dec_lpi_count(d, its_dev->event_map.col_map[event]); in its_irq_domain_deactivate()
3642 struct its_node *its = its_dev->its; in its_irq_domain_free()
3645 bitmap_release_region(its_dev->event_map.lpi_map, in its_irq_domain_free()
3656 mutex_lock(&its->dev_alloc_lock); in its_irq_domain_free()
3662 if (!its_dev->shared && in its_irq_domain_free()
3663 bitmap_empty(its_dev->event_map.lpi_map, in its_irq_domain_free()
3664 its_dev->event_map.nr_lpis)) { in its_irq_domain_free()
3665 its_lpi_free(its_dev->event_map.lpi_map, in its_irq_domain_free()
3666 its_dev->event_map.lpi_base, in its_irq_domain_free()
3667 its_dev->event_map.nr_lpis); in its_irq_domain_free()
3674 mutex_unlock(&its->dev_alloc_lock); in its_irq_domain_free()
3709 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_unmap_locked()
3713 if (vpe->vpe_proxy_event == -1) in its_vpe_db_proxy_unmap_locked()
3716 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); in its_vpe_db_proxy_unmap_locked()
3717 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; in its_vpe_db_proxy_unmap_locked()
3727 vpe_proxy.next_victim = vpe->vpe_proxy_event; in its_vpe_db_proxy_unmap_locked()
3729 vpe->vpe_proxy_event = -1; in its_vpe_db_proxy_unmap_locked()
3735 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_unmap()
3738 if (!gic_rdists->has_direct_lpi) { in its_vpe_db_proxy_unmap()
3750 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_map_locked()
3754 if (vpe->vpe_proxy_event != -1) in its_vpe_db_proxy_map_locked()
3763 vpe->vpe_proxy_event = vpe_proxy.next_victim; in its_vpe_db_proxy_map_locked()
3764 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; in its_vpe_db_proxy_map_locked()
3766 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; in its_vpe_db_proxy_map_locked()
3767 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); in its_vpe_db_proxy_map_locked()
3776 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_move()
3779 if (gic_rdists->has_direct_lpi) { in its_vpe_db_proxy_move()
3782 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; in its_vpe_db_proxy_move()
3783 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); in its_vpe_db_proxy_move()
3793 target_col = &vpe_proxy.dev->its->collections[to]; in its_vpe_db_proxy_move()
3794 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); in its_vpe_db_proxy_move()
3795 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; in its_vpe_db_proxy_move()
3818 * protect us, and that we must ensure nobody samples vpe->col_idx in its_vpe_set_affinity()
3820 * taken on any vLPI handling path that evaluates vpe->col_idx. in its_vpe_set_affinity()
3827 raw_spin_lock(&vpe->its_vm->vmapp_lock); in its_vpe_set_affinity()
3830 table_mask = gic_data_rdist_cpu(from)->vpe_table_mask; in its_vpe_set_affinity()
3849 vpe->col_idx = cpu; in its_vpe_set_affinity()
3859 raw_spin_unlock(&vpe->its_vm->vmapp_lock); in its_vpe_set_affinity()
3869 if (!gic_rdists->has_vpend_valid_dirty) in its_wait_vpt_parse_complete()
3884 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & in its_vpe_schedule()
3886 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; in its_vpe_schedule()
3893 val = virt_to_phys(page_address(vpe->vpt_page)) & in its_vpe_schedule()
3902 * easily. So in the end, vpe->pending_last is only an in its_vpe_schedule()
3909 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; in its_vpe_schedule()
3921 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); in its_vpe_deschedule()
3922 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); in its_vpe_deschedule()
3929 guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock); in its_vpe_invall()
3935 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) in its_vpe_invall()
3952 switch (info->cmd_type) { in its_vpe_set_vcpu_affinity()
3970 return -EINVAL; in its_vpe_set_vcpu_affinity()
3982 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); in its_vpe_send_cmd()
3991 if (gic_rdists->has_direct_lpi) in its_vpe_send_inv()
3992 __direct_lpi_inv(d, d->parent_data->hwirq); in its_vpe_send_inv()
4005 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); in its_vpe_mask_irq()
4012 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); in its_vpe_unmask_irq()
4023 return -EINVAL; in its_vpe_set_irqchip_state()
4025 if (gic_rdists->has_direct_lpi) { in its_vpe_set_irqchip_state()
4028 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; in its_vpe_set_irqchip_state()
4030 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); in its_vpe_set_irqchip_state()
4032 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); in its_vpe_set_irqchip_state()
4051 .name = "GICv4-vpe",
4095 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); in its_vpe_4_1_mask_irq()
4101 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); in its_vpe_4_1_unmask_irq()
4113 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; in its_vpe_4_1_schedule()
4114 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; in its_vpe_4_1_schedule()
4115 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); in its_vpe_4_1_schedule()
4126 if (info->req_db) { in its_vpe_4_1_deschedule()
4130 * vPE is going to block: make the vPE non-resident with in its_vpe_4_1_deschedule()
4132 * we read-back PendingLast clear, then a doorbell will be in its_vpe_4_1_deschedule()
4139 raw_spin_lock_irqsave(&vpe->vpe_lock, flags); in its_vpe_4_1_deschedule()
4143 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); in its_vpe_4_1_deschedule()
4144 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); in its_vpe_4_1_deschedule()
4147 * We're not blocking, so just make the vPE non-resident in its_vpe_4_1_deschedule()
4153 vpe->pending_last = true; in its_vpe_4_1_deschedule()
4165 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); in its_vpe_4_1_invall()
4169 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_vpe_4_1_invall()
4170 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; in its_vpe_4_1_invall()
4174 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_vpe_4_1_invall()
4183 switch (info->cmd_type) { in its_vpe_4_1_set_vcpu_affinity()
4201 return -EINVAL; in its_vpe_4_1_set_vcpu_affinity()
4206 .name = "GICv4.1-vpe",
4220 desc.its_vsgi_cmd.sgi = d->hwirq; in its_configure_sgi()
4221 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority; in its_configure_sgi()
4222 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled; in its_configure_sgi()
4223 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group; in its_configure_sgi()
4238 vpe->sgi_config[d->hwirq].enabled = false; in its_sgi_mask_irq()
4246 vpe->sgi_config[d->hwirq].enabled = true; in its_sgi_unmask_irq()
4268 return -EINVAL; in its_sgi_set_irqchip_state()
4275 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); in its_sgi_set_irqchip_state()
4276 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); in its_sgi_set_irqchip_state()
4277 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); in its_sgi_set_irqchip_state()
4296 return -EINVAL; in its_sgi_get_irqchip_state()
4301 * - Concurrent vPE affinity change: we must make sure it cannot in its_sgi_get_irqchip_state()
4305 * - Concurrent VSGIPENDR access: As it involves accessing two in its_sgi_get_irqchip_state()
4309 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_sgi_get_irqchip_state()
4310 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K; in its_sgi_get_irqchip_state()
4311 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR); in its_sgi_get_irqchip_state()
4317 count--; in its_sgi_get_irqchip_state()
4327 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_sgi_get_irqchip_state()
4331 return -ENXIO; in its_sgi_get_irqchip_state()
4333 *val = !!(status & (1 << d->hwirq)); in its_sgi_get_irqchip_state()
4343 switch (info->cmd_type) { in its_sgi_set_vcpu_affinity()
4345 vpe->sgi_config[d->hwirq].priority = info->priority; in its_sgi_set_vcpu_affinity()
4346 vpe->sgi_config[d->hwirq].group = info->group; in its_sgi_set_vcpu_affinity()
4351 return -EINVAL; in its_sgi_set_vcpu_affinity()
4356 .name = "GICv4.1-sgi",
4376 vpe->sgi_config[i].priority = 0; in its_sgi_irq_domain_alloc()
4377 vpe->sgi_config[i].enabled = false; in its_sgi_irq_domain_alloc()
4378 vpe->sgi_config[i].group = false; in its_sgi_irq_domain_alloc()
4411 * - To change the configuration, CLEAR must be set to false, in its_sgi_irq_domain_deactivate()
4413 * - To clear the pending bit, CLEAR must be set to true, leaving in its_sgi_irq_domain_deactivate()
4418 vpe->sgi_config[d->hwirq].enabled = false; in its_sgi_irq_domain_deactivate()
4432 return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL); in its_vpe_id_alloc()
4454 return -ENOMEM; in its_vpe_init()
4460 return -ENOMEM; in its_vpe_init()
4463 raw_spin_lock_init(&vpe->vpe_lock); in its_vpe_init()
4464 vpe->vpe_id = vpe_id; in its_vpe_init()
4465 vpe->vpt_page = vpt_page; in its_vpe_init()
4466 if (gic_rdists->has_rvpeid) in its_vpe_init()
4467 atomic_set(&vpe->vmapp_count, 0); in its_vpe_init()
4469 vpe->vpe_proxy_event = -1; in its_vpe_init()
4477 its_vpe_id_free(vpe->vpe_id); in its_vpe_teardown()
4478 its_free_pending_table(vpe->vpt_page); in its_vpe_teardown()
4485 struct its_vm *vm = domain->host_data; in its_vpe_irq_domain_free()
4495 BUG_ON(vm != vpe->its_vm); in its_vpe_irq_domain_free()
4497 clear_bit(data->hwirq, vm->db_bitmap); in its_vpe_irq_domain_free()
4502 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { in its_vpe_irq_domain_free()
4503 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); in its_vpe_irq_domain_free()
4504 its_free_prop_table(vm->vprop_page); in its_vpe_irq_domain_free()
4519 return -ENOMEM; in its_vpe_irq_domain_alloc()
4523 return -ENOMEM; in its_vpe_irq_domain_alloc()
4529 return -ENOMEM; in its_vpe_irq_domain_alloc()
4532 vm->db_bitmap = bitmap; in its_vpe_irq_domain_alloc()
4533 vm->db_lpi_base = base; in its_vpe_irq_domain_alloc()
4534 vm->nr_db_lpis = nr_ids; in its_vpe_irq_domain_alloc()
4535 vm->vprop_page = vprop_page; in its_vpe_irq_domain_alloc()
4536 raw_spin_lock_init(&vm->vmapp_lock); in its_vpe_irq_domain_alloc()
4538 if (gic_rdists->has_rvpeid) in its_vpe_irq_domain_alloc()
4542 vm->vpes[i]->vpe_db_lpi = base + i; in its_vpe_irq_domain_alloc()
4543 err = its_vpe_init(vm->vpes[i]); in its_vpe_irq_domain_alloc()
4547 vm->vpes[i]->vpe_db_lpi); in its_vpe_irq_domain_alloc()
4551 irqchip, vm->vpes[i]); in its_vpe_irq_domain_alloc()
4569 vpe->col_idx = cpumask_first(cpu_online_mask); in its_vpe_irq_domain_activate()
4570 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); in its_vpe_irq_domain_activate()
4616 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count)) in its_vpe_irq_domain_deactivate()
4617 gic_flush_dcache_to_poc(page_address(vpe->vpt_page), in its_vpe_irq_domain_deactivate()
4652 count--; in its_force_quiescent()
4654 return -EBUSY; in its_force_quiescent()
4666 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_cavium_22375()
4667 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); in its_enable_quirk_cavium_22375()
4668 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; in its_enable_quirk_cavium_22375()
4677 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; in its_enable_quirk_cavium_23144()
4687 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; in its_enable_quirk_qdf2400_e0065()
4688 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); in its_enable_quirk_qdf2400_e0065()
4695 struct its_node *its = its_dev->its; in its_irq_get_msi_base_pre_its()
4698 * The Socionext Synquacer SoC has a so-called 'pre-ITS', in its_irq_get_msi_base_pre_its()
4699 * which maps 32-bit writes targeted at a separate window of in its_irq_get_msi_base_pre_its()
4704 return its->pre_its_base + (its_dev->device_id << 2); in its_irq_get_msi_base_pre_its()
4713 if (!fwnode_property_read_u32_array(its->fwnode_handle, in its_enable_quirk_socionext_synquacer()
4714 "socionext,synquacer-pre-its", in its_enable_quirk_socionext_synquacer()
4718 its->pre_its_base = pre_its_window[0]; in its_enable_quirk_socionext_synquacer()
4719 its->get_msi_base = its_irq_get_msi_base_pre_its; in its_enable_quirk_socionext_synquacer()
4721 ids = ilog2(pre_its_window[1]) - 2; in its_enable_quirk_socionext_synquacer()
4723 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_socionext_synquacer()
4724 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); in its_enable_quirk_socionext_synquacer()
4727 /* the pre-ITS breaks isolation, so disable MSI remapping */ in its_enable_quirk_socionext_synquacer()
4728 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_enable_quirk_socionext_synquacer()
4742 its->vlpi_redist_offset = SZ_128K; in its_enable_quirk_hip07_161600802()
4754 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in its_enable_rk3588001()
4755 gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; in its_enable_rk3588001()
4764 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in its_set_non_coherent()
4796 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4797 * implementation, but with a 'pre-ITS' added that requires
4800 .desc = "ITS: Socionext Synquacer pre-ITS",
4823 .desc = "ITS: non-coherent attribute",
4824 .property = "dma-noncoherent",
4833 u32 iidr = readl_relaxed(its->base + GITS_IIDR); in its_enable_quirks()
4837 if (is_of_node(its->fwnode_handle)) in its_enable_quirks()
4838 gic_enable_of_quirks(to_of_node(its->fwnode_handle), in its_enable_quirks()
4851 base = its->base; in its_save_disable()
4852 its->ctlr_save = readl_relaxed(base + GITS_CTLR); in its_save_disable()
4856 &its->phys_base, err); in its_save_disable()
4857 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
4861 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); in its_save_disable()
4869 base = its->base; in its_save_disable()
4870 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
4888 base = its->base; in its_restore_enable()
4902 &its->phys_base, ret); in its_restore_enable()
4906 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); in its_restore_enable()
4912 its->cmd_write = its->cmd_base; in its_restore_enable()
4917 struct its_baser *baser = &its->tables[i]; in its_restore_enable()
4919 if (!(baser->val & GITS_BASER_VALID)) in its_restore_enable()
4922 its_write_baser(its, baser, baser->val); in its_restore_enable()
4924 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_restore_enable()
4931 if (its->collections[smp_processor_id()].col_id < in its_restore_enable()
4948 its_base = ioremap(res->start, SZ_64K); in its_map_one()
4950 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); in its_map_one()
4951 *err = -ENOMEM; in its_map_one()
4957 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); in its_map_one()
4958 *err = -ENODEV; in its_map_one()
4964 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); in its_map_one()
4982 return -ENOMEM; in its_init_domain()
4984 info->ops = &its_msi_domain_ops; in its_init_domain()
4985 info->data = its; in its_init_domain()
4988 its->msi_domain_flags, 0, in its_init_domain()
4989 its->fwnode_handle, &its_domain_ops, in its_init_domain()
4993 return -ENOMEM; in its_init_domain()
4998 inner_domain->msi_parent_ops = &gic_v3_its_msi_parent_ops; in its_init_domain()
4999 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; in its_init_domain()
5010 if (gic_rdists->has_direct_lpi) { in its_init_vpe_domain()
5022 return -ENOMEM; in its_init_vpe_domain()
5025 devid = GENMASK(device_ids(its) - 1, 0); in its_init_vpe_domain()
5030 return -ENOMEM; in its_init_vpe_domain()
5033 BUG_ON(entries > vpe_proxy.dev->nr_ites); in its_init_vpe_domain()
5038 devid, vpe_proxy.dev->nr_ites); in its_init_vpe_domain()
5050 * guaranteed to be single-threaded, hence no in its_compute_its_list_map()
5057 &its->phys_base); in its_compute_its_list_map()
5058 return -EINVAL; in its_compute_its_list_map()
5061 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_compute_its_list_map()
5064 writel_relaxed(ctlr, its->base + GITS_CTLR); in its_compute_its_list_map()
5065 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_compute_its_list_map()
5073 &its->phys_base, its_number); in its_compute_its_list_map()
5074 return -EINVAL; in its_compute_its_list_map()
5090 if (!(its->typer & GITS_TYPER_VMOVP)) { in its_probe_one()
5095 its->list_nr = err; in its_probe_one()
5098 &its->phys_base, err); in its_probe_one()
5100 pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base); in its_probe_one()
5104 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); in its_probe_one()
5106 its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K); in its_probe_one()
5107 if (!its->sgir_base) { in its_probe_one()
5108 err = -ENOMEM; in its_probe_one()
5112 its->mpidr = readl_relaxed(its->base + GITS_MPIDR); in its_probe_one()
5115 &its->phys_base, its->mpidr, svpet); in its_probe_one()
5119 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, in its_probe_one()
5122 err = -ENOMEM; in its_probe_one()
5125 its->cmd_base = (void *)page_address(page); in its_probe_one()
5126 its->cmd_write = its->cmd_base; in its_probe_one()
5136 baser = (virt_to_phys(its->cmd_base) | in its_probe_one()
5139 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | in its_probe_one()
5142 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5143 tmp = gits_read_cbaser(its->base + GITS_CBASER); in its_probe_one()
5145 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) in its_probe_one()
5151 * The HW reports non-shareable, we must in its_probe_one()
5158 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5161 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; in its_probe_one()
5164 gits_write_cwriter(0, its->base + GITS_CWRITER); in its_probe_one()
5165 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_probe_one()
5169 writel_relaxed(ctlr, its->base + GITS_CTLR); in its_probe_one()
5176 list_add(&its->entry, &its_nodes); in its_probe_one()
5184 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); in its_probe_one()
5186 if (its->sgir_base) in its_probe_one()
5187 iounmap(its->sgir_base); in its_probe_one()
5189 pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err); in its_probe_one()
5206 return -ENXIO; in redist_disable_lpis()
5215 * LPIs before trying to re-enable them. They are already in redist_disable_lpis()
5220 if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) || in redist_disable_lpis()
5221 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) in redist_disable_lpis()
5247 return -ETIMEDOUT; in redist_disable_lpis()
5250 timeout--; in redist_disable_lpis()
5260 return -EBUSY; in redist_disable_lpis()
5284 cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); in rdist_memreserve_cpuhp_cleanup_workfn()
5285 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; in rdist_memreserve_cpuhp_cleanup_workfn()
5297 if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE) in its_cpu_memreserve_lpi()
5300 pend_page = gic_data_rdist()->pend_page; in its_cpu_memreserve_lpi()
5302 ret = -ENOMEM; in its_cpu_memreserve_lpi()
5306 * If the pending table was pre-programmed, free the memory we in its_cpu_memreserve_lpi()
5310 if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) { in its_cpu_memreserve_lpi()
5312 gic_data_rdist()->pend_page = NULL; in its_cpu_memreserve_lpi()
5324 gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; in its_cpu_memreserve_lpi()
5346 { .compatible = "arm,gic-v3-its", },
5367 raw_spin_lock_init(&its->lock); in its_node_init()
5368 mutex_init(&its->dev_alloc_lock); in its_node_init()
5369 INIT_LIST_HEAD(&its->entry); in its_node_init()
5370 INIT_LIST_HEAD(&its->its_device_list); in its_node_init()
5372 its->typer = gic_read_typer(its_base + GITS_TYPER); in its_node_init()
5373 its->base = its_base; in its_node_init()
5374 its->phys_base = res->start; in its_node_init()
5375 its->get_msi_base = its_irq_get_msi_base; in its_node_init()
5376 its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_node_init()
5378 its->numa_node = numa_node; in its_node_init()
5379 its->fwnode_handle = handle; in its_node_init()
5390 iounmap(its->base); in its_node_destroy()
5409 !of_property_read_bool(np, "msi-controller") || in its_of_probe()
5424 if (!of_property_read_bool(np, "msi-controller")) { in its_of_probe()
5425 pr_warn("%pOF: no msi-controller property, ITS ignored\n", in its_of_probe()
5436 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np)); in its_of_probe()
5438 return -ENOMEM; in its_of_probe()
5489 return -EINVAL; in gic_acpi_parse_srat_its()
5491 if (its_affinity->header.length < sizeof(*its_affinity)) { in gic_acpi_parse_srat_its()
5493 its_affinity->header.length); in gic_acpi_parse_srat_its()
5494 return -EINVAL; in gic_acpi_parse_srat_its()
5502 node = pxm_to_node(its_affinity->proximity_domain); in gic_acpi_parse_srat_its()
5510 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; in gic_acpi_parse_srat_its()
5512 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", in gic_acpi_parse_srat_its()
5513 its_affinity->proximity_domain, its_affinity->its_id, node); in gic_acpi_parse_srat_its()
5562 res.start = its_entry->base_address; in gic_acpi_parse_madt_its()
5563 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; in gic_acpi_parse_madt_its()
5570 return -ENOMEM; in gic_acpi_parse_madt_its()
5573 err = iort_register_domain_token(its_entry->translation_id, res.start, in gic_acpi_parse_madt_its()
5577 &res.start, its_entry->translation_id); in gic_acpi_parse_madt_its()
5582 acpi_get_its_numa_node(its_entry->translation_id)); in gic_acpi_parse_madt_its()
5584 err = -ENOMEM; in gic_acpi_parse_madt_its()
5589 (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT)) in gic_acpi_parse_madt_its()
5590 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in gic_acpi_parse_madt_its()
5597 iort_deregister_domain_token(its_entry->translation_id); in gic_acpi_parse_madt_its()
5611 .start = its_entry->base_address, in its_acpi_reset()
5612 .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1, in its_acpi_reset()
5648 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; in its_lpi_memreserve_init()
5656 gic_rdists->cpuhp_memreserve_state = state; in its_lpi_memreserve_init()
5682 return -ENXIO; in its_init()
5695 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) in its_init()
5696 rdists->has_rvpeid = false; in its_init()
5698 if (has_v4 & rdists->has_vlpis) { in its_init()
5708 rdists->has_vlpis = false; in its_init()