1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/acpi.h> 8 #include <linux/acpi_iort.h> 9 #include <linux/bitfield.h> 10 #include <linux/bitmap.h> 11 #include <linux/cpu.h> 12 #include <linux/crash_dump.h> 13 #include <linux/delay.h> 14 #include <linux/dma-iommu.h> 15 #include <linux/efi.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/irqdomain.h> 19 #include <linux/list.h> 20 #include <linux/log2.h> 21 #include <linux/memblock.h> 22 #include <linux/mm.h> 23 #include <linux/msi.h> 24 #include <linux/of.h> 25 #include <linux/of_address.h> 26 #include <linux/of_irq.h> 27 #include <linux/of_pci.h> 28 #include <linux/of_platform.h> 29 #include <linux/percpu.h> 30 #include <linux/slab.h> 31 #include <linux/syscore_ops.h> 32 33 #include <linux/irqchip.h> 34 #include <linux/irqchip/arm-gic-v3.h> 35 #include <linux/irqchip/arm-gic-v4.h> 36 37 #include <asm/cputype.h> 38 #include <asm/exception.h> 39 40 #include "irq-gic-common.h" 41 42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) 43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) 44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) 45 46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 47 #define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) 48 49 #define RD_LOCAL_LPI_ENABLED BIT(0) 50 #define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1) 51 #define RD_LOCAL_MEMRESERVE_DONE BIT(2) 52 53 static u32 lpi_id_bits; 54 55 /* 56 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to 57 * deal with (one configuration byte per interrupt). PENDBASE has to 58 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 59 */ 60 #define LPI_NRBITS lpi_id_bits 61 #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) 62 #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) 63 64 #define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI 65 66 /* 67 * Collection structure - just an ID, and a redistributor address to 68 * ping. We use one per CPU as a bag of interrupts assigned to this 69 * CPU. 70 */ 71 struct its_collection { 72 u64 target_address; 73 u16 col_id; 74 }; 75 76 /* 77 * The ITS_BASER structure - contains memory information, cached 78 * value of BASER register configuration and ITS page size. 79 */ 80 struct its_baser { 81 void *base; 82 u64 val; 83 u32 order; 84 u32 psz; 85 }; 86 87 struct its_device; 88 89 /* 90 * The ITS structure - contains most of the infrastructure, with the 91 * top-level MSI domain, the command queue, the collections, and the 92 * list of devices writing to it. 93 * 94 * dev_alloc_lock has to be taken for device allocations, while the 95 * spinlock must be taken to parse data structures such as the device 96 * list. 97 */ 98 struct its_node { 99 raw_spinlock_t lock; 100 struct mutex dev_alloc_lock; 101 struct list_head entry; 102 void __iomem *base; 103 void __iomem *sgir_base; 104 phys_addr_t phys_base; 105 struct its_cmd_block *cmd_base; 106 struct its_cmd_block *cmd_write; 107 struct its_baser tables[GITS_BASER_NR_REGS]; 108 struct its_collection *collections; 109 struct fwnode_handle *fwnode_handle; 110 u64 (*get_msi_base)(struct its_device *its_dev); 111 u64 typer; 112 u64 cbaser_save; 113 u32 ctlr_save; 114 u32 mpidr; 115 struct list_head its_device_list; 116 u64 flags; 117 unsigned long list_nr; 118 int numa_node; 119 unsigned int msi_domain_flags; 120 u32 pre_its_base; /* for Socionext Synquacer */ 121 int vlpi_redist_offset; 122 }; 123 124 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) 125 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) 126 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) 127 128 #define ITS_ITT_ALIGN SZ_256 129 130 /* The maximum number of VPEID bits supported by VLPI commands */ 131 #define ITS_MAX_VPEID_BITS \ 132 ({ \ 133 int nvpeid = 16; \ 134 if (gic_rdists->has_rvpeid && \ 135 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \ 136 nvpeid = 1 + (gic_rdists->gicd_typer2 & \ 137 GICD_TYPER2_VID); \ 138 \ 139 nvpeid; \ 140 }) 141 #define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS)) 142 143 /* Convert page order to size in bytes */ 144 #define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o)) 145 146 struct event_lpi_map { 147 unsigned long *lpi_map; 148 u16 *col_map; 149 irq_hw_number_t lpi_base; 150 int nr_lpis; 151 raw_spinlock_t vlpi_lock; 152 struct its_vm *vm; 153 struct its_vlpi_map *vlpi_maps; 154 int nr_vlpis; 155 }; 156 157 /* 158 * The ITS view of a device - belongs to an ITS, owns an interrupt 159 * translation table, and a list of interrupts. If it some of its 160 * LPIs are injected into a guest (GICv4), the event_map.vm field 161 * indicates which one. 162 */ 163 struct its_device { 164 struct list_head entry; 165 struct its_node *its; 166 struct event_lpi_map event_map; 167 void *itt; 168 u32 nr_ites; 169 u32 device_id; 170 bool shared; 171 }; 172 173 static struct { 174 raw_spinlock_t lock; 175 struct its_device *dev; 176 struct its_vpe **vpes; 177 int next_victim; 178 } vpe_proxy; 179 180 struct cpu_lpi_count { 181 atomic_t managed; 182 atomic_t unmanaged; 183 }; 184 185 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count); 186 187 static LIST_HEAD(its_nodes); 188 static DEFINE_RAW_SPINLOCK(its_lock); 189 static struct rdists *gic_rdists; 190 static struct irq_domain *its_parent; 191 192 static unsigned long its_list_map; 193 static u16 vmovp_seq_num; 194 static DEFINE_RAW_SPINLOCK(vmovp_lock); 195 196 static DEFINE_IDA(its_vpeid_ida); 197 198 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 199 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu)) 200 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 201 #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) 202 203 /* 204 * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we 205 * always have vSGIs mapped. 206 */ 207 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) 208 { 209 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); 210 } 211 212 static u16 get_its_list(struct its_vm *vm) 213 { 214 struct its_node *its; 215 unsigned long its_list = 0; 216 217 list_for_each_entry(its, &its_nodes, entry) { 218 if (!is_v4(its)) 219 continue; 220 221 if (require_its_list_vmovp(vm, its)) 222 __set_bit(its->list_nr, &its_list); 223 } 224 225 return (u16)its_list; 226 } 227 228 static inline u32 its_get_event_id(struct irq_data *d) 229 { 230 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 231 return d->hwirq - its_dev->event_map.lpi_base; 232 } 233 234 static struct its_collection *dev_event_to_col(struct its_device *its_dev, 235 u32 event) 236 { 237 struct its_node *its = its_dev->its; 238 239 return its->collections + its_dev->event_map.col_map[event]; 240 } 241 242 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev, 243 u32 event) 244 { 245 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis)) 246 return NULL; 247 248 return &its_dev->event_map.vlpi_maps[event]; 249 } 250 251 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d) 252 { 253 if (irqd_is_forwarded_to_vcpu(d)) { 254 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 255 u32 event = its_get_event_id(d); 256 257 return dev_event_to_vlpi_map(its_dev, event); 258 } 259 260 return NULL; 261 } 262 263 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags) 264 { 265 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags); 266 return vpe->col_idx; 267 } 268 269 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags) 270 { 271 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); 272 } 273 274 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags) 275 { 276 struct its_vlpi_map *map = get_vlpi_map(d); 277 int cpu; 278 279 if (map) { 280 cpu = vpe_to_cpuid_lock(map->vpe, flags); 281 } else { 282 /* Physical LPIs are already locked via the irq_desc lock */ 283 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 284 cpu = its_dev->event_map.col_map[its_get_event_id(d)]; 285 /* Keep GCC quiet... */ 286 *flags = 0; 287 } 288 289 return cpu; 290 } 291 292 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags) 293 { 294 struct its_vlpi_map *map = get_vlpi_map(d); 295 296 if (map) 297 vpe_to_cpuid_unlock(map->vpe, flags); 298 } 299 300 static struct its_collection *valid_col(struct its_collection *col) 301 { 302 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0))) 303 return NULL; 304 305 return col; 306 } 307 308 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe) 309 { 310 if (valid_col(its->collections + vpe->col_idx)) 311 return vpe; 312 313 return NULL; 314 } 315 316 /* 317 * ITS command descriptors - parameters to be encoded in a command 318 * block. 319 */ 320 struct its_cmd_desc { 321 union { 322 struct { 323 struct its_device *dev; 324 u32 event_id; 325 } its_inv_cmd; 326 327 struct { 328 struct its_device *dev; 329 u32 event_id; 330 } its_clear_cmd; 331 332 struct { 333 struct its_device *dev; 334 u32 event_id; 335 } its_int_cmd; 336 337 struct { 338 struct its_device *dev; 339 int valid; 340 } its_mapd_cmd; 341 342 struct { 343 struct its_collection *col; 344 int valid; 345 } its_mapc_cmd; 346 347 struct { 348 struct its_device *dev; 349 u32 phys_id; 350 u32 event_id; 351 } its_mapti_cmd; 352 353 struct { 354 struct its_device *dev; 355 struct its_collection *col; 356 u32 event_id; 357 } its_movi_cmd; 358 359 struct { 360 struct its_device *dev; 361 u32 event_id; 362 } its_discard_cmd; 363 364 struct { 365 struct its_collection *col; 366 } its_invall_cmd; 367 368 struct { 369 struct its_vpe *vpe; 370 } its_vinvall_cmd; 371 372 struct { 373 struct its_vpe *vpe; 374 struct its_collection *col; 375 bool valid; 376 } its_vmapp_cmd; 377 378 struct { 379 struct its_vpe *vpe; 380 struct its_device *dev; 381 u32 virt_id; 382 u32 event_id; 383 bool db_enabled; 384 } its_vmapti_cmd; 385 386 struct { 387 struct its_vpe *vpe; 388 struct its_device *dev; 389 u32 event_id; 390 bool db_enabled; 391 } its_vmovi_cmd; 392 393 struct { 394 struct its_vpe *vpe; 395 struct its_collection *col; 396 u16 seq_num; 397 u16 its_list; 398 } its_vmovp_cmd; 399 400 struct { 401 struct its_vpe *vpe; 402 } its_invdb_cmd; 403 404 struct { 405 struct its_vpe *vpe; 406 u8 sgi; 407 u8 priority; 408 bool enable; 409 bool group; 410 bool clear; 411 } its_vsgi_cmd; 412 }; 413 }; 414 415 /* 416 * The ITS command block, which is what the ITS actually parses. 417 */ 418 struct its_cmd_block { 419 union { 420 u64 raw_cmd[4]; 421 __le64 raw_cmd_le[4]; 422 }; 423 }; 424 425 #define ITS_CMD_QUEUE_SZ SZ_64K 426 #define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 427 428 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *, 429 struct its_cmd_block *, 430 struct its_cmd_desc *); 431 432 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *, 433 struct its_cmd_block *, 434 struct its_cmd_desc *); 435 436 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l) 437 { 438 u64 mask = GENMASK_ULL(h, l); 439 *raw_cmd &= ~mask; 440 *raw_cmd |= (val << l) & mask; 441 } 442 443 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 444 { 445 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); 446 } 447 448 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 449 { 450 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); 451 } 452 453 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 454 { 455 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); 456 } 457 458 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 459 { 460 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); 461 } 462 463 static void its_encode_size(struct its_cmd_block *cmd, u8 size) 464 { 465 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); 466 } 467 468 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 469 { 470 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); 471 } 472 473 static void its_encode_valid(struct its_cmd_block *cmd, int valid) 474 { 475 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); 476 } 477 478 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 479 { 480 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); 481 } 482 483 static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 484 { 485 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); 486 } 487 488 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid) 489 { 490 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); 491 } 492 493 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id) 494 { 495 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); 496 } 497 498 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id) 499 { 500 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); 501 } 502 503 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid) 504 { 505 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); 506 } 507 508 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num) 509 { 510 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); 511 } 512 513 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list) 514 { 515 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); 516 } 517 518 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa) 519 { 520 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); 521 } 522 523 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size) 524 { 525 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); 526 } 527 528 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa) 529 { 530 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); 531 } 532 533 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc) 534 { 535 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); 536 } 537 538 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz) 539 { 540 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); 541 } 542 543 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd, 544 u32 vpe_db_lpi) 545 { 546 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); 547 } 548 549 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd, 550 u32 vpe_db_lpi) 551 { 552 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0); 553 } 554 555 static void its_encode_db(struct its_cmd_block *cmd, bool db) 556 { 557 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); 558 } 559 560 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi) 561 { 562 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); 563 } 564 565 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio) 566 { 567 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); 568 } 569 570 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp) 571 { 572 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10); 573 } 574 575 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr) 576 { 577 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9); 578 } 579 580 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en) 581 { 582 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8); 583 } 584 585 static inline void its_fixup_cmd(struct its_cmd_block *cmd) 586 { 587 /* Let's fixup BE commands */ 588 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); 589 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); 590 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); 591 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); 592 } 593 594 static struct its_collection *its_build_mapd_cmd(struct its_node *its, 595 struct its_cmd_block *cmd, 596 struct its_cmd_desc *desc) 597 { 598 unsigned long itt_addr; 599 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 600 601 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 602 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 603 604 its_encode_cmd(cmd, GITS_CMD_MAPD); 605 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 606 its_encode_size(cmd, size - 1); 607 its_encode_itt(cmd, itt_addr); 608 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 609 610 its_fixup_cmd(cmd); 611 612 return NULL; 613 } 614 615 static struct its_collection *its_build_mapc_cmd(struct its_node *its, 616 struct its_cmd_block *cmd, 617 struct its_cmd_desc *desc) 618 { 619 its_encode_cmd(cmd, GITS_CMD_MAPC); 620 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 621 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 622 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 623 624 its_fixup_cmd(cmd); 625 626 return desc->its_mapc_cmd.col; 627 } 628 629 static struct its_collection *its_build_mapti_cmd(struct its_node *its, 630 struct its_cmd_block *cmd, 631 struct its_cmd_desc *desc) 632 { 633 struct its_collection *col; 634 635 col = dev_event_to_col(desc->its_mapti_cmd.dev, 636 desc->its_mapti_cmd.event_id); 637 638 its_encode_cmd(cmd, GITS_CMD_MAPTI); 639 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); 640 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); 641 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); 642 its_encode_collection(cmd, col->col_id); 643 644 its_fixup_cmd(cmd); 645 646 return valid_col(col); 647 } 648 649 static struct its_collection *its_build_movi_cmd(struct its_node *its, 650 struct its_cmd_block *cmd, 651 struct its_cmd_desc *desc) 652 { 653 struct its_collection *col; 654 655 col = dev_event_to_col(desc->its_movi_cmd.dev, 656 desc->its_movi_cmd.event_id); 657 658 its_encode_cmd(cmd, GITS_CMD_MOVI); 659 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 660 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 661 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 662 663 its_fixup_cmd(cmd); 664 665 return valid_col(col); 666 } 667 668 static struct its_collection *its_build_discard_cmd(struct its_node *its, 669 struct its_cmd_block *cmd, 670 struct its_cmd_desc *desc) 671 { 672 struct its_collection *col; 673 674 col = dev_event_to_col(desc->its_discard_cmd.dev, 675 desc->its_discard_cmd.event_id); 676 677 its_encode_cmd(cmd, GITS_CMD_DISCARD); 678 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 679 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 680 681 its_fixup_cmd(cmd); 682 683 return valid_col(col); 684 } 685 686 static struct its_collection *its_build_inv_cmd(struct its_node *its, 687 struct its_cmd_block *cmd, 688 struct its_cmd_desc *desc) 689 { 690 struct its_collection *col; 691 692 col = dev_event_to_col(desc->its_inv_cmd.dev, 693 desc->its_inv_cmd.event_id); 694 695 its_encode_cmd(cmd, GITS_CMD_INV); 696 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 697 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 698 699 its_fixup_cmd(cmd); 700 701 return valid_col(col); 702 } 703 704 static struct its_collection *its_build_int_cmd(struct its_node *its, 705 struct its_cmd_block *cmd, 706 struct its_cmd_desc *desc) 707 { 708 struct its_collection *col; 709 710 col = dev_event_to_col(desc->its_int_cmd.dev, 711 desc->its_int_cmd.event_id); 712 713 its_encode_cmd(cmd, GITS_CMD_INT); 714 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 715 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 716 717 its_fixup_cmd(cmd); 718 719 return valid_col(col); 720 } 721 722 static struct its_collection *its_build_clear_cmd(struct its_node *its, 723 struct its_cmd_block *cmd, 724 struct its_cmd_desc *desc) 725 { 726 struct its_collection *col; 727 728 col = dev_event_to_col(desc->its_clear_cmd.dev, 729 desc->its_clear_cmd.event_id); 730 731 its_encode_cmd(cmd, GITS_CMD_CLEAR); 732 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 733 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 734 735 its_fixup_cmd(cmd); 736 737 return valid_col(col); 738 } 739 740 static struct its_collection *its_build_invall_cmd(struct its_node *its, 741 struct its_cmd_block *cmd, 742 struct its_cmd_desc *desc) 743 { 744 its_encode_cmd(cmd, GITS_CMD_INVALL); 745 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); 746 747 its_fixup_cmd(cmd); 748 749 return desc->its_invall_cmd.col; 750 } 751 752 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, 753 struct its_cmd_block *cmd, 754 struct its_cmd_desc *desc) 755 { 756 its_encode_cmd(cmd, GITS_CMD_VINVALL); 757 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); 758 759 its_fixup_cmd(cmd); 760 761 return valid_vpe(its, desc->its_vinvall_cmd.vpe); 762 } 763 764 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, 765 struct its_cmd_block *cmd, 766 struct its_cmd_desc *desc) 767 { 768 unsigned long vpt_addr, vconf_addr; 769 u64 target; 770 bool alloc; 771 772 its_encode_cmd(cmd, GITS_CMD_VMAPP); 773 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); 774 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); 775 776 if (!desc->its_vmapp_cmd.valid) { 777 if (is_v4_1(its)) { 778 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); 779 its_encode_alloc(cmd, alloc); 780 } 781 782 goto out; 783 } 784 785 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); 786 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; 787 788 its_encode_target(cmd, target); 789 its_encode_vpt_addr(cmd, vpt_addr); 790 its_encode_vpt_size(cmd, LPI_NRBITS - 1); 791 792 if (!is_v4_1(its)) 793 goto out; 794 795 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); 796 797 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); 798 799 its_encode_alloc(cmd, alloc); 800 801 /* 802 * GICv4.1 provides a way to get the VLPI state, which needs the vPE 803 * to be unmapped first, and in this case, we may remap the vPE 804 * back while the VPT is not empty. So we can't assume that the 805 * VPT is empty on map. This is why we never advertise PTZ. 806 */ 807 its_encode_ptz(cmd, false); 808 its_encode_vconf_addr(cmd, vconf_addr); 809 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); 810 811 out: 812 its_fixup_cmd(cmd); 813 814 return valid_vpe(its, desc->its_vmapp_cmd.vpe); 815 } 816 817 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, 818 struct its_cmd_block *cmd, 819 struct its_cmd_desc *desc) 820 { 821 u32 db; 822 823 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) 824 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; 825 else 826 db = 1023; 827 828 its_encode_cmd(cmd, GITS_CMD_VMAPTI); 829 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); 830 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); 831 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); 832 its_encode_db_phys_id(cmd, db); 833 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); 834 835 its_fixup_cmd(cmd); 836 837 return valid_vpe(its, desc->its_vmapti_cmd.vpe); 838 } 839 840 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, 841 struct its_cmd_block *cmd, 842 struct its_cmd_desc *desc) 843 { 844 u32 db; 845 846 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) 847 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; 848 else 849 db = 1023; 850 851 its_encode_cmd(cmd, GITS_CMD_VMOVI); 852 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); 853 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); 854 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); 855 its_encode_db_phys_id(cmd, db); 856 its_encode_db_valid(cmd, true); 857 858 its_fixup_cmd(cmd); 859 860 return valid_vpe(its, desc->its_vmovi_cmd.vpe); 861 } 862 863 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, 864 struct its_cmd_block *cmd, 865 struct its_cmd_desc *desc) 866 { 867 u64 target; 868 869 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; 870 its_encode_cmd(cmd, GITS_CMD_VMOVP); 871 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); 872 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); 873 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); 874 its_encode_target(cmd, target); 875 876 if (is_v4_1(its)) { 877 its_encode_db(cmd, true); 878 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi); 879 } 880 881 its_fixup_cmd(cmd); 882 883 return valid_vpe(its, desc->its_vmovp_cmd.vpe); 884 } 885 886 static struct its_vpe *its_build_vinv_cmd(struct its_node *its, 887 struct its_cmd_block *cmd, 888 struct its_cmd_desc *desc) 889 { 890 struct its_vlpi_map *map; 891 892 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev, 893 desc->its_inv_cmd.event_id); 894 895 its_encode_cmd(cmd, GITS_CMD_INV); 896 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 897 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 898 899 its_fixup_cmd(cmd); 900 901 return valid_vpe(its, map->vpe); 902 } 903 904 static struct its_vpe *its_build_vint_cmd(struct its_node *its, 905 struct its_cmd_block *cmd, 906 struct its_cmd_desc *desc) 907 { 908 struct its_vlpi_map *map; 909 910 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev, 911 desc->its_int_cmd.event_id); 912 913 its_encode_cmd(cmd, GITS_CMD_INT); 914 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); 915 its_encode_event_id(cmd, desc->its_int_cmd.event_id); 916 917 its_fixup_cmd(cmd); 918 919 return valid_vpe(its, map->vpe); 920 } 921 922 static struct its_vpe *its_build_vclear_cmd(struct its_node *its, 923 struct its_cmd_block *cmd, 924 struct its_cmd_desc *desc) 925 { 926 struct its_vlpi_map *map; 927 928 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev, 929 desc->its_clear_cmd.event_id); 930 931 its_encode_cmd(cmd, GITS_CMD_CLEAR); 932 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); 933 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); 934 935 its_fixup_cmd(cmd); 936 937 return valid_vpe(its, map->vpe); 938 } 939 940 static struct its_vpe *its_build_invdb_cmd(struct its_node *its, 941 struct its_cmd_block *cmd, 942 struct its_cmd_desc *desc) 943 { 944 if (WARN_ON(!is_v4_1(its))) 945 return NULL; 946 947 its_encode_cmd(cmd, GITS_CMD_INVDB); 948 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); 949 950 its_fixup_cmd(cmd); 951 952 return valid_vpe(its, desc->its_invdb_cmd.vpe); 953 } 954 955 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, 956 struct its_cmd_block *cmd, 957 struct its_cmd_desc *desc) 958 { 959 if (WARN_ON(!is_v4_1(its))) 960 return NULL; 961 962 its_encode_cmd(cmd, GITS_CMD_VSGI); 963 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); 964 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); 965 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); 966 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); 967 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); 968 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable); 969 970 its_fixup_cmd(cmd); 971 972 return valid_vpe(its, desc->its_vsgi_cmd.vpe); 973 } 974 975 static u64 its_cmd_ptr_to_offset(struct its_node *its, 976 struct its_cmd_block *ptr) 977 { 978 return (ptr - its->cmd_base) * sizeof(*ptr); 979 } 980 981 static int its_queue_full(struct its_node *its) 982 { 983 int widx; 984 int ridx; 985 986 widx = its->cmd_write - its->cmd_base; 987 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 988 989 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 990 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 991 return 1; 992 993 return 0; 994 } 995 996 static struct its_cmd_block *its_allocate_entry(struct its_node *its) 997 { 998 struct its_cmd_block *cmd; 999 u32 count = 1000000; /* 1s! */ 1000 1001 while (its_queue_full(its)) { 1002 count--; 1003 if (!count) { 1004 pr_err_ratelimited("ITS queue not draining\n"); 1005 return NULL; 1006 } 1007 cpu_relax(); 1008 udelay(1); 1009 } 1010 1011 cmd = its->cmd_write++; 1012 1013 /* Handle queue wrapping */ 1014 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 1015 its->cmd_write = its->cmd_base; 1016 1017 /* Clear command */ 1018 cmd->raw_cmd[0] = 0; 1019 cmd->raw_cmd[1] = 0; 1020 cmd->raw_cmd[2] = 0; 1021 cmd->raw_cmd[3] = 0; 1022 1023 return cmd; 1024 } 1025 1026 static struct its_cmd_block *its_post_commands(struct its_node *its) 1027 { 1028 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 1029 1030 writel_relaxed(wr, its->base + GITS_CWRITER); 1031 1032 return its->cmd_write; 1033 } 1034 1035 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 1036 { 1037 /* 1038 * Make sure the commands written to memory are observable by 1039 * the ITS. 1040 */ 1041 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 1042 gic_flush_dcache_to_poc(cmd, sizeof(*cmd)); 1043 else 1044 dsb(ishst); 1045 } 1046 1047 static int its_wait_for_range_completion(struct its_node *its, 1048 u64 prev_idx, 1049 struct its_cmd_block *to) 1050 { 1051 u64 rd_idx, to_idx, linear_idx; 1052 u32 count = 1000000; /* 1s! */ 1053 1054 /* Linearize to_idx if the command set has wrapped around */ 1055 to_idx = its_cmd_ptr_to_offset(its, to); 1056 if (to_idx < prev_idx) 1057 to_idx += ITS_CMD_QUEUE_SZ; 1058 1059 linear_idx = prev_idx; 1060 1061 while (1) { 1062 s64 delta; 1063 1064 rd_idx = readl_relaxed(its->base + GITS_CREADR); 1065 1066 /* 1067 * Compute the read pointer progress, taking the 1068 * potential wrap-around into account. 1069 */ 1070 delta = rd_idx - prev_idx; 1071 if (rd_idx < prev_idx) 1072 delta += ITS_CMD_QUEUE_SZ; 1073 1074 linear_idx += delta; 1075 if (linear_idx >= to_idx) 1076 break; 1077 1078 count--; 1079 if (!count) { 1080 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n", 1081 to_idx, linear_idx); 1082 return -1; 1083 } 1084 prev_idx = rd_idx; 1085 cpu_relax(); 1086 udelay(1); 1087 } 1088 1089 return 0; 1090 } 1091 1092 /* Warning, macro hell follows */ 1093 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \ 1094 void name(struct its_node *its, \ 1095 buildtype builder, \ 1096 struct its_cmd_desc *desc) \ 1097 { \ 1098 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \ 1099 synctype *sync_obj; \ 1100 unsigned long flags; \ 1101 u64 rd_idx; \ 1102 \ 1103 raw_spin_lock_irqsave(&its->lock, flags); \ 1104 \ 1105 cmd = its_allocate_entry(its); \ 1106 if (!cmd) { /* We're soooooo screewed... */ \ 1107 raw_spin_unlock_irqrestore(&its->lock, flags); \ 1108 return; \ 1109 } \ 1110 sync_obj = builder(its, cmd, desc); \ 1111 its_flush_cmd(its, cmd); \ 1112 \ 1113 if (sync_obj) { \ 1114 sync_cmd = its_allocate_entry(its); \ 1115 if (!sync_cmd) \ 1116 goto post; \ 1117 \ 1118 buildfn(its, sync_cmd, sync_obj); \ 1119 its_flush_cmd(its, sync_cmd); \ 1120 } \ 1121 \ 1122 post: \ 1123 rd_idx = readl_relaxed(its->base + GITS_CREADR); \ 1124 next_cmd = its_post_commands(its); \ 1125 raw_spin_unlock_irqrestore(&its->lock, flags); \ 1126 \ 1127 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \ 1128 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \ 1129 } 1130 1131 static void its_build_sync_cmd(struct its_node *its, 1132 struct its_cmd_block *sync_cmd, 1133 struct its_collection *sync_col) 1134 { 1135 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 1136 its_encode_target(sync_cmd, sync_col->target_address); 1137 1138 its_fixup_cmd(sync_cmd); 1139 } 1140 1141 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t, 1142 struct its_collection, its_build_sync_cmd) 1143 1144 static void its_build_vsync_cmd(struct its_node *its, 1145 struct its_cmd_block *sync_cmd, 1146 struct its_vpe *sync_vpe) 1147 { 1148 its_encode_cmd(sync_cmd, GITS_CMD_VSYNC); 1149 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); 1150 1151 its_fixup_cmd(sync_cmd); 1152 } 1153 1154 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t, 1155 struct its_vpe, its_build_vsync_cmd) 1156 1157 static void its_send_int(struct its_device *dev, u32 event_id) 1158 { 1159 struct its_cmd_desc desc; 1160 1161 desc.its_int_cmd.dev = dev; 1162 desc.its_int_cmd.event_id = event_id; 1163 1164 its_send_single_command(dev->its, its_build_int_cmd, &desc); 1165 } 1166 1167 static void its_send_clear(struct its_device *dev, u32 event_id) 1168 { 1169 struct its_cmd_desc desc; 1170 1171 desc.its_clear_cmd.dev = dev; 1172 desc.its_clear_cmd.event_id = event_id; 1173 1174 its_send_single_command(dev->its, its_build_clear_cmd, &desc); 1175 } 1176 1177 static void its_send_inv(struct its_device *dev, u32 event_id) 1178 { 1179 struct its_cmd_desc desc; 1180 1181 desc.its_inv_cmd.dev = dev; 1182 desc.its_inv_cmd.event_id = event_id; 1183 1184 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 1185 } 1186 1187 static void its_send_mapd(struct its_device *dev, int valid) 1188 { 1189 struct its_cmd_desc desc; 1190 1191 desc.its_mapd_cmd.dev = dev; 1192 desc.its_mapd_cmd.valid = !!valid; 1193 1194 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 1195 } 1196 1197 static void its_send_mapc(struct its_node *its, struct its_collection *col, 1198 int valid) 1199 { 1200 struct its_cmd_desc desc; 1201 1202 desc.its_mapc_cmd.col = col; 1203 desc.its_mapc_cmd.valid = !!valid; 1204 1205 its_send_single_command(its, its_build_mapc_cmd, &desc); 1206 } 1207 1208 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id) 1209 { 1210 struct its_cmd_desc desc; 1211 1212 desc.its_mapti_cmd.dev = dev; 1213 desc.its_mapti_cmd.phys_id = irq_id; 1214 desc.its_mapti_cmd.event_id = id; 1215 1216 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); 1217 } 1218 1219 static void its_send_movi(struct its_device *dev, 1220 struct its_collection *col, u32 id) 1221 { 1222 struct its_cmd_desc desc; 1223 1224 desc.its_movi_cmd.dev = dev; 1225 desc.its_movi_cmd.col = col; 1226 desc.its_movi_cmd.event_id = id; 1227 1228 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 1229 } 1230 1231 static void its_send_discard(struct its_device *dev, u32 id) 1232 { 1233 struct its_cmd_desc desc; 1234 1235 desc.its_discard_cmd.dev = dev; 1236 desc.its_discard_cmd.event_id = id; 1237 1238 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 1239 } 1240 1241 static void its_send_invall(struct its_node *its, struct its_collection *col) 1242 { 1243 struct its_cmd_desc desc; 1244 1245 desc.its_invall_cmd.col = col; 1246 1247 its_send_single_command(its, its_build_invall_cmd, &desc); 1248 } 1249 1250 static void its_send_vmapti(struct its_device *dev, u32 id) 1251 { 1252 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); 1253 struct its_cmd_desc desc; 1254 1255 desc.its_vmapti_cmd.vpe = map->vpe; 1256 desc.its_vmapti_cmd.dev = dev; 1257 desc.its_vmapti_cmd.virt_id = map->vintid; 1258 desc.its_vmapti_cmd.event_id = id; 1259 desc.its_vmapti_cmd.db_enabled = map->db_enabled; 1260 1261 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); 1262 } 1263 1264 static void its_send_vmovi(struct its_device *dev, u32 id) 1265 { 1266 struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id); 1267 struct its_cmd_desc desc; 1268 1269 desc.its_vmovi_cmd.vpe = map->vpe; 1270 desc.its_vmovi_cmd.dev = dev; 1271 desc.its_vmovi_cmd.event_id = id; 1272 desc.its_vmovi_cmd.db_enabled = map->db_enabled; 1273 1274 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); 1275 } 1276 1277 static void its_send_vmapp(struct its_node *its, 1278 struct its_vpe *vpe, bool valid) 1279 { 1280 struct its_cmd_desc desc; 1281 1282 desc.its_vmapp_cmd.vpe = vpe; 1283 desc.its_vmapp_cmd.valid = valid; 1284 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; 1285 1286 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); 1287 } 1288 1289 static void its_send_vmovp(struct its_vpe *vpe) 1290 { 1291 struct its_cmd_desc desc = {}; 1292 struct its_node *its; 1293 unsigned long flags; 1294 int col_id = vpe->col_idx; 1295 1296 desc.its_vmovp_cmd.vpe = vpe; 1297 1298 if (!its_list_map) { 1299 its = list_first_entry(&its_nodes, struct its_node, entry); 1300 desc.its_vmovp_cmd.col = &its->collections[col_id]; 1301 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 1302 return; 1303 } 1304 1305 /* 1306 * Yet another marvel of the architecture. If using the 1307 * its_list "feature", we need to make sure that all ITSs 1308 * receive all VMOVP commands in the same order. The only way 1309 * to guarantee this is to make vmovp a serialization point. 1310 * 1311 * Wall <-- Head. 1312 */ 1313 raw_spin_lock_irqsave(&vmovp_lock, flags); 1314 1315 desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; 1316 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm); 1317 1318 /* Emit VMOVPs */ 1319 list_for_each_entry(its, &its_nodes, entry) { 1320 if (!is_v4(its)) 1321 continue; 1322 1323 if (!require_its_list_vmovp(vpe->its_vm, its)) 1324 continue; 1325 1326 desc.its_vmovp_cmd.col = &its->collections[col_id]; 1327 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); 1328 } 1329 1330 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1331 } 1332 1333 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) 1334 { 1335 struct its_cmd_desc desc; 1336 1337 desc.its_vinvall_cmd.vpe = vpe; 1338 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); 1339 } 1340 1341 static void its_send_vinv(struct its_device *dev, u32 event_id) 1342 { 1343 struct its_cmd_desc desc; 1344 1345 /* 1346 * There is no real VINV command. This is just a normal INV, 1347 * with a VSYNC instead of a SYNC. 1348 */ 1349 desc.its_inv_cmd.dev = dev; 1350 desc.its_inv_cmd.event_id = event_id; 1351 1352 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); 1353 } 1354 1355 static void its_send_vint(struct its_device *dev, u32 event_id) 1356 { 1357 struct its_cmd_desc desc; 1358 1359 /* 1360 * There is no real VINT command. This is just a normal INT, 1361 * with a VSYNC instead of a SYNC. 1362 */ 1363 desc.its_int_cmd.dev = dev; 1364 desc.its_int_cmd.event_id = event_id; 1365 1366 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); 1367 } 1368 1369 static void its_send_vclear(struct its_device *dev, u32 event_id) 1370 { 1371 struct its_cmd_desc desc; 1372 1373 /* 1374 * There is no real VCLEAR command. This is just a normal CLEAR, 1375 * with a VSYNC instead of a SYNC. 1376 */ 1377 desc.its_clear_cmd.dev = dev; 1378 desc.its_clear_cmd.event_id = event_id; 1379 1380 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); 1381 } 1382 1383 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) 1384 { 1385 struct its_cmd_desc desc; 1386 1387 desc.its_invdb_cmd.vpe = vpe; 1388 its_send_single_vcommand(its, its_build_invdb_cmd, &desc); 1389 } 1390 1391 /* 1392 * irqchip functions - assumes MSI, mostly. 1393 */ 1394 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set) 1395 { 1396 struct its_vlpi_map *map = get_vlpi_map(d); 1397 irq_hw_number_t hwirq; 1398 void *va; 1399 u8 *cfg; 1400 1401 if (map) { 1402 va = page_address(map->vm->vprop_page); 1403 hwirq = map->vintid; 1404 1405 /* Remember the updated property */ 1406 map->properties &= ~clr; 1407 map->properties |= set | LPI_PROP_GROUP1; 1408 } else { 1409 va = gic_rdists->prop_table_va; 1410 hwirq = d->hwirq; 1411 } 1412 1413 cfg = va + hwirq - 8192; 1414 *cfg &= ~clr; 1415 *cfg |= set | LPI_PROP_GROUP1; 1416 1417 /* 1418 * Make the above write visible to the redistributors. 1419 * And yes, we're flushing exactly: One. Single. Byte. 1420 * Humpf... 1421 */ 1422 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 1423 gic_flush_dcache_to_poc(cfg, sizeof(*cfg)); 1424 else 1425 dsb(ishst); 1426 } 1427 1428 static void wait_for_syncr(void __iomem *rdbase) 1429 { 1430 while (readl_relaxed(rdbase + GICR_SYNCR) & 1) 1431 cpu_relax(); 1432 } 1433 1434 static void direct_lpi_inv(struct irq_data *d) 1435 { 1436 struct its_vlpi_map *map = get_vlpi_map(d); 1437 void __iomem *rdbase; 1438 unsigned long flags; 1439 u64 val; 1440 int cpu; 1441 1442 if (map) { 1443 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1444 1445 WARN_ON(!is_v4_1(its_dev->its)); 1446 1447 val = GICR_INVLPIR_V; 1448 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); 1449 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); 1450 } else { 1451 val = d->hwirq; 1452 } 1453 1454 /* Target the redistributor this LPI is currently routed to */ 1455 cpu = irq_to_cpuid_lock(d, &flags); 1456 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 1457 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; 1458 gic_write_lpir(val, rdbase + GICR_INVLPIR); 1459 1460 wait_for_syncr(rdbase); 1461 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 1462 irq_to_cpuid_unlock(d, flags); 1463 } 1464 1465 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set) 1466 { 1467 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1468 1469 lpi_write_config(d, clr, set); 1470 if (gic_rdists->has_direct_lpi && 1471 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) 1472 direct_lpi_inv(d); 1473 else if (!irqd_is_forwarded_to_vcpu(d)) 1474 its_send_inv(its_dev, its_get_event_id(d)); 1475 else 1476 its_send_vinv(its_dev, its_get_event_id(d)); 1477 } 1478 1479 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable) 1480 { 1481 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1482 u32 event = its_get_event_id(d); 1483 struct its_vlpi_map *map; 1484 1485 /* 1486 * GICv4.1 does away with the per-LPI nonsense, nothing to do 1487 * here. 1488 */ 1489 if (is_v4_1(its_dev->its)) 1490 return; 1491 1492 map = dev_event_to_vlpi_map(its_dev, event); 1493 1494 if (map->db_enabled == enable) 1495 return; 1496 1497 map->db_enabled = enable; 1498 1499 /* 1500 * More fun with the architecture: 1501 * 1502 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI 1503 * value or to 1023, depending on the enable bit. But that 1504 * would be issuing a mapping for an /existing/ DevID+EventID 1505 * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI 1506 * to the /same/ vPE, using this opportunity to adjust the 1507 * doorbell. Mouahahahaha. We loves it, Precious. 1508 */ 1509 its_send_vmovi(its_dev, event); 1510 } 1511 1512 static void its_mask_irq(struct irq_data *d) 1513 { 1514 if (irqd_is_forwarded_to_vcpu(d)) 1515 its_vlpi_set_doorbell(d, false); 1516 1517 lpi_update_config(d, LPI_PROP_ENABLED, 0); 1518 } 1519 1520 static void its_unmask_irq(struct irq_data *d) 1521 { 1522 if (irqd_is_forwarded_to_vcpu(d)) 1523 its_vlpi_set_doorbell(d, true); 1524 1525 lpi_update_config(d, 0, LPI_PROP_ENABLED); 1526 } 1527 1528 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu) 1529 { 1530 if (irqd_affinity_is_managed(d)) 1531 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1532 1533 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1534 } 1535 1536 static void its_inc_lpi_count(struct irq_data *d, int cpu) 1537 { 1538 if (irqd_affinity_is_managed(d)) 1539 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1540 else 1541 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1542 } 1543 1544 static void its_dec_lpi_count(struct irq_data *d, int cpu) 1545 { 1546 if (irqd_affinity_is_managed(d)) 1547 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); 1548 else 1549 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); 1550 } 1551 1552 static unsigned int cpumask_pick_least_loaded(struct irq_data *d, 1553 const struct cpumask *cpu_mask) 1554 { 1555 unsigned int cpu = nr_cpu_ids, tmp; 1556 int count = S32_MAX; 1557 1558 for_each_cpu(tmp, cpu_mask) { 1559 int this_count = its_read_lpi_count(d, tmp); 1560 if (this_count < count) { 1561 cpu = tmp; 1562 count = this_count; 1563 } 1564 } 1565 1566 return cpu; 1567 } 1568 1569 /* 1570 * As suggested by Thomas Gleixner in: 1571 * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de 1572 */ 1573 static int its_select_cpu(struct irq_data *d, 1574 const struct cpumask *aff_mask) 1575 { 1576 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1577 static DEFINE_RAW_SPINLOCK(tmpmask_lock); 1578 static struct cpumask __tmpmask; 1579 struct cpumask *tmpmask; 1580 unsigned long flags; 1581 int cpu, node; 1582 node = its_dev->its->numa_node; 1583 tmpmask = &__tmpmask; 1584 1585 raw_spin_lock_irqsave(&tmpmask_lock, flags); 1586 1587 if (!irqd_affinity_is_managed(d)) { 1588 /* First try the NUMA node */ 1589 if (node != NUMA_NO_NODE) { 1590 /* 1591 * Try the intersection of the affinity mask and the 1592 * node mask (and the online mask, just to be safe). 1593 */ 1594 cpumask_and(tmpmask, cpumask_of_node(node), aff_mask); 1595 cpumask_and(tmpmask, tmpmask, cpu_online_mask); 1596 1597 /* 1598 * Ideally, we would check if the mask is empty, and 1599 * try again on the full node here. 1600 * 1601 * But it turns out that the way ACPI describes the 1602 * affinity for ITSs only deals about memory, and 1603 * not target CPUs, so it cannot describe a single 1604 * ITS placed next to two NUMA nodes. 1605 * 1606 * Instead, just fallback on the online mask. This 1607 * diverges from Thomas' suggestion above. 1608 */ 1609 cpu = cpumask_pick_least_loaded(d, tmpmask); 1610 if (cpu < nr_cpu_ids) 1611 goto out; 1612 1613 /* If we can't cross sockets, give up */ 1614 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) 1615 goto out; 1616 1617 /* If the above failed, expand the search */ 1618 } 1619 1620 /* Try the intersection of the affinity and online masks */ 1621 cpumask_and(tmpmask, aff_mask, cpu_online_mask); 1622 1623 /* If that doesn't fly, the online mask is the last resort */ 1624 if (cpumask_empty(tmpmask)) 1625 cpumask_copy(tmpmask, cpu_online_mask); 1626 1627 cpu = cpumask_pick_least_loaded(d, tmpmask); 1628 } else { 1629 cpumask_copy(tmpmask, aff_mask); 1630 1631 /* If we cannot cross sockets, limit the search to that node */ 1632 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && 1633 node != NUMA_NO_NODE) 1634 cpumask_and(tmpmask, tmpmask, cpumask_of_node(node)); 1635 1636 cpu = cpumask_pick_least_loaded(d, tmpmask); 1637 } 1638 out: 1639 raw_spin_unlock_irqrestore(&tmpmask_lock, flags); 1640 1641 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); 1642 return cpu; 1643 } 1644 1645 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 1646 bool force) 1647 { 1648 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1649 struct its_collection *target_col; 1650 u32 id = its_get_event_id(d); 1651 int cpu, prev_cpu; 1652 1653 /* A forwarded interrupt should use irq_set_vcpu_affinity */ 1654 if (irqd_is_forwarded_to_vcpu(d)) 1655 return -EINVAL; 1656 1657 prev_cpu = its_dev->event_map.col_map[id]; 1658 its_dec_lpi_count(d, prev_cpu); 1659 1660 if (!force) 1661 cpu = its_select_cpu(d, mask_val); 1662 else 1663 cpu = cpumask_pick_least_loaded(d, mask_val); 1664 1665 if (cpu < 0 || cpu >= nr_cpu_ids) 1666 goto err; 1667 1668 /* don't set the affinity when the target cpu is same as current one */ 1669 if (cpu != prev_cpu) { 1670 target_col = &its_dev->its->collections[cpu]; 1671 its_send_movi(its_dev, target_col, id); 1672 its_dev->event_map.col_map[id] = cpu; 1673 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 1674 } 1675 1676 its_inc_lpi_count(d, cpu); 1677 1678 return IRQ_SET_MASK_OK_DONE; 1679 1680 err: 1681 its_inc_lpi_count(d, prev_cpu); 1682 return -EINVAL; 1683 } 1684 1685 static u64 its_irq_get_msi_base(struct its_device *its_dev) 1686 { 1687 struct its_node *its = its_dev->its; 1688 1689 return its->phys_base + GITS_TRANSLATER; 1690 } 1691 1692 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 1693 { 1694 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1695 struct its_node *its; 1696 u64 addr; 1697 1698 its = its_dev->its; 1699 addr = its->get_msi_base(its_dev); 1700 1701 msg->address_lo = lower_32_bits(addr); 1702 msg->address_hi = upper_32_bits(addr); 1703 msg->data = its_get_event_id(d); 1704 1705 iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg); 1706 } 1707 1708 static int its_irq_set_irqchip_state(struct irq_data *d, 1709 enum irqchip_irq_state which, 1710 bool state) 1711 { 1712 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1713 u32 event = its_get_event_id(d); 1714 1715 if (which != IRQCHIP_STATE_PENDING) 1716 return -EINVAL; 1717 1718 if (irqd_is_forwarded_to_vcpu(d)) { 1719 if (state) 1720 its_send_vint(its_dev, event); 1721 else 1722 its_send_vclear(its_dev, event); 1723 } else { 1724 if (state) 1725 its_send_int(its_dev, event); 1726 else 1727 its_send_clear(its_dev, event); 1728 } 1729 1730 return 0; 1731 } 1732 1733 static int its_irq_retrigger(struct irq_data *d) 1734 { 1735 return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true); 1736 } 1737 1738 /* 1739 * Two favourable cases: 1740 * 1741 * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times 1742 * for vSGI delivery 1743 * 1744 * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough 1745 * and we're better off mapping all VPEs always 1746 * 1747 * If neither (a) nor (b) is true, then we map vPEs on demand. 1748 * 1749 */ 1750 static bool gic_requires_eager_mapping(void) 1751 { 1752 if (!its_list_map || gic_rdists->has_rvpeid) 1753 return true; 1754 1755 return false; 1756 } 1757 1758 static void its_map_vm(struct its_node *its, struct its_vm *vm) 1759 { 1760 unsigned long flags; 1761 1762 if (gic_requires_eager_mapping()) 1763 return; 1764 1765 raw_spin_lock_irqsave(&vmovp_lock, flags); 1766 1767 /* 1768 * If the VM wasn't mapped yet, iterate over the vpes and get 1769 * them mapped now. 1770 */ 1771 vm->vlpi_count[its->list_nr]++; 1772 1773 if (vm->vlpi_count[its->list_nr] == 1) { 1774 int i; 1775 1776 for (i = 0; i < vm->nr_vpes; i++) { 1777 struct its_vpe *vpe = vm->vpes[i]; 1778 struct irq_data *d = irq_get_irq_data(vpe->irq); 1779 1780 /* Map the VPE to the first possible CPU */ 1781 vpe->col_idx = cpumask_first(cpu_online_mask); 1782 its_send_vmapp(its, vpe, true); 1783 its_send_vinvall(its, vpe); 1784 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 1785 } 1786 } 1787 1788 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1789 } 1790 1791 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) 1792 { 1793 unsigned long flags; 1794 1795 /* Not using the ITS list? Everything is always mapped. */ 1796 if (gic_requires_eager_mapping()) 1797 return; 1798 1799 raw_spin_lock_irqsave(&vmovp_lock, flags); 1800 1801 if (!--vm->vlpi_count[its->list_nr]) { 1802 int i; 1803 1804 for (i = 0; i < vm->nr_vpes; i++) 1805 its_send_vmapp(its, vm->vpes[i], false); 1806 } 1807 1808 raw_spin_unlock_irqrestore(&vmovp_lock, flags); 1809 } 1810 1811 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) 1812 { 1813 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1814 u32 event = its_get_event_id(d); 1815 int ret = 0; 1816 1817 if (!info->map) 1818 return -EINVAL; 1819 1820 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1821 1822 if (!its_dev->event_map.vm) { 1823 struct its_vlpi_map *maps; 1824 1825 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), 1826 GFP_ATOMIC); 1827 if (!maps) { 1828 ret = -ENOMEM; 1829 goto out; 1830 } 1831 1832 its_dev->event_map.vm = info->map->vm; 1833 its_dev->event_map.vlpi_maps = maps; 1834 } else if (its_dev->event_map.vm != info->map->vm) { 1835 ret = -EINVAL; 1836 goto out; 1837 } 1838 1839 /* Get our private copy of the mapping information */ 1840 its_dev->event_map.vlpi_maps[event] = *info->map; 1841 1842 if (irqd_is_forwarded_to_vcpu(d)) { 1843 /* Already mapped, move it around */ 1844 its_send_vmovi(its_dev, event); 1845 } else { 1846 /* Ensure all the VPEs are mapped on this ITS */ 1847 its_map_vm(its_dev->its, info->map->vm); 1848 1849 /* 1850 * Flag the interrupt as forwarded so that we can 1851 * start poking the virtual property table. 1852 */ 1853 irqd_set_forwarded_to_vcpu(d); 1854 1855 /* Write out the property to the prop table */ 1856 lpi_write_config(d, 0xff, info->map->properties); 1857 1858 /* Drop the physical mapping */ 1859 its_send_discard(its_dev, event); 1860 1861 /* and install the virtual one */ 1862 its_send_vmapti(its_dev, event); 1863 1864 /* Increment the number of VLPIs */ 1865 its_dev->event_map.nr_vlpis++; 1866 } 1867 1868 out: 1869 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1870 return ret; 1871 } 1872 1873 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) 1874 { 1875 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1876 struct its_vlpi_map *map; 1877 int ret = 0; 1878 1879 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1880 1881 map = get_vlpi_map(d); 1882 1883 if (!its_dev->event_map.vm || !map) { 1884 ret = -EINVAL; 1885 goto out; 1886 } 1887 1888 /* Copy our mapping information to the incoming request */ 1889 *info->map = *map; 1890 1891 out: 1892 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1893 return ret; 1894 } 1895 1896 static int its_vlpi_unmap(struct irq_data *d) 1897 { 1898 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1899 u32 event = its_get_event_id(d); 1900 int ret = 0; 1901 1902 raw_spin_lock(&its_dev->event_map.vlpi_lock); 1903 1904 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { 1905 ret = -EINVAL; 1906 goto out; 1907 } 1908 1909 /* Drop the virtual mapping */ 1910 its_send_discard(its_dev, event); 1911 1912 /* and restore the physical one */ 1913 irqd_clr_forwarded_to_vcpu(d); 1914 its_send_mapti(its_dev, d->hwirq, event); 1915 lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | 1916 LPI_PROP_ENABLED | 1917 LPI_PROP_GROUP1)); 1918 1919 /* Potentially unmap the VM from this ITS */ 1920 its_unmap_vm(its_dev->its, its_dev->event_map.vm); 1921 1922 /* 1923 * Drop the refcount and make the device available again if 1924 * this was the last VLPI. 1925 */ 1926 if (!--its_dev->event_map.nr_vlpis) { 1927 its_dev->event_map.vm = NULL; 1928 kfree(its_dev->event_map.vlpi_maps); 1929 } 1930 1931 out: 1932 raw_spin_unlock(&its_dev->event_map.vlpi_lock); 1933 return ret; 1934 } 1935 1936 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) 1937 { 1938 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1939 1940 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) 1941 return -EINVAL; 1942 1943 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) 1944 lpi_update_config(d, 0xff, info->config); 1945 else 1946 lpi_write_config(d, 0xff, info->config); 1947 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); 1948 1949 return 0; 1950 } 1951 1952 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 1953 { 1954 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1955 struct its_cmd_info *info = vcpu_info; 1956 1957 /* Need a v4 ITS */ 1958 if (!is_v4(its_dev->its)) 1959 return -EINVAL; 1960 1961 /* Unmap request? */ 1962 if (!info) 1963 return its_vlpi_unmap(d); 1964 1965 switch (info->cmd_type) { 1966 case MAP_VLPI: 1967 return its_vlpi_map(d, info); 1968 1969 case GET_VLPI: 1970 return its_vlpi_get(d, info); 1971 1972 case PROP_UPDATE_VLPI: 1973 case PROP_UPDATE_AND_INV_VLPI: 1974 return its_vlpi_prop_update(d, info); 1975 1976 default: 1977 return -EINVAL; 1978 } 1979 } 1980 1981 static struct irq_chip its_irq_chip = { 1982 .name = "ITS", 1983 .irq_mask = its_mask_irq, 1984 .irq_unmask = its_unmask_irq, 1985 .irq_eoi = irq_chip_eoi_parent, 1986 .irq_set_affinity = its_set_affinity, 1987 .irq_compose_msi_msg = its_irq_compose_msi_msg, 1988 .irq_set_irqchip_state = its_irq_set_irqchip_state, 1989 .irq_retrigger = its_irq_retrigger, 1990 .irq_set_vcpu_affinity = its_irq_set_vcpu_affinity, 1991 }; 1992 1993 1994 /* 1995 * How we allocate LPIs: 1996 * 1997 * lpi_range_list contains ranges of LPIs that are to available to 1998 * allocate from. To allocate LPIs, just pick the first range that 1999 * fits the required allocation, and reduce it by the required 2000 * amount. Once empty, remove the range from the list. 2001 * 2002 * To free a range of LPIs, add a free range to the list, sort it and 2003 * merge the result if the new range happens to be adjacent to an 2004 * already free block. 2005 * 2006 * The consequence of the above is that allocation is cost is low, but 2007 * freeing is expensive. We assumes that freeing rarely occurs. 2008 */ 2009 #define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */ 2010 2011 static DEFINE_MUTEX(lpi_range_lock); 2012 static LIST_HEAD(lpi_range_list); 2013 2014 struct lpi_range { 2015 struct list_head entry; 2016 u32 base_id; 2017 u32 span; 2018 }; 2019 2020 static struct lpi_range *mk_lpi_range(u32 base, u32 span) 2021 { 2022 struct lpi_range *range; 2023 2024 range = kmalloc(sizeof(*range), GFP_KERNEL); 2025 if (range) { 2026 range->base_id = base; 2027 range->span = span; 2028 } 2029 2030 return range; 2031 } 2032 2033 static int alloc_lpi_range(u32 nr_lpis, u32 *base) 2034 { 2035 struct lpi_range *range, *tmp; 2036 int err = -ENOSPC; 2037 2038 mutex_lock(&lpi_range_lock); 2039 2040 list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) { 2041 if (range->span >= nr_lpis) { 2042 *base = range->base_id; 2043 range->base_id += nr_lpis; 2044 range->span -= nr_lpis; 2045 2046 if (range->span == 0) { 2047 list_del(&range->entry); 2048 kfree(range); 2049 } 2050 2051 err = 0; 2052 break; 2053 } 2054 } 2055 2056 mutex_unlock(&lpi_range_lock); 2057 2058 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis); 2059 return err; 2060 } 2061 2062 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b) 2063 { 2064 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list) 2065 return; 2066 if (a->base_id + a->span != b->base_id) 2067 return; 2068 b->base_id = a->base_id; 2069 b->span += a->span; 2070 list_del(&a->entry); 2071 kfree(a); 2072 } 2073 2074 static int free_lpi_range(u32 base, u32 nr_lpis) 2075 { 2076 struct lpi_range *new, *old; 2077 2078 new = mk_lpi_range(base, nr_lpis); 2079 if (!new) 2080 return -ENOMEM; 2081 2082 mutex_lock(&lpi_range_lock); 2083 2084 list_for_each_entry_reverse(old, &lpi_range_list, entry) { 2085 if (old->base_id < base) 2086 break; 2087 } 2088 /* 2089 * old is the last element with ->base_id smaller than base, 2090 * so new goes right after it. If there are no elements with 2091 * ->base_id smaller than base, &old->entry ends up pointing 2092 * at the head of the list, and inserting new it the start of 2093 * the list is the right thing to do in that case as well. 2094 */ 2095 list_add(&new->entry, &old->entry); 2096 /* 2097 * Now check if we can merge with the preceding and/or 2098 * following ranges. 2099 */ 2100 merge_lpi_ranges(old, new); 2101 merge_lpi_ranges(new, list_next_entry(new, entry)); 2102 2103 mutex_unlock(&lpi_range_lock); 2104 return 0; 2105 } 2106 2107 static int __init its_lpi_init(u32 id_bits) 2108 { 2109 u32 lpis = (1UL << id_bits) - 8192; 2110 u32 numlpis; 2111 int err; 2112 2113 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer); 2114 2115 if (numlpis > 2 && !WARN_ON(numlpis > lpis)) { 2116 lpis = numlpis; 2117 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n", 2118 lpis); 2119 } 2120 2121 /* 2122 * Initializing the allocator is just the same as freeing the 2123 * full range of LPIs. 2124 */ 2125 err = free_lpi_range(8192, lpis); 2126 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis); 2127 return err; 2128 } 2129 2130 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids) 2131 { 2132 unsigned long *bitmap = NULL; 2133 int err = 0; 2134 2135 do { 2136 err = alloc_lpi_range(nr_irqs, base); 2137 if (!err) 2138 break; 2139 2140 nr_irqs /= 2; 2141 } while (nr_irqs > 0); 2142 2143 if (!nr_irqs) 2144 err = -ENOSPC; 2145 2146 if (err) 2147 goto out; 2148 2149 bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC); 2150 if (!bitmap) 2151 goto out; 2152 2153 *nr_ids = nr_irqs; 2154 2155 out: 2156 if (!bitmap) 2157 *base = *nr_ids = 0; 2158 2159 return bitmap; 2160 } 2161 2162 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids) 2163 { 2164 WARN_ON(free_lpi_range(base, nr_ids)); 2165 bitmap_free(bitmap); 2166 } 2167 2168 static void gic_reset_prop_table(void *va) 2169 { 2170 /* Priority 0xa0, Group-1, disabled */ 2171 memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ); 2172 2173 /* Make sure the GIC will observe the written configuration */ 2174 gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ); 2175 } 2176 2177 static struct page *its_allocate_prop_table(gfp_t gfp_flags) 2178 { 2179 struct page *prop_page; 2180 2181 prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); 2182 if (!prop_page) 2183 return NULL; 2184 2185 gic_reset_prop_table(page_address(prop_page)); 2186 2187 return prop_page; 2188 } 2189 2190 static void its_free_prop_table(struct page *prop_page) 2191 { 2192 free_pages((unsigned long)page_address(prop_page), 2193 get_order(LPI_PROPBASE_SZ)); 2194 } 2195 2196 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) 2197 { 2198 phys_addr_t start, end, addr_end; 2199 u64 i; 2200 2201 /* 2202 * We don't bother checking for a kdump kernel as by 2203 * construction, the LPI tables are out of this kernel's 2204 * memory map. 2205 */ 2206 if (is_kdump_kernel()) 2207 return true; 2208 2209 addr_end = addr + size - 1; 2210 2211 for_each_reserved_mem_range(i, &start, &end) { 2212 if (addr >= start && addr_end <= end) 2213 return true; 2214 } 2215 2216 /* Not found, not a good sign... */ 2217 pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n", 2218 &addr, &addr_end); 2219 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 2220 return false; 2221 } 2222 2223 static int gic_reserve_range(phys_addr_t addr, unsigned long size) 2224 { 2225 if (efi_enabled(EFI_CONFIG_TABLES)) 2226 return efi_mem_reserve_persistent(addr, size); 2227 2228 return 0; 2229 } 2230 2231 static int __init its_setup_lpi_prop_table(void) 2232 { 2233 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) { 2234 u64 val; 2235 2236 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 2237 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1; 2238 2239 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12); 2240 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa, 2241 LPI_PROPBASE_SZ, 2242 MEMREMAP_WB); 2243 gic_reset_prop_table(gic_rdists->prop_table_va); 2244 } else { 2245 struct page *page; 2246 2247 lpi_id_bits = min_t(u32, 2248 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer), 2249 ITS_MAX_LPI_NRBITS); 2250 page = its_allocate_prop_table(GFP_NOWAIT); 2251 if (!page) { 2252 pr_err("Failed to allocate PROPBASE\n"); 2253 return -ENOMEM; 2254 } 2255 2256 gic_rdists->prop_table_pa = page_to_phys(page); 2257 gic_rdists->prop_table_va = page_address(page); 2258 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa, 2259 LPI_PROPBASE_SZ)); 2260 } 2261 2262 pr_info("GICv3: using LPI property table @%pa\n", 2263 &gic_rdists->prop_table_pa); 2264 2265 return its_lpi_init(lpi_id_bits); 2266 } 2267 2268 static const char *its_base_type_string[] = { 2269 [GITS_BASER_TYPE_DEVICE] = "Devices", 2270 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 2271 [GITS_BASER_TYPE_RESERVED3] = "Reserved (3)", 2272 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 2273 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 2274 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 2275 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 2276 }; 2277 2278 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) 2279 { 2280 u32 idx = baser - its->tables; 2281 2282 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); 2283 } 2284 2285 static void its_write_baser(struct its_node *its, struct its_baser *baser, 2286 u64 val) 2287 { 2288 u32 idx = baser - its->tables; 2289 2290 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); 2291 baser->val = its_read_baser(its, baser); 2292 } 2293 2294 static int its_setup_baser(struct its_node *its, struct its_baser *baser, 2295 u64 cache, u64 shr, u32 order, bool indirect) 2296 { 2297 u64 val = its_read_baser(its, baser); 2298 u64 esz = GITS_BASER_ENTRY_SIZE(val); 2299 u64 type = GITS_BASER_TYPE(val); 2300 u64 baser_phys, tmp; 2301 u32 alloc_pages, psz; 2302 struct page *page; 2303 void *base; 2304 2305 psz = baser->psz; 2306 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz); 2307 if (alloc_pages > GITS_BASER_PAGES_MAX) { 2308 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", 2309 &its->phys_base, its_base_type_string[type], 2310 alloc_pages, GITS_BASER_PAGES_MAX); 2311 alloc_pages = GITS_BASER_PAGES_MAX; 2312 order = get_order(GITS_BASER_PAGES_MAX * psz); 2313 } 2314 2315 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); 2316 if (!page) 2317 return -ENOMEM; 2318 2319 base = (void *)page_address(page); 2320 baser_phys = virt_to_phys(base); 2321 2322 /* Check if the physical address of the memory is above 48bits */ 2323 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) { 2324 2325 /* 52bit PA is supported only when PageSize=64K */ 2326 if (psz != SZ_64K) { 2327 pr_err("ITS: no 52bit PA support when psz=%d\n", psz); 2328 free_pages((unsigned long)base, order); 2329 return -ENXIO; 2330 } 2331 2332 /* Convert 52bit PA to 48bit field */ 2333 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys); 2334 } 2335 2336 retry_baser: 2337 val = (baser_phys | 2338 (type << GITS_BASER_TYPE_SHIFT) | 2339 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 2340 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | 2341 cache | 2342 shr | 2343 GITS_BASER_VALID); 2344 2345 val |= indirect ? GITS_BASER_INDIRECT : 0x0; 2346 2347 switch (psz) { 2348 case SZ_4K: 2349 val |= GITS_BASER_PAGE_SIZE_4K; 2350 break; 2351 case SZ_16K: 2352 val |= GITS_BASER_PAGE_SIZE_16K; 2353 break; 2354 case SZ_64K: 2355 val |= GITS_BASER_PAGE_SIZE_64K; 2356 break; 2357 } 2358 2359 its_write_baser(its, baser, val); 2360 tmp = baser->val; 2361 2362 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 2363 /* 2364 * Shareability didn't stick. Just use 2365 * whatever the read reported, which is likely 2366 * to be the only thing this redistributor 2367 * supports. If that's zero, make it 2368 * non-cacheable as well. 2369 */ 2370 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 2371 if (!shr) { 2372 cache = GITS_BASER_nC; 2373 gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); 2374 } 2375 goto retry_baser; 2376 } 2377 2378 if (val != tmp) { 2379 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", 2380 &its->phys_base, its_base_type_string[type], 2381 val, tmp); 2382 free_pages((unsigned long)base, order); 2383 return -ENXIO; 2384 } 2385 2386 baser->order = order; 2387 baser->base = base; 2388 baser->psz = psz; 2389 tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz; 2390 2391 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", 2392 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), 2393 its_base_type_string[type], 2394 (unsigned long)virt_to_phys(base), 2395 indirect ? "indirect" : "flat", (int)esz, 2396 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 2397 2398 return 0; 2399 } 2400 2401 static bool its_parse_indirect_baser(struct its_node *its, 2402 struct its_baser *baser, 2403 u32 *order, u32 ids) 2404 { 2405 u64 tmp = its_read_baser(its, baser); 2406 u64 type = GITS_BASER_TYPE(tmp); 2407 u64 esz = GITS_BASER_ENTRY_SIZE(tmp); 2408 u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb; 2409 u32 new_order = *order; 2410 u32 psz = baser->psz; 2411 bool indirect = false; 2412 2413 /* No need to enable Indirection if memory requirement < (psz*2)bytes */ 2414 if ((esz << ids) > (psz * 2)) { 2415 /* 2416 * Find out whether hw supports a single or two-level table by 2417 * table by reading bit at offset '62' after writing '1' to it. 2418 */ 2419 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); 2420 indirect = !!(baser->val & GITS_BASER_INDIRECT); 2421 2422 if (indirect) { 2423 /* 2424 * The size of the lvl2 table is equal to ITS page size 2425 * which is 'psz'. For computing lvl1 table size, 2426 * subtract ID bits that sparse lvl2 table from 'ids' 2427 * which is reported by ITS hardware times lvl1 table 2428 * entry size. 2429 */ 2430 ids -= ilog2(psz / (int)esz); 2431 esz = GITS_LVL1_ENTRY_SIZE; 2432 } 2433 } 2434 2435 /* 2436 * Allocate as many entries as required to fit the 2437 * range of device IDs that the ITS can grok... The ID 2438 * space being incredibly sparse, this results in a 2439 * massive waste of memory if two-level device table 2440 * feature is not supported by hardware. 2441 */ 2442 new_order = max_t(u32, get_order(esz << ids), new_order); 2443 if (new_order >= MAX_ORDER) { 2444 new_order = MAX_ORDER - 1; 2445 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); 2446 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", 2447 &its->phys_base, its_base_type_string[type], 2448 device_ids(its), ids); 2449 } 2450 2451 *order = new_order; 2452 2453 return indirect; 2454 } 2455 2456 static u32 compute_common_aff(u64 val) 2457 { 2458 u32 aff, clpiaff; 2459 2460 aff = FIELD_GET(GICR_TYPER_AFFINITY, val); 2461 clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val); 2462 2463 return aff & ~(GENMASK(31, 0) >> (clpiaff * 8)); 2464 } 2465 2466 static u32 compute_its_aff(struct its_node *its) 2467 { 2468 u64 val; 2469 u32 svpet; 2470 2471 /* 2472 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute 2473 * the resulting affinity. We then use that to see if this match 2474 * our own affinity. 2475 */ 2476 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); 2477 val = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet); 2478 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); 2479 return compute_common_aff(val); 2480 } 2481 2482 static struct its_node *find_sibling_its(struct its_node *cur_its) 2483 { 2484 struct its_node *its; 2485 u32 aff; 2486 2487 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) 2488 return NULL; 2489 2490 aff = compute_its_aff(cur_its); 2491 2492 list_for_each_entry(its, &its_nodes, entry) { 2493 u64 baser; 2494 2495 if (!is_v4_1(its) || its == cur_its) 2496 continue; 2497 2498 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) 2499 continue; 2500 2501 if (aff != compute_its_aff(its)) 2502 continue; 2503 2504 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ 2505 baser = its->tables[2].val; 2506 if (!(baser & GITS_BASER_VALID)) 2507 continue; 2508 2509 return its; 2510 } 2511 2512 return NULL; 2513 } 2514 2515 static void its_free_tables(struct its_node *its) 2516 { 2517 int i; 2518 2519 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2520 if (its->tables[i].base) { 2521 free_pages((unsigned long)its->tables[i].base, 2522 its->tables[i].order); 2523 its->tables[i].base = NULL; 2524 } 2525 } 2526 } 2527 2528 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser) 2529 { 2530 u64 psz = SZ_64K; 2531 2532 while (psz) { 2533 u64 val, gpsz; 2534 2535 val = its_read_baser(its, baser); 2536 val &= ~GITS_BASER_PAGE_SIZE_MASK; 2537 2538 switch (psz) { 2539 case SZ_64K: 2540 gpsz = GITS_BASER_PAGE_SIZE_64K; 2541 break; 2542 case SZ_16K: 2543 gpsz = GITS_BASER_PAGE_SIZE_16K; 2544 break; 2545 case SZ_4K: 2546 default: 2547 gpsz = GITS_BASER_PAGE_SIZE_4K; 2548 break; 2549 } 2550 2551 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT; 2552 2553 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz); 2554 its_write_baser(its, baser, val); 2555 2556 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz) 2557 break; 2558 2559 switch (psz) { 2560 case SZ_64K: 2561 psz = SZ_16K; 2562 break; 2563 case SZ_16K: 2564 psz = SZ_4K; 2565 break; 2566 case SZ_4K: 2567 default: 2568 return -1; 2569 } 2570 } 2571 2572 baser->psz = psz; 2573 return 0; 2574 } 2575 2576 static int its_alloc_tables(struct its_node *its) 2577 { 2578 u64 shr = GITS_BASER_InnerShareable; 2579 u64 cache = GITS_BASER_RaWaWb; 2580 int err, i; 2581 2582 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) 2583 /* erratum 24313: ignore memory access type */ 2584 cache = GITS_BASER_nCnB; 2585 2586 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 2587 struct its_baser *baser = its->tables + i; 2588 u64 val = its_read_baser(its, baser); 2589 u64 type = GITS_BASER_TYPE(val); 2590 bool indirect = false; 2591 u32 order; 2592 2593 if (type == GITS_BASER_TYPE_NONE) 2594 continue; 2595 2596 if (its_probe_baser_psz(its, baser)) { 2597 its_free_tables(its); 2598 return -ENXIO; 2599 } 2600 2601 order = get_order(baser->psz); 2602 2603 switch (type) { 2604 case GITS_BASER_TYPE_DEVICE: 2605 indirect = its_parse_indirect_baser(its, baser, &order, 2606 device_ids(its)); 2607 break; 2608 2609 case GITS_BASER_TYPE_VCPU: 2610 if (is_v4_1(its)) { 2611 struct its_node *sibling; 2612 2613 WARN_ON(i != 2); 2614 if ((sibling = find_sibling_its(its))) { 2615 *baser = sibling->tables[2]; 2616 its_write_baser(its, baser, baser->val); 2617 continue; 2618 } 2619 } 2620 2621 indirect = its_parse_indirect_baser(its, baser, &order, 2622 ITS_MAX_VPEID_BITS); 2623 break; 2624 } 2625 2626 err = its_setup_baser(its, baser, cache, shr, order, indirect); 2627 if (err < 0) { 2628 its_free_tables(its); 2629 return err; 2630 } 2631 2632 /* Update settings which will be used for next BASERn */ 2633 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; 2634 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; 2635 } 2636 2637 return 0; 2638 } 2639 2640 static u64 inherit_vpe_l1_table_from_its(void) 2641 { 2642 struct its_node *its; 2643 u64 val; 2644 u32 aff; 2645 2646 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 2647 aff = compute_common_aff(val); 2648 2649 list_for_each_entry(its, &its_nodes, entry) { 2650 u64 baser, addr; 2651 2652 if (!is_v4_1(its)) 2653 continue; 2654 2655 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) 2656 continue; 2657 2658 if (aff != compute_its_aff(its)) 2659 continue; 2660 2661 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */ 2662 baser = its->tables[2].val; 2663 if (!(baser & GITS_BASER_VALID)) 2664 continue; 2665 2666 /* We have a winner! */ 2667 gic_data_rdist()->vpe_l1_base = its->tables[2].base; 2668 2669 val = GICR_VPROPBASER_4_1_VALID; 2670 if (baser & GITS_BASER_INDIRECT) 2671 val |= GICR_VPROPBASER_4_1_INDIRECT; 2672 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, 2673 FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)); 2674 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) { 2675 case GIC_PAGE_SIZE_64K: 2676 addr = GITS_BASER_ADDR_48_to_52(baser); 2677 break; 2678 default: 2679 addr = baser & GENMASK_ULL(47, 12); 2680 break; 2681 } 2682 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12); 2683 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, 2684 FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); 2685 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, 2686 FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); 2687 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); 2688 2689 return val; 2690 } 2691 2692 return 0; 2693 } 2694 2695 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) 2696 { 2697 u32 aff; 2698 u64 val; 2699 int cpu; 2700 2701 val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 2702 aff = compute_common_aff(val); 2703 2704 for_each_possible_cpu(cpu) { 2705 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; 2706 2707 if (!base || cpu == smp_processor_id()) 2708 continue; 2709 2710 val = gic_read_typer(base + GICR_TYPER); 2711 if (aff != compute_common_aff(val)) 2712 continue; 2713 2714 /* 2715 * At this point, we have a victim. This particular CPU 2716 * has already booted, and has an affinity that matches 2717 * ours wrt CommonLPIAff. Let's use its own VPROPBASER. 2718 * Make sure we don't write the Z bit in that case. 2719 */ 2720 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); 2721 val &= ~GICR_VPROPBASER_4_1_Z; 2722 2723 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; 2724 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; 2725 2726 return val; 2727 } 2728 2729 return 0; 2730 } 2731 2732 static bool allocate_vpe_l2_table(int cpu, u32 id) 2733 { 2734 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; 2735 unsigned int psz, esz, idx, npg, gpsz; 2736 u64 val; 2737 struct page *page; 2738 __le64 *table; 2739 2740 if (!gic_rdists->has_rvpeid) 2741 return true; 2742 2743 /* Skip non-present CPUs */ 2744 if (!base) 2745 return true; 2746 2747 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER); 2748 2749 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1; 2750 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); 2751 npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1; 2752 2753 switch (gpsz) { 2754 default: 2755 WARN_ON(1); 2756 fallthrough; 2757 case GIC_PAGE_SIZE_4K: 2758 psz = SZ_4K; 2759 break; 2760 case GIC_PAGE_SIZE_16K: 2761 psz = SZ_16K; 2762 break; 2763 case GIC_PAGE_SIZE_64K: 2764 psz = SZ_64K; 2765 break; 2766 } 2767 2768 /* Don't allow vpe_id that exceeds single, flat table limit */ 2769 if (!(val & GICR_VPROPBASER_4_1_INDIRECT)) 2770 return (id < (npg * psz / (esz * SZ_8))); 2771 2772 /* Compute 1st level table index & check if that exceeds table limit */ 2773 idx = id >> ilog2(psz / (esz * SZ_8)); 2774 if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE)) 2775 return false; 2776 2777 table = gic_data_rdist_cpu(cpu)->vpe_l1_base; 2778 2779 /* Allocate memory for 2nd level table */ 2780 if (!table[idx]) { 2781 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); 2782 if (!page) 2783 return false; 2784 2785 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 2786 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) 2787 gic_flush_dcache_to_poc(page_address(page), psz); 2788 2789 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 2790 2791 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 2792 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK)) 2793 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 2794 2795 /* Ensure updated table contents are visible to RD hardware */ 2796 dsb(sy); 2797 } 2798 2799 return true; 2800 } 2801 2802 static int allocate_vpe_l1_table(void) 2803 { 2804 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 2805 u64 val, gpsz, npg, pa; 2806 unsigned int psz = SZ_64K; 2807 unsigned int np, epp, esz; 2808 struct page *page; 2809 2810 if (!gic_rdists->has_rvpeid) 2811 return 0; 2812 2813 /* 2814 * if VPENDBASER.Valid is set, disable any previously programmed 2815 * VPE by setting PendingLast while clearing Valid. This has the 2816 * effect of making sure no doorbell will be generated and we can 2817 * then safely clear VPROPBASER.Valid. 2818 */ 2819 if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid) 2820 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast, 2821 vlpi_base + GICR_VPENDBASER); 2822 2823 /* 2824 * If we can inherit the configuration from another RD, let's do 2825 * so. Otherwise, we have to go through the allocation process. We 2826 * assume that all RDs have the exact same requirements, as 2827 * nothing will work otherwise. 2828 */ 2829 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); 2830 if (val & GICR_VPROPBASER_4_1_VALID) 2831 goto out; 2832 2833 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); 2834 if (!gic_data_rdist()->vpe_table_mask) 2835 return -ENOMEM; 2836 2837 val = inherit_vpe_l1_table_from_its(); 2838 if (val & GICR_VPROPBASER_4_1_VALID) 2839 goto out; 2840 2841 /* First probe the page size */ 2842 val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K); 2843 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2844 val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER); 2845 gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val); 2846 esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val); 2847 2848 switch (gpsz) { 2849 default: 2850 gpsz = GIC_PAGE_SIZE_4K; 2851 fallthrough; 2852 case GIC_PAGE_SIZE_4K: 2853 psz = SZ_4K; 2854 break; 2855 case GIC_PAGE_SIZE_16K: 2856 psz = SZ_16K; 2857 break; 2858 case GIC_PAGE_SIZE_64K: 2859 psz = SZ_64K; 2860 break; 2861 } 2862 2863 /* 2864 * Start populating the register from scratch, including RO fields 2865 * (which we want to print in debug cases...) 2866 */ 2867 val = 0; 2868 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz); 2869 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz); 2870 2871 /* How many entries per GIC page? */ 2872 esz++; 2873 epp = psz / (esz * SZ_8); 2874 2875 /* 2876 * If we need more than just a single L1 page, flag the table 2877 * as indirect and compute the number of required L1 pages. 2878 */ 2879 if (epp < ITS_MAX_VPEID) { 2880 int nl2; 2881 2882 val |= GICR_VPROPBASER_4_1_INDIRECT; 2883 2884 /* Number of L2 pages required to cover the VPEID space */ 2885 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp); 2886 2887 /* Number of L1 pages to point to the L2 pages */ 2888 npg = DIV_ROUND_UP(nl2 * SZ_8, psz); 2889 } else { 2890 npg = 1; 2891 } 2892 2893 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); 2894 2895 /* Right, that's the number of CPU pages we need for L1 */ 2896 np = DIV_ROUND_UP(npg * psz, PAGE_SIZE); 2897 2898 pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", 2899 np, npg, psz, epp, esz); 2900 page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); 2901 if (!page) 2902 return -ENOMEM; 2903 2904 gic_data_rdist()->vpe_l1_base = page_address(page); 2905 pa = virt_to_phys(page_address(page)); 2906 WARN_ON(!IS_ALIGNED(pa, psz)); 2907 2908 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12); 2909 val |= GICR_VPROPBASER_RaWb; 2910 val |= GICR_VPROPBASER_InnerShareable; 2911 val |= GICR_VPROPBASER_4_1_Z; 2912 val |= GICR_VPROPBASER_4_1_VALID; 2913 2914 out: 2915 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 2916 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); 2917 2918 pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n", 2919 smp_processor_id(), val, 2920 cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); 2921 2922 return 0; 2923 } 2924 2925 static int its_alloc_collections(struct its_node *its) 2926 { 2927 int i; 2928 2929 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), 2930 GFP_KERNEL); 2931 if (!its->collections) 2932 return -ENOMEM; 2933 2934 for (i = 0; i < nr_cpu_ids; i++) 2935 its->collections[i].target_address = ~0ULL; 2936 2937 return 0; 2938 } 2939 2940 static struct page *its_allocate_pending_table(gfp_t gfp_flags) 2941 { 2942 struct page *pend_page; 2943 2944 pend_page = alloc_pages(gfp_flags | __GFP_ZERO, 2945 get_order(LPI_PENDBASE_SZ)); 2946 if (!pend_page) 2947 return NULL; 2948 2949 /* Make sure the GIC will observe the zero-ed page */ 2950 gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ); 2951 2952 return pend_page; 2953 } 2954 2955 static void its_free_pending_table(struct page *pt) 2956 { 2957 free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); 2958 } 2959 2960 /* 2961 * Booting with kdump and LPIs enabled is generally fine. Any other 2962 * case is wrong in the absence of firmware/EFI support. 2963 */ 2964 static bool enabled_lpis_allowed(void) 2965 { 2966 phys_addr_t addr; 2967 u64 val; 2968 2969 /* Check whether the property table is in a reserved region */ 2970 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER); 2971 addr = val & GENMASK_ULL(51, 12); 2972 2973 return gic_check_reserved_range(addr, LPI_PROPBASE_SZ); 2974 } 2975 2976 static int __init allocate_lpi_tables(void) 2977 { 2978 u64 val; 2979 int err, cpu; 2980 2981 /* 2982 * If LPIs are enabled while we run this from the boot CPU, 2983 * flag the RD tables as pre-allocated if the stars do align. 2984 */ 2985 val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR); 2986 if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) { 2987 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED | 2988 RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING); 2989 pr_info("GICv3: Using preallocated redistributor tables\n"); 2990 } 2991 2992 err = its_setup_lpi_prop_table(); 2993 if (err) 2994 return err; 2995 2996 /* 2997 * We allocate all the pending tables anyway, as we may have a 2998 * mix of RDs that have had LPIs enabled, and some that 2999 * don't. We'll free the unused ones as each CPU comes online. 3000 */ 3001 for_each_possible_cpu(cpu) { 3002 struct page *pend_page; 3003 3004 pend_page = its_allocate_pending_table(GFP_NOWAIT); 3005 if (!pend_page) { 3006 pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu); 3007 return -ENOMEM; 3008 } 3009 3010 gic_data_rdist_cpu(cpu)->pend_page = pend_page; 3011 } 3012 3013 return 0; 3014 } 3015 3016 static u64 read_vpend_dirty_clear(void __iomem *vlpi_base) 3017 { 3018 u32 count = 1000000; /* 1s! */ 3019 bool clean; 3020 u64 val; 3021 3022 do { 3023 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER); 3024 clean = !(val & GICR_VPENDBASER_Dirty); 3025 if (!clean) { 3026 count--; 3027 cpu_relax(); 3028 udelay(1); 3029 } 3030 } while (!clean && count); 3031 3032 if (unlikely(!clean)) 3033 pr_err_ratelimited("ITS virtual pending table not cleaning\n"); 3034 3035 return val; 3036 } 3037 3038 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) 3039 { 3040 u64 val; 3041 3042 /* Make sure we wait until the RD is done with the initial scan */ 3043 val = read_vpend_dirty_clear(vlpi_base); 3044 val &= ~GICR_VPENDBASER_Valid; 3045 val &= ~clr; 3046 val |= set; 3047 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 3048 3049 val = read_vpend_dirty_clear(vlpi_base); 3050 if (unlikely(val & GICR_VPENDBASER_Dirty)) 3051 val |= GICR_VPENDBASER_PendingLast; 3052 3053 return val; 3054 } 3055 3056 static void its_cpu_init_lpis(void) 3057 { 3058 void __iomem *rbase = gic_data_rdist_rd_base(); 3059 struct page *pend_page; 3060 phys_addr_t paddr; 3061 u64 val, tmp; 3062 3063 if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) 3064 return; 3065 3066 val = readl_relaxed(rbase + GICR_CTLR); 3067 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && 3068 (val & GICR_CTLR_ENABLE_LPIS)) { 3069 /* 3070 * Check that we get the same property table on all 3071 * RDs. If we don't, this is hopeless. 3072 */ 3073 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER); 3074 paddr &= GENMASK_ULL(51, 12); 3075 if (WARN_ON(gic_rdists->prop_table_pa != paddr)) 3076 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 3077 3078 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER); 3079 paddr &= GENMASK_ULL(51, 16); 3080 3081 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ)); 3082 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED; 3083 3084 goto out; 3085 } 3086 3087 pend_page = gic_data_rdist()->pend_page; 3088 paddr = page_to_phys(pend_page); 3089 3090 /* set PROPBASE */ 3091 val = (gic_rdists->prop_table_pa | 3092 GICR_PROPBASER_InnerShareable | 3093 GICR_PROPBASER_RaWaWb | 3094 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 3095 3096 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 3097 tmp = gicr_read_propbaser(rbase + GICR_PROPBASER); 3098 3099 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 3100 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 3101 /* 3102 * The HW reports non-shareable, we must 3103 * remove the cacheability attributes as 3104 * well. 3105 */ 3106 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 3107 GICR_PROPBASER_CACHEABILITY_MASK); 3108 val |= GICR_PROPBASER_nC; 3109 gicr_write_propbaser(val, rbase + GICR_PROPBASER); 3110 } 3111 pr_info_once("GIC: using cache flushing for LPI property table\n"); 3112 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 3113 } 3114 3115 /* set PENDBASE */ 3116 val = (page_to_phys(pend_page) | 3117 GICR_PENDBASER_InnerShareable | 3118 GICR_PENDBASER_RaWaWb); 3119 3120 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 3121 tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER); 3122 3123 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 3124 /* 3125 * The HW reports non-shareable, we must remove the 3126 * cacheability attributes as well. 3127 */ 3128 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 3129 GICR_PENDBASER_CACHEABILITY_MASK); 3130 val |= GICR_PENDBASER_nC; 3131 gicr_write_pendbaser(val, rbase + GICR_PENDBASER); 3132 } 3133 3134 /* Enable LPIs */ 3135 val = readl_relaxed(rbase + GICR_CTLR); 3136 val |= GICR_CTLR_ENABLE_LPIS; 3137 writel_relaxed(val, rbase + GICR_CTLR); 3138 3139 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { 3140 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3141 3142 /* 3143 * It's possible for CPU to receive VLPIs before it is 3144 * scheduled as a vPE, especially for the first CPU, and the 3145 * VLPI with INTID larger than 2^(IDbits+1) will be considered 3146 * as out of range and dropped by GIC. 3147 * So we initialize IDbits to known value to avoid VLPI drop. 3148 */ 3149 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 3150 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n", 3151 smp_processor_id(), val); 3152 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 3153 3154 /* 3155 * Also clear Valid bit of GICR_VPENDBASER, in case some 3156 * ancient programming gets left in and has possibility of 3157 * corrupting memory. 3158 */ 3159 val = its_clear_vpend_valid(vlpi_base, 0, 0); 3160 } 3161 3162 if (allocate_vpe_l1_table()) { 3163 /* 3164 * If the allocation has failed, we're in massive trouble. 3165 * Disable direct injection, and pray that no VM was 3166 * already running... 3167 */ 3168 gic_rdists->has_rvpeid = false; 3169 gic_rdists->has_vlpis = false; 3170 } 3171 3172 /* Make sure the GIC has seen the above */ 3173 dsb(sy); 3174 out: 3175 gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; 3176 pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", 3177 smp_processor_id(), 3178 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ? 3179 "reserved" : "allocated", 3180 &paddr); 3181 } 3182 3183 static void its_cpu_init_collection(struct its_node *its) 3184 { 3185 int cpu = smp_processor_id(); 3186 u64 target; 3187 3188 /* avoid cross node collections and its mapping */ 3189 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { 3190 struct device_node *cpu_node; 3191 3192 cpu_node = of_get_cpu_node(cpu, NULL); 3193 if (its->numa_node != NUMA_NO_NODE && 3194 its->numa_node != of_node_to_nid(cpu_node)) 3195 return; 3196 } 3197 3198 /* 3199 * We now have to bind each collection to its target 3200 * redistributor. 3201 */ 3202 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 3203 /* 3204 * This ITS wants the physical address of the 3205 * redistributor. 3206 */ 3207 target = gic_data_rdist()->phys_base; 3208 } else { 3209 /* This ITS wants a linear CPU number. */ 3210 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER); 3211 target = GICR_TYPER_CPU_NUMBER(target) << 16; 3212 } 3213 3214 /* Perform collection mapping */ 3215 its->collections[cpu].target_address = target; 3216 its->collections[cpu].col_id = cpu; 3217 3218 its_send_mapc(its, &its->collections[cpu], 1); 3219 its_send_invall(its, &its->collections[cpu]); 3220 } 3221 3222 static void its_cpu_init_collections(void) 3223 { 3224 struct its_node *its; 3225 3226 raw_spin_lock(&its_lock); 3227 3228 list_for_each_entry(its, &its_nodes, entry) 3229 its_cpu_init_collection(its); 3230 3231 raw_spin_unlock(&its_lock); 3232 } 3233 3234 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 3235 { 3236 struct its_device *its_dev = NULL, *tmp; 3237 unsigned long flags; 3238 3239 raw_spin_lock_irqsave(&its->lock, flags); 3240 3241 list_for_each_entry(tmp, &its->its_device_list, entry) { 3242 if (tmp->device_id == dev_id) { 3243 its_dev = tmp; 3244 break; 3245 } 3246 } 3247 3248 raw_spin_unlock_irqrestore(&its->lock, flags); 3249 3250 return its_dev; 3251 } 3252 3253 static struct its_baser *its_get_baser(struct its_node *its, u32 type) 3254 { 3255 int i; 3256 3257 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 3258 if (GITS_BASER_TYPE(its->tables[i].val) == type) 3259 return &its->tables[i]; 3260 } 3261 3262 return NULL; 3263 } 3264 3265 static bool its_alloc_table_entry(struct its_node *its, 3266 struct its_baser *baser, u32 id) 3267 { 3268 struct page *page; 3269 u32 esz, idx; 3270 __le64 *table; 3271 3272 /* Don't allow device id that exceeds single, flat table limit */ 3273 esz = GITS_BASER_ENTRY_SIZE(baser->val); 3274 if (!(baser->val & GITS_BASER_INDIRECT)) 3275 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); 3276 3277 /* Compute 1st level table index & check if that exceeds table limit */ 3278 idx = id >> ilog2(baser->psz / esz); 3279 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) 3280 return false; 3281 3282 table = baser->base; 3283 3284 /* Allocate memory for 2nd level table */ 3285 if (!table[idx]) { 3286 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 3287 get_order(baser->psz)); 3288 if (!page) 3289 return false; 3290 3291 /* Flush Lvl2 table to PoC if hw doesn't support coherency */ 3292 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 3293 gic_flush_dcache_to_poc(page_address(page), baser->psz); 3294 3295 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID); 3296 3297 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */ 3298 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) 3299 gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE); 3300 3301 /* Ensure updated table contents are visible to ITS hardware */ 3302 dsb(sy); 3303 } 3304 3305 return true; 3306 } 3307 3308 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) 3309 { 3310 struct its_baser *baser; 3311 3312 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); 3313 3314 /* Don't allow device id that exceeds ITS hardware limit */ 3315 if (!baser) 3316 return (ilog2(dev_id) < device_ids(its)); 3317 3318 return its_alloc_table_entry(its, baser, dev_id); 3319 } 3320 3321 static bool its_alloc_vpe_table(u32 vpe_id) 3322 { 3323 struct its_node *its; 3324 int cpu; 3325 3326 /* 3327 * Make sure the L2 tables are allocated on *all* v4 ITSs. We 3328 * could try and only do it on ITSs corresponding to devices 3329 * that have interrupts targeted at this VPE, but the 3330 * complexity becomes crazy (and you have tons of memory 3331 * anyway, right?). 3332 */ 3333 list_for_each_entry(its, &its_nodes, entry) { 3334 struct its_baser *baser; 3335 3336 if (!is_v4(its)) 3337 continue; 3338 3339 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); 3340 if (!baser) 3341 return false; 3342 3343 if (!its_alloc_table_entry(its, baser, vpe_id)) 3344 return false; 3345 } 3346 3347 /* Non v4.1? No need to iterate RDs and go back early. */ 3348 if (!gic_rdists->has_rvpeid) 3349 return true; 3350 3351 /* 3352 * Make sure the L2 tables are allocated for all copies of 3353 * the L1 table on *all* v4.1 RDs. 3354 */ 3355 for_each_possible_cpu(cpu) { 3356 if (!allocate_vpe_l2_table(cpu, vpe_id)) 3357 return false; 3358 } 3359 3360 return true; 3361 } 3362 3363 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 3364 int nvecs, bool alloc_lpis) 3365 { 3366 struct its_device *dev; 3367 unsigned long *lpi_map = NULL; 3368 unsigned long flags; 3369 u16 *col_map = NULL; 3370 void *itt; 3371 int lpi_base; 3372 int nr_lpis; 3373 int nr_ites; 3374 int sz; 3375 3376 if (!its_alloc_device_table(its, dev_id)) 3377 return NULL; 3378 3379 if (WARN_ON(!is_power_of_2(nvecs))) 3380 nvecs = roundup_pow_of_two(nvecs); 3381 3382 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 3383 /* 3384 * Even if the device wants a single LPI, the ITT must be 3385 * sized as a power of two (and you need at least one bit...). 3386 */ 3387 nr_ites = max(2, nvecs); 3388 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); 3389 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 3390 itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); 3391 if (alloc_lpis) { 3392 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); 3393 if (lpi_map) 3394 col_map = kcalloc(nr_lpis, sizeof(*col_map), 3395 GFP_KERNEL); 3396 } else { 3397 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL); 3398 nr_lpis = 0; 3399 lpi_base = 0; 3400 } 3401 3402 if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { 3403 kfree(dev); 3404 kfree(itt); 3405 bitmap_free(lpi_map); 3406 kfree(col_map); 3407 return NULL; 3408 } 3409 3410 gic_flush_dcache_to_poc(itt, sz); 3411 3412 dev->its = its; 3413 dev->itt = itt; 3414 dev->nr_ites = nr_ites; 3415 dev->event_map.lpi_map = lpi_map; 3416 dev->event_map.col_map = col_map; 3417 dev->event_map.lpi_base = lpi_base; 3418 dev->event_map.nr_lpis = nr_lpis; 3419 raw_spin_lock_init(&dev->event_map.vlpi_lock); 3420 dev->device_id = dev_id; 3421 INIT_LIST_HEAD(&dev->entry); 3422 3423 raw_spin_lock_irqsave(&its->lock, flags); 3424 list_add(&dev->entry, &its->its_device_list); 3425 raw_spin_unlock_irqrestore(&its->lock, flags); 3426 3427 /* Map device to its ITT */ 3428 its_send_mapd(dev, 1); 3429 3430 return dev; 3431 } 3432 3433 static void its_free_device(struct its_device *its_dev) 3434 { 3435 unsigned long flags; 3436 3437 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 3438 list_del(&its_dev->entry); 3439 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 3440 kfree(its_dev->event_map.col_map); 3441 kfree(its_dev->itt); 3442 kfree(its_dev); 3443 } 3444 3445 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq) 3446 { 3447 int idx; 3448 3449 /* Find a free LPI region in lpi_map and allocate them. */ 3450 idx = bitmap_find_free_region(dev->event_map.lpi_map, 3451 dev->event_map.nr_lpis, 3452 get_count_order(nvecs)); 3453 if (idx < 0) 3454 return -ENOSPC; 3455 3456 *hwirq = dev->event_map.lpi_base + idx; 3457 3458 return 0; 3459 } 3460 3461 static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 3462 int nvec, msi_alloc_info_t *info) 3463 { 3464 struct its_node *its; 3465 struct its_device *its_dev; 3466 struct msi_domain_info *msi_info; 3467 u32 dev_id; 3468 int err = 0; 3469 3470 /* 3471 * We ignore "dev" entirely, and rely on the dev_id that has 3472 * been passed via the scratchpad. This limits this domain's 3473 * usefulness to upper layers that definitely know that they 3474 * are built on top of the ITS. 3475 */ 3476 dev_id = info->scratchpad[0].ul; 3477 3478 msi_info = msi_get_domain_info(domain); 3479 its = msi_info->data; 3480 3481 if (!gic_rdists->has_direct_lpi && 3482 vpe_proxy.dev && 3483 vpe_proxy.dev->its == its && 3484 dev_id == vpe_proxy.dev->device_id) { 3485 /* Bad luck. Get yourself a better implementation */ 3486 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n", 3487 dev_id); 3488 return -EINVAL; 3489 } 3490 3491 mutex_lock(&its->dev_alloc_lock); 3492 its_dev = its_find_device(its, dev_id); 3493 if (its_dev) { 3494 /* 3495 * We already have seen this ID, probably through 3496 * another alias (PCI bridge of some sort). No need to 3497 * create the device. 3498 */ 3499 its_dev->shared = true; 3500 pr_debug("Reusing ITT for devID %x\n", dev_id); 3501 goto out; 3502 } 3503 3504 its_dev = its_create_device(its, dev_id, nvec, true); 3505 if (!its_dev) { 3506 err = -ENOMEM; 3507 goto out; 3508 } 3509 3510 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE) 3511 its_dev->shared = true; 3512 3513 pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec)); 3514 out: 3515 mutex_unlock(&its->dev_alloc_lock); 3516 info->scratchpad[0].ptr = its_dev; 3517 return err; 3518 } 3519 3520 static struct msi_domain_ops its_msi_domain_ops = { 3521 .msi_prepare = its_msi_prepare, 3522 }; 3523 3524 static int its_irq_gic_domain_alloc(struct irq_domain *domain, 3525 unsigned int virq, 3526 irq_hw_number_t hwirq) 3527 { 3528 struct irq_fwspec fwspec; 3529 3530 if (irq_domain_get_of_node(domain->parent)) { 3531 fwspec.fwnode = domain->parent->fwnode; 3532 fwspec.param_count = 3; 3533 fwspec.param[0] = GIC_IRQ_TYPE_LPI; 3534 fwspec.param[1] = hwirq; 3535 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 3536 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { 3537 fwspec.fwnode = domain->parent->fwnode; 3538 fwspec.param_count = 2; 3539 fwspec.param[0] = hwirq; 3540 fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 3541 } else { 3542 return -EINVAL; 3543 } 3544 3545 return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 3546 } 3547 3548 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 3549 unsigned int nr_irqs, void *args) 3550 { 3551 msi_alloc_info_t *info = args; 3552 struct its_device *its_dev = info->scratchpad[0].ptr; 3553 struct its_node *its = its_dev->its; 3554 struct irq_data *irqd; 3555 irq_hw_number_t hwirq; 3556 int err; 3557 int i; 3558 3559 err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq); 3560 if (err) 3561 return err; 3562 3563 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); 3564 if (err) 3565 return err; 3566 3567 for (i = 0; i < nr_irqs; i++) { 3568 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i); 3569 if (err) 3570 return err; 3571 3572 irq_domain_set_hwirq_and_chip(domain, virq + i, 3573 hwirq + i, &its_irq_chip, its_dev); 3574 irqd = irq_get_irq_data(virq + i); 3575 irqd_set_single_target(irqd); 3576 irqd_set_affinity_on_activate(irqd); 3577 pr_debug("ID:%d pID:%d vID:%d\n", 3578 (int)(hwirq + i - its_dev->event_map.lpi_base), 3579 (int)(hwirq + i), virq + i); 3580 } 3581 3582 return 0; 3583 } 3584 3585 static int its_irq_domain_activate(struct irq_domain *domain, 3586 struct irq_data *d, bool reserve) 3587 { 3588 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3589 u32 event = its_get_event_id(d); 3590 int cpu; 3591 3592 cpu = its_select_cpu(d, cpu_online_mask); 3593 if (cpu < 0 || cpu >= nr_cpu_ids) 3594 return -EINVAL; 3595 3596 its_inc_lpi_count(d, cpu); 3597 its_dev->event_map.col_map[event] = cpu; 3598 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 3599 3600 /* Map the GIC IRQ and event to the device */ 3601 its_send_mapti(its_dev, d->hwirq, event); 3602 return 0; 3603 } 3604 3605 static void its_irq_domain_deactivate(struct irq_domain *domain, 3606 struct irq_data *d) 3607 { 3608 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3609 u32 event = its_get_event_id(d); 3610 3611 its_dec_lpi_count(d, its_dev->event_map.col_map[event]); 3612 /* Stop the delivery of interrupts */ 3613 its_send_discard(its_dev, event); 3614 } 3615 3616 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 3617 unsigned int nr_irqs) 3618 { 3619 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 3620 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 3621 struct its_node *its = its_dev->its; 3622 int i; 3623 3624 bitmap_release_region(its_dev->event_map.lpi_map, 3625 its_get_event_id(irq_domain_get_irq_data(domain, virq)), 3626 get_count_order(nr_irqs)); 3627 3628 for (i = 0; i < nr_irqs; i++) { 3629 struct irq_data *data = irq_domain_get_irq_data(domain, 3630 virq + i); 3631 /* Nuke the entry in the domain */ 3632 irq_domain_reset_irq_data(data); 3633 } 3634 3635 mutex_lock(&its->dev_alloc_lock); 3636 3637 /* 3638 * If all interrupts have been freed, start mopping the 3639 * floor. This is conditioned on the device not being shared. 3640 */ 3641 if (!its_dev->shared && 3642 bitmap_empty(its_dev->event_map.lpi_map, 3643 its_dev->event_map.nr_lpis)) { 3644 its_lpi_free(its_dev->event_map.lpi_map, 3645 its_dev->event_map.lpi_base, 3646 its_dev->event_map.nr_lpis); 3647 3648 /* Unmap device/itt */ 3649 its_send_mapd(its_dev, 0); 3650 its_free_device(its_dev); 3651 } 3652 3653 mutex_unlock(&its->dev_alloc_lock); 3654 3655 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 3656 } 3657 3658 static const struct irq_domain_ops its_domain_ops = { 3659 .alloc = its_irq_domain_alloc, 3660 .free = its_irq_domain_free, 3661 .activate = its_irq_domain_activate, 3662 .deactivate = its_irq_domain_deactivate, 3663 }; 3664 3665 /* 3666 * This is insane. 3667 * 3668 * If a GICv4.0 doesn't implement Direct LPIs (which is extremely 3669 * likely), the only way to perform an invalidate is to use a fake 3670 * device to issue an INV command, implying that the LPI has first 3671 * been mapped to some event on that device. Since this is not exactly 3672 * cheap, we try to keep that mapping around as long as possible, and 3673 * only issue an UNMAP if we're short on available slots. 3674 * 3675 * Broken by design(tm). 3676 * 3677 * GICv4.1, on the other hand, mandates that we're able to invalidate 3678 * by writing to a MMIO register. It doesn't implement the whole of 3679 * DirectLPI, but that's good enough. And most of the time, we don't 3680 * even have to invalidate anything, as the redistributor can be told 3681 * whether to generate a doorbell or not (we thus leave it enabled, 3682 * always). 3683 */ 3684 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe) 3685 { 3686 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3687 if (gic_rdists->has_rvpeid) 3688 return; 3689 3690 /* Already unmapped? */ 3691 if (vpe->vpe_proxy_event == -1) 3692 return; 3693 3694 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); 3695 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; 3696 3697 /* 3698 * We don't track empty slots at all, so let's move the 3699 * next_victim pointer if we can quickly reuse that slot 3700 * instead of nuking an existing entry. Not clear that this is 3701 * always a win though, and this might just generate a ripple 3702 * effect... Let's just hope VPEs don't migrate too often. 3703 */ 3704 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 3705 vpe_proxy.next_victim = vpe->vpe_proxy_event; 3706 3707 vpe->vpe_proxy_event = -1; 3708 } 3709 3710 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe) 3711 { 3712 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3713 if (gic_rdists->has_rvpeid) 3714 return; 3715 3716 if (!gic_rdists->has_direct_lpi) { 3717 unsigned long flags; 3718 3719 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3720 its_vpe_db_proxy_unmap_locked(vpe); 3721 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3722 } 3723 } 3724 3725 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe) 3726 { 3727 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3728 if (gic_rdists->has_rvpeid) 3729 return; 3730 3731 /* Already mapped? */ 3732 if (vpe->vpe_proxy_event != -1) 3733 return; 3734 3735 /* This slot was already allocated. Kick the other VPE out. */ 3736 if (vpe_proxy.vpes[vpe_proxy.next_victim]) 3737 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]); 3738 3739 /* Map the new VPE instead */ 3740 vpe_proxy.vpes[vpe_proxy.next_victim] = vpe; 3741 vpe->vpe_proxy_event = vpe_proxy.next_victim; 3742 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; 3743 3744 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; 3745 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); 3746 } 3747 3748 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) 3749 { 3750 unsigned long flags; 3751 struct its_collection *target_col; 3752 3753 /* GICv4.1 doesn't use a proxy, so nothing to do here */ 3754 if (gic_rdists->has_rvpeid) 3755 return; 3756 3757 if (gic_rdists->has_direct_lpi) { 3758 void __iomem *rdbase; 3759 3760 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; 3761 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 3762 wait_for_syncr(rdbase); 3763 3764 return; 3765 } 3766 3767 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3768 3769 its_vpe_db_proxy_map_locked(vpe); 3770 3771 target_col = &vpe_proxy.dev->its->collections[to]; 3772 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); 3773 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; 3774 3775 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3776 } 3777 3778 static int its_vpe_set_affinity(struct irq_data *d, 3779 const struct cpumask *mask_val, 3780 bool force) 3781 { 3782 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3783 int from, cpu = cpumask_first(mask_val); 3784 unsigned long flags; 3785 3786 /* 3787 * Changing affinity is mega expensive, so let's be as lazy as 3788 * we can and only do it if we really have to. Also, if mapped 3789 * into the proxy device, we need to move the doorbell 3790 * interrupt to its new location. 3791 * 3792 * Another thing is that changing the affinity of a vPE affects 3793 * *other interrupts* such as all the vLPIs that are routed to 3794 * this vPE. This means that the irq_desc lock is not enough to 3795 * protect us, and that we must ensure nobody samples vpe->col_idx 3796 * during the update, hence the lock below which must also be 3797 * taken on any vLPI handling path that evaluates vpe->col_idx. 3798 */ 3799 from = vpe_to_cpuid_lock(vpe, &flags); 3800 if (from == cpu) 3801 goto out; 3802 3803 vpe->col_idx = cpu; 3804 3805 /* 3806 * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD 3807 * is sharing its VPE table with the current one. 3808 */ 3809 if (gic_data_rdist_cpu(cpu)->vpe_table_mask && 3810 cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask)) 3811 goto out; 3812 3813 its_send_vmovp(vpe); 3814 its_vpe_db_proxy_move(vpe, from, cpu); 3815 3816 out: 3817 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 3818 vpe_to_cpuid_unlock(vpe, flags); 3819 3820 return IRQ_SET_MASK_OK_DONE; 3821 } 3822 3823 static void its_wait_vpt_parse_complete(void) 3824 { 3825 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3826 u64 val; 3827 3828 if (!gic_rdists->has_vpend_valid_dirty) 3829 return; 3830 3831 WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER, 3832 val, 3833 !(val & GICR_VPENDBASER_Dirty), 3834 1, 500)); 3835 } 3836 3837 static void its_vpe_schedule(struct its_vpe *vpe) 3838 { 3839 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3840 u64 val; 3841 3842 /* Schedule the VPE */ 3843 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & 3844 GENMASK_ULL(51, 12); 3845 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; 3846 val |= GICR_VPROPBASER_RaWb; 3847 val |= GICR_VPROPBASER_InnerShareable; 3848 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); 3849 3850 val = virt_to_phys(page_address(vpe->vpt_page)) & 3851 GENMASK_ULL(51, 16); 3852 val |= GICR_VPENDBASER_RaWaWb; 3853 val |= GICR_VPENDBASER_InnerShareable; 3854 /* 3855 * There is no good way of finding out if the pending table is 3856 * empty as we can race against the doorbell interrupt very 3857 * easily. So in the end, vpe->pending_last is only an 3858 * indication that the vcpu has something pending, not one 3859 * that the pending table is empty. A good implementation 3860 * would be able to read its coarse map pretty quickly anyway, 3861 * making this a tolerable issue. 3862 */ 3863 val |= GICR_VPENDBASER_PendingLast; 3864 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; 3865 val |= GICR_VPENDBASER_Valid; 3866 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 3867 } 3868 3869 static void its_vpe_deschedule(struct its_vpe *vpe) 3870 { 3871 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 3872 u64 val; 3873 3874 val = its_clear_vpend_valid(vlpi_base, 0, 0); 3875 3876 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); 3877 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 3878 } 3879 3880 static void its_vpe_invall(struct its_vpe *vpe) 3881 { 3882 struct its_node *its; 3883 3884 list_for_each_entry(its, &its_nodes, entry) { 3885 if (!is_v4(its)) 3886 continue; 3887 3888 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) 3889 continue; 3890 3891 /* 3892 * Sending a VINVALL to a single ITS is enough, as all 3893 * we need is to reach the redistributors. 3894 */ 3895 its_send_vinvall(its, vpe); 3896 return; 3897 } 3898 } 3899 3900 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 3901 { 3902 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3903 struct its_cmd_info *info = vcpu_info; 3904 3905 switch (info->cmd_type) { 3906 case SCHEDULE_VPE: 3907 its_vpe_schedule(vpe); 3908 return 0; 3909 3910 case DESCHEDULE_VPE: 3911 its_vpe_deschedule(vpe); 3912 return 0; 3913 3914 case COMMIT_VPE: 3915 its_wait_vpt_parse_complete(); 3916 return 0; 3917 3918 case INVALL_VPE: 3919 its_vpe_invall(vpe); 3920 return 0; 3921 3922 default: 3923 return -EINVAL; 3924 } 3925 } 3926 3927 static void its_vpe_send_cmd(struct its_vpe *vpe, 3928 void (*cmd)(struct its_device *, u32)) 3929 { 3930 unsigned long flags; 3931 3932 raw_spin_lock_irqsave(&vpe_proxy.lock, flags); 3933 3934 its_vpe_db_proxy_map_locked(vpe); 3935 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); 3936 3937 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); 3938 } 3939 3940 static void its_vpe_send_inv(struct irq_data *d) 3941 { 3942 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3943 3944 if (gic_rdists->has_direct_lpi) { 3945 void __iomem *rdbase; 3946 3947 /* Target the redistributor this VPE is currently known on */ 3948 raw_spin_lock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); 3949 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 3950 gic_write_lpir(d->parent_data->hwirq, rdbase + GICR_INVLPIR); 3951 wait_for_syncr(rdbase); 3952 raw_spin_unlock(&gic_data_rdist_cpu(vpe->col_idx)->rd_lock); 3953 } else { 3954 its_vpe_send_cmd(vpe, its_send_inv); 3955 } 3956 } 3957 3958 static void its_vpe_mask_irq(struct irq_data *d) 3959 { 3960 /* 3961 * We need to unmask the LPI, which is described by the parent 3962 * irq_data. Instead of calling into the parent (which won't 3963 * exactly do the right thing, let's simply use the 3964 * parent_data pointer. Yes, I'm naughty. 3965 */ 3966 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 3967 its_vpe_send_inv(d); 3968 } 3969 3970 static void its_vpe_unmask_irq(struct irq_data *d) 3971 { 3972 /* Same hack as above... */ 3973 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 3974 its_vpe_send_inv(d); 3975 } 3976 3977 static int its_vpe_set_irqchip_state(struct irq_data *d, 3978 enum irqchip_irq_state which, 3979 bool state) 3980 { 3981 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 3982 3983 if (which != IRQCHIP_STATE_PENDING) 3984 return -EINVAL; 3985 3986 if (gic_rdists->has_direct_lpi) { 3987 void __iomem *rdbase; 3988 3989 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; 3990 if (state) { 3991 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); 3992 } else { 3993 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); 3994 wait_for_syncr(rdbase); 3995 } 3996 } else { 3997 if (state) 3998 its_vpe_send_cmd(vpe, its_send_int); 3999 else 4000 its_vpe_send_cmd(vpe, its_send_clear); 4001 } 4002 4003 return 0; 4004 } 4005 4006 static int its_vpe_retrigger(struct irq_data *d) 4007 { 4008 return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true); 4009 } 4010 4011 static struct irq_chip its_vpe_irq_chip = { 4012 .name = "GICv4-vpe", 4013 .irq_mask = its_vpe_mask_irq, 4014 .irq_unmask = its_vpe_unmask_irq, 4015 .irq_eoi = irq_chip_eoi_parent, 4016 .irq_set_affinity = its_vpe_set_affinity, 4017 .irq_retrigger = its_vpe_retrigger, 4018 .irq_set_irqchip_state = its_vpe_set_irqchip_state, 4019 .irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity, 4020 }; 4021 4022 static struct its_node *find_4_1_its(void) 4023 { 4024 static struct its_node *its = NULL; 4025 4026 if (!its) { 4027 list_for_each_entry(its, &its_nodes, entry) { 4028 if (is_v4_1(its)) 4029 return its; 4030 } 4031 4032 /* Oops? */ 4033 its = NULL; 4034 } 4035 4036 return its; 4037 } 4038 4039 static void its_vpe_4_1_send_inv(struct irq_data *d) 4040 { 4041 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4042 struct its_node *its; 4043 4044 /* 4045 * GICv4.1 wants doorbells to be invalidated using the 4046 * INVDB command in order to be broadcast to all RDs. Send 4047 * it to the first valid ITS, and let the HW do its magic. 4048 */ 4049 its = find_4_1_its(); 4050 if (its) 4051 its_send_invdb(its, vpe); 4052 } 4053 4054 static void its_vpe_4_1_mask_irq(struct irq_data *d) 4055 { 4056 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); 4057 its_vpe_4_1_send_inv(d); 4058 } 4059 4060 static void its_vpe_4_1_unmask_irq(struct irq_data *d) 4061 { 4062 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); 4063 its_vpe_4_1_send_inv(d); 4064 } 4065 4066 static void its_vpe_4_1_schedule(struct its_vpe *vpe, 4067 struct its_cmd_info *info) 4068 { 4069 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 4070 u64 val = 0; 4071 4072 /* Schedule the VPE */ 4073 val |= GICR_VPENDBASER_Valid; 4074 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; 4075 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; 4076 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); 4077 4078 gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER); 4079 } 4080 4081 static void its_vpe_4_1_deschedule(struct its_vpe *vpe, 4082 struct its_cmd_info *info) 4083 { 4084 void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); 4085 u64 val; 4086 4087 if (info->req_db) { 4088 unsigned long flags; 4089 4090 /* 4091 * vPE is going to block: make the vPE non-resident with 4092 * PendingLast clear and DB set. The GIC guarantees that if 4093 * we read-back PendingLast clear, then a doorbell will be 4094 * delivered when an interrupt comes. 4095 * 4096 * Note the locking to deal with the concurrent update of 4097 * pending_last from the doorbell interrupt handler that can 4098 * run concurrently. 4099 */ 4100 raw_spin_lock_irqsave(&vpe->vpe_lock, flags); 4101 val = its_clear_vpend_valid(vlpi_base, 4102 GICR_VPENDBASER_PendingLast, 4103 GICR_VPENDBASER_4_1_DB); 4104 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); 4105 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); 4106 } else { 4107 /* 4108 * We're not blocking, so just make the vPE non-resident 4109 * with PendingLast set, indicating that we'll be back. 4110 */ 4111 val = its_clear_vpend_valid(vlpi_base, 4112 0, 4113 GICR_VPENDBASER_PendingLast); 4114 vpe->pending_last = true; 4115 } 4116 } 4117 4118 static void its_vpe_4_1_invall(struct its_vpe *vpe) 4119 { 4120 void __iomem *rdbase; 4121 unsigned long flags; 4122 u64 val; 4123 int cpu; 4124 4125 val = GICR_INVALLR_V; 4126 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); 4127 4128 /* Target the redistributor this vPE is currently known on */ 4129 cpu = vpe_to_cpuid_lock(vpe, &flags); 4130 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 4131 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; 4132 gic_write_lpir(val, rdbase + GICR_INVALLR); 4133 4134 wait_for_syncr(rdbase); 4135 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 4136 vpe_to_cpuid_unlock(vpe, flags); 4137 } 4138 4139 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 4140 { 4141 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4142 struct its_cmd_info *info = vcpu_info; 4143 4144 switch (info->cmd_type) { 4145 case SCHEDULE_VPE: 4146 its_vpe_4_1_schedule(vpe, info); 4147 return 0; 4148 4149 case DESCHEDULE_VPE: 4150 its_vpe_4_1_deschedule(vpe, info); 4151 return 0; 4152 4153 case COMMIT_VPE: 4154 its_wait_vpt_parse_complete(); 4155 return 0; 4156 4157 case INVALL_VPE: 4158 its_vpe_4_1_invall(vpe); 4159 return 0; 4160 4161 default: 4162 return -EINVAL; 4163 } 4164 } 4165 4166 static struct irq_chip its_vpe_4_1_irq_chip = { 4167 .name = "GICv4.1-vpe", 4168 .irq_mask = its_vpe_4_1_mask_irq, 4169 .irq_unmask = its_vpe_4_1_unmask_irq, 4170 .irq_eoi = irq_chip_eoi_parent, 4171 .irq_set_affinity = its_vpe_set_affinity, 4172 .irq_set_vcpu_affinity = its_vpe_4_1_set_vcpu_affinity, 4173 }; 4174 4175 static void its_configure_sgi(struct irq_data *d, bool clear) 4176 { 4177 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4178 struct its_cmd_desc desc; 4179 4180 desc.its_vsgi_cmd.vpe = vpe; 4181 desc.its_vsgi_cmd.sgi = d->hwirq; 4182 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority; 4183 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled; 4184 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group; 4185 desc.its_vsgi_cmd.clear = clear; 4186 4187 /* 4188 * GICv4.1 allows us to send VSGI commands to any ITS as long as the 4189 * destination VPE is mapped there. Since we map them eagerly at 4190 * activation time, we're pretty sure the first GICv4.1 ITS will do. 4191 */ 4192 its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc); 4193 } 4194 4195 static void its_sgi_mask_irq(struct irq_data *d) 4196 { 4197 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4198 4199 vpe->sgi_config[d->hwirq].enabled = false; 4200 its_configure_sgi(d, false); 4201 } 4202 4203 static void its_sgi_unmask_irq(struct irq_data *d) 4204 { 4205 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4206 4207 vpe->sgi_config[d->hwirq].enabled = true; 4208 its_configure_sgi(d, false); 4209 } 4210 4211 static int its_sgi_set_affinity(struct irq_data *d, 4212 const struct cpumask *mask_val, 4213 bool force) 4214 { 4215 /* 4216 * There is no notion of affinity for virtual SGIs, at least 4217 * not on the host (since they can only be targeting a vPE). 4218 * Tell the kernel we've done whatever it asked for. 4219 */ 4220 irq_data_update_effective_affinity(d, mask_val); 4221 return IRQ_SET_MASK_OK; 4222 } 4223 4224 static int its_sgi_set_irqchip_state(struct irq_data *d, 4225 enum irqchip_irq_state which, 4226 bool state) 4227 { 4228 if (which != IRQCHIP_STATE_PENDING) 4229 return -EINVAL; 4230 4231 if (state) { 4232 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4233 struct its_node *its = find_4_1_its(); 4234 u64 val; 4235 4236 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); 4237 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); 4238 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); 4239 } else { 4240 its_configure_sgi(d, true); 4241 } 4242 4243 return 0; 4244 } 4245 4246 static int its_sgi_get_irqchip_state(struct irq_data *d, 4247 enum irqchip_irq_state which, bool *val) 4248 { 4249 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4250 void __iomem *base; 4251 unsigned long flags; 4252 u32 count = 1000000; /* 1s! */ 4253 u32 status; 4254 int cpu; 4255 4256 if (which != IRQCHIP_STATE_PENDING) 4257 return -EINVAL; 4258 4259 /* 4260 * Locking galore! We can race against two different events: 4261 * 4262 * - Concurrent vPE affinity change: we must make sure it cannot 4263 * happen, or we'll talk to the wrong redistributor. This is 4264 * identical to what happens with vLPIs. 4265 * 4266 * - Concurrent VSGIPENDR access: As it involves accessing two 4267 * MMIO registers, this must be made atomic one way or another. 4268 */ 4269 cpu = vpe_to_cpuid_lock(vpe, &flags); 4270 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); 4271 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K; 4272 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR); 4273 do { 4274 status = readl_relaxed(base + GICR_VSGIPENDR); 4275 if (!(status & GICR_VSGIPENDR_BUSY)) 4276 goto out; 4277 4278 count--; 4279 if (!count) { 4280 pr_err_ratelimited("Unable to get SGI status\n"); 4281 goto out; 4282 } 4283 cpu_relax(); 4284 udelay(1); 4285 } while (count); 4286 4287 out: 4288 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); 4289 vpe_to_cpuid_unlock(vpe, flags); 4290 4291 if (!count) 4292 return -ENXIO; 4293 4294 *val = !!(status & (1 << d->hwirq)); 4295 4296 return 0; 4297 } 4298 4299 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) 4300 { 4301 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4302 struct its_cmd_info *info = vcpu_info; 4303 4304 switch (info->cmd_type) { 4305 case PROP_UPDATE_VSGI: 4306 vpe->sgi_config[d->hwirq].priority = info->priority; 4307 vpe->sgi_config[d->hwirq].group = info->group; 4308 its_configure_sgi(d, false); 4309 return 0; 4310 4311 default: 4312 return -EINVAL; 4313 } 4314 } 4315 4316 static struct irq_chip its_sgi_irq_chip = { 4317 .name = "GICv4.1-sgi", 4318 .irq_mask = its_sgi_mask_irq, 4319 .irq_unmask = its_sgi_unmask_irq, 4320 .irq_set_affinity = its_sgi_set_affinity, 4321 .irq_set_irqchip_state = its_sgi_set_irqchip_state, 4322 .irq_get_irqchip_state = its_sgi_get_irqchip_state, 4323 .irq_set_vcpu_affinity = its_sgi_set_vcpu_affinity, 4324 }; 4325 4326 static int its_sgi_irq_domain_alloc(struct irq_domain *domain, 4327 unsigned int virq, unsigned int nr_irqs, 4328 void *args) 4329 { 4330 struct its_vpe *vpe = args; 4331 int i; 4332 4333 /* Yes, we do want 16 SGIs */ 4334 WARN_ON(nr_irqs != 16); 4335 4336 for (i = 0; i < 16; i++) { 4337 vpe->sgi_config[i].priority = 0; 4338 vpe->sgi_config[i].enabled = false; 4339 vpe->sgi_config[i].group = false; 4340 4341 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 4342 &its_sgi_irq_chip, vpe); 4343 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY); 4344 } 4345 4346 return 0; 4347 } 4348 4349 static void its_sgi_irq_domain_free(struct irq_domain *domain, 4350 unsigned int virq, 4351 unsigned int nr_irqs) 4352 { 4353 /* Nothing to do */ 4354 } 4355 4356 static int its_sgi_irq_domain_activate(struct irq_domain *domain, 4357 struct irq_data *d, bool reserve) 4358 { 4359 /* Write out the initial SGI configuration */ 4360 its_configure_sgi(d, false); 4361 return 0; 4362 } 4363 4364 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain, 4365 struct irq_data *d) 4366 { 4367 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4368 4369 /* 4370 * The VSGI command is awkward: 4371 * 4372 * - To change the configuration, CLEAR must be set to false, 4373 * leaving the pending bit unchanged. 4374 * - To clear the pending bit, CLEAR must be set to true, leaving 4375 * the configuration unchanged. 4376 * 4377 * You just can't do both at once, hence the two commands below. 4378 */ 4379 vpe->sgi_config[d->hwirq].enabled = false; 4380 its_configure_sgi(d, false); 4381 its_configure_sgi(d, true); 4382 } 4383 4384 static const struct irq_domain_ops its_sgi_domain_ops = { 4385 .alloc = its_sgi_irq_domain_alloc, 4386 .free = its_sgi_irq_domain_free, 4387 .activate = its_sgi_irq_domain_activate, 4388 .deactivate = its_sgi_irq_domain_deactivate, 4389 }; 4390 4391 static int its_vpe_id_alloc(void) 4392 { 4393 return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); 4394 } 4395 4396 static void its_vpe_id_free(u16 id) 4397 { 4398 ida_simple_remove(&its_vpeid_ida, id); 4399 } 4400 4401 static int its_vpe_init(struct its_vpe *vpe) 4402 { 4403 struct page *vpt_page; 4404 int vpe_id; 4405 4406 /* Allocate vpe_id */ 4407 vpe_id = its_vpe_id_alloc(); 4408 if (vpe_id < 0) 4409 return vpe_id; 4410 4411 /* Allocate VPT */ 4412 vpt_page = its_allocate_pending_table(GFP_KERNEL); 4413 if (!vpt_page) { 4414 its_vpe_id_free(vpe_id); 4415 return -ENOMEM; 4416 } 4417 4418 if (!its_alloc_vpe_table(vpe_id)) { 4419 its_vpe_id_free(vpe_id); 4420 its_free_pending_table(vpt_page); 4421 return -ENOMEM; 4422 } 4423 4424 raw_spin_lock_init(&vpe->vpe_lock); 4425 vpe->vpe_id = vpe_id; 4426 vpe->vpt_page = vpt_page; 4427 if (gic_rdists->has_rvpeid) 4428 atomic_set(&vpe->vmapp_count, 0); 4429 else 4430 vpe->vpe_proxy_event = -1; 4431 4432 return 0; 4433 } 4434 4435 static void its_vpe_teardown(struct its_vpe *vpe) 4436 { 4437 its_vpe_db_proxy_unmap(vpe); 4438 its_vpe_id_free(vpe->vpe_id); 4439 its_free_pending_table(vpe->vpt_page); 4440 } 4441 4442 static void its_vpe_irq_domain_free(struct irq_domain *domain, 4443 unsigned int virq, 4444 unsigned int nr_irqs) 4445 { 4446 struct its_vm *vm = domain->host_data; 4447 int i; 4448 4449 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 4450 4451 for (i = 0; i < nr_irqs; i++) { 4452 struct irq_data *data = irq_domain_get_irq_data(domain, 4453 virq + i); 4454 struct its_vpe *vpe = irq_data_get_irq_chip_data(data); 4455 4456 BUG_ON(vm != vpe->its_vm); 4457 4458 clear_bit(data->hwirq, vm->db_bitmap); 4459 its_vpe_teardown(vpe); 4460 irq_domain_reset_irq_data(data); 4461 } 4462 4463 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { 4464 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); 4465 its_free_prop_table(vm->vprop_page); 4466 } 4467 } 4468 4469 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 4470 unsigned int nr_irqs, void *args) 4471 { 4472 struct irq_chip *irqchip = &its_vpe_irq_chip; 4473 struct its_vm *vm = args; 4474 unsigned long *bitmap; 4475 struct page *vprop_page; 4476 int base, nr_ids, i, err = 0; 4477 4478 BUG_ON(!vm); 4479 4480 bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids); 4481 if (!bitmap) 4482 return -ENOMEM; 4483 4484 if (nr_ids < nr_irqs) { 4485 its_lpi_free(bitmap, base, nr_ids); 4486 return -ENOMEM; 4487 } 4488 4489 vprop_page = its_allocate_prop_table(GFP_KERNEL); 4490 if (!vprop_page) { 4491 its_lpi_free(bitmap, base, nr_ids); 4492 return -ENOMEM; 4493 } 4494 4495 vm->db_bitmap = bitmap; 4496 vm->db_lpi_base = base; 4497 vm->nr_db_lpis = nr_ids; 4498 vm->vprop_page = vprop_page; 4499 4500 if (gic_rdists->has_rvpeid) 4501 irqchip = &its_vpe_4_1_irq_chip; 4502 4503 for (i = 0; i < nr_irqs; i++) { 4504 vm->vpes[i]->vpe_db_lpi = base + i; 4505 err = its_vpe_init(vm->vpes[i]); 4506 if (err) 4507 break; 4508 err = its_irq_gic_domain_alloc(domain, virq + i, 4509 vm->vpes[i]->vpe_db_lpi); 4510 if (err) 4511 break; 4512 irq_domain_set_hwirq_and_chip(domain, virq + i, i, 4513 irqchip, vm->vpes[i]); 4514 set_bit(i, bitmap); 4515 } 4516 4517 if (err) { 4518 if (i > 0) 4519 its_vpe_irq_domain_free(domain, virq, i); 4520 4521 its_lpi_free(bitmap, base, nr_ids); 4522 its_free_prop_table(vprop_page); 4523 } 4524 4525 return err; 4526 } 4527 4528 static int its_vpe_irq_domain_activate(struct irq_domain *domain, 4529 struct irq_data *d, bool reserve) 4530 { 4531 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4532 struct its_node *its; 4533 4534 /* 4535 * If we use the list map, we issue VMAPP on demand... Unless 4536 * we're on a GICv4.1 and we eagerly map the VPE on all ITSs 4537 * so that VSGIs can work. 4538 */ 4539 if (!gic_requires_eager_mapping()) 4540 return 0; 4541 4542 /* Map the VPE to the first possible CPU */ 4543 vpe->col_idx = cpumask_first(cpu_online_mask); 4544 4545 list_for_each_entry(its, &its_nodes, entry) { 4546 if (!is_v4(its)) 4547 continue; 4548 4549 its_send_vmapp(its, vpe, true); 4550 its_send_vinvall(its, vpe); 4551 } 4552 4553 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); 4554 4555 return 0; 4556 } 4557 4558 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain, 4559 struct irq_data *d) 4560 { 4561 struct its_vpe *vpe = irq_data_get_irq_chip_data(d); 4562 struct its_node *its; 4563 4564 /* 4565 * If we use the list map on GICv4.0, we unmap the VPE once no 4566 * VLPIs are associated with the VM. 4567 */ 4568 if (!gic_requires_eager_mapping()) 4569 return; 4570 4571 list_for_each_entry(its, &its_nodes, entry) { 4572 if (!is_v4(its)) 4573 continue; 4574 4575 its_send_vmapp(its, vpe, false); 4576 } 4577 4578 /* 4579 * There may be a direct read to the VPT after unmapping the 4580 * vPE, to guarantee the validity of this, we make the VPT 4581 * memory coherent with the CPU caches here. 4582 */ 4583 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count)) 4584 gic_flush_dcache_to_poc(page_address(vpe->vpt_page), 4585 LPI_PENDBASE_SZ); 4586 } 4587 4588 static const struct irq_domain_ops its_vpe_domain_ops = { 4589 .alloc = its_vpe_irq_domain_alloc, 4590 .free = its_vpe_irq_domain_free, 4591 .activate = its_vpe_irq_domain_activate, 4592 .deactivate = its_vpe_irq_domain_deactivate, 4593 }; 4594 4595 static int its_force_quiescent(void __iomem *base) 4596 { 4597 u32 count = 1000000; /* 1s */ 4598 u32 val; 4599 4600 val = readl_relaxed(base + GITS_CTLR); 4601 /* 4602 * GIC architecture specification requires the ITS to be both 4603 * disabled and quiescent for writes to GITS_BASER<n> or 4604 * GITS_CBASER to not have UNPREDICTABLE results. 4605 */ 4606 if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE)) 4607 return 0; 4608 4609 /* Disable the generation of all interrupts to this ITS */ 4610 val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe); 4611 writel_relaxed(val, base + GITS_CTLR); 4612 4613 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 4614 while (1) { 4615 val = readl_relaxed(base + GITS_CTLR); 4616 if (val & GITS_CTLR_QUIESCENT) 4617 return 0; 4618 4619 count--; 4620 if (!count) 4621 return -EBUSY; 4622 4623 cpu_relax(); 4624 udelay(1); 4625 } 4626 } 4627 4628 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data) 4629 { 4630 struct its_node *its = data; 4631 4632 /* erratum 22375: only alloc 8MB table size (20 bits) */ 4633 its->typer &= ~GITS_TYPER_DEVBITS; 4634 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); 4635 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; 4636 4637 return true; 4638 } 4639 4640 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data) 4641 { 4642 struct its_node *its = data; 4643 4644 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; 4645 4646 return true; 4647 } 4648 4649 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data) 4650 { 4651 struct its_node *its = data; 4652 4653 /* On QDF2400, the size of the ITE is 16Bytes */ 4654 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; 4655 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); 4656 4657 return true; 4658 } 4659 4660 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev) 4661 { 4662 struct its_node *its = its_dev->its; 4663 4664 /* 4665 * The Socionext Synquacer SoC has a so-called 'pre-ITS', 4666 * which maps 32-bit writes targeted at a separate window of 4667 * size '4 << device_id_bits' onto writes to GITS_TRANSLATER 4668 * with device ID taken from bits [device_id_bits + 1:2] of 4669 * the window offset. 4670 */ 4671 return its->pre_its_base + (its_dev->device_id << 2); 4672 } 4673 4674 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data) 4675 { 4676 struct its_node *its = data; 4677 u32 pre_its_window[2]; 4678 u32 ids; 4679 4680 if (!fwnode_property_read_u32_array(its->fwnode_handle, 4681 "socionext,synquacer-pre-its", 4682 pre_its_window, 4683 ARRAY_SIZE(pre_its_window))) { 4684 4685 its->pre_its_base = pre_its_window[0]; 4686 its->get_msi_base = its_irq_get_msi_base_pre_its; 4687 4688 ids = ilog2(pre_its_window[1]) - 2; 4689 if (device_ids(its) > ids) { 4690 its->typer &= ~GITS_TYPER_DEVBITS; 4691 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); 4692 } 4693 4694 /* the pre-ITS breaks isolation, so disable MSI remapping */ 4695 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP; 4696 return true; 4697 } 4698 return false; 4699 } 4700 4701 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data) 4702 { 4703 struct its_node *its = data; 4704 4705 /* 4706 * Hip07 insists on using the wrong address for the VLPI 4707 * page. Trick it into doing the right thing... 4708 */ 4709 its->vlpi_redist_offset = SZ_128K; 4710 return true; 4711 } 4712 4713 static const struct gic_quirk its_quirks[] = { 4714 #ifdef CONFIG_CAVIUM_ERRATUM_22375 4715 { 4716 .desc = "ITS: Cavium errata 22375, 24313", 4717 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 4718 .mask = 0xffff0fff, 4719 .init = its_enable_quirk_cavium_22375, 4720 }, 4721 #endif 4722 #ifdef CONFIG_CAVIUM_ERRATUM_23144 4723 { 4724 .desc = "ITS: Cavium erratum 23144", 4725 .iidr = 0xa100034c, /* ThunderX pass 1.x */ 4726 .mask = 0xffff0fff, 4727 .init = its_enable_quirk_cavium_23144, 4728 }, 4729 #endif 4730 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065 4731 { 4732 .desc = "ITS: QDF2400 erratum 0065", 4733 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */ 4734 .mask = 0xffffffff, 4735 .init = its_enable_quirk_qdf2400_e0065, 4736 }, 4737 #endif 4738 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS 4739 { 4740 /* 4741 * The Socionext Synquacer SoC incorporates ARM's own GIC-500 4742 * implementation, but with a 'pre-ITS' added that requires 4743 * special handling in software. 4744 */ 4745 .desc = "ITS: Socionext Synquacer pre-ITS", 4746 .iidr = 0x0001143b, 4747 .mask = 0xffffffff, 4748 .init = its_enable_quirk_socionext_synquacer, 4749 }, 4750 #endif 4751 #ifdef CONFIG_HISILICON_ERRATUM_161600802 4752 { 4753 .desc = "ITS: Hip07 erratum 161600802", 4754 .iidr = 0x00000004, 4755 .mask = 0xffffffff, 4756 .init = its_enable_quirk_hip07_161600802, 4757 }, 4758 #endif 4759 { 4760 } 4761 }; 4762 4763 static void its_enable_quirks(struct its_node *its) 4764 { 4765 u32 iidr = readl_relaxed(its->base + GITS_IIDR); 4766 4767 gic_enable_quirks(iidr, its_quirks, its); 4768 } 4769 4770 static int its_save_disable(void) 4771 { 4772 struct its_node *its; 4773 int err = 0; 4774 4775 raw_spin_lock(&its_lock); 4776 list_for_each_entry(its, &its_nodes, entry) { 4777 void __iomem *base; 4778 4779 base = its->base; 4780 its->ctlr_save = readl_relaxed(base + GITS_CTLR); 4781 err = its_force_quiescent(base); 4782 if (err) { 4783 pr_err("ITS@%pa: failed to quiesce: %d\n", 4784 &its->phys_base, err); 4785 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4786 goto err; 4787 } 4788 4789 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); 4790 } 4791 4792 err: 4793 if (err) { 4794 list_for_each_entry_continue_reverse(its, &its_nodes, entry) { 4795 void __iomem *base; 4796 4797 base = its->base; 4798 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4799 } 4800 } 4801 raw_spin_unlock(&its_lock); 4802 4803 return err; 4804 } 4805 4806 static void its_restore_enable(void) 4807 { 4808 struct its_node *its; 4809 int ret; 4810 4811 raw_spin_lock(&its_lock); 4812 list_for_each_entry(its, &its_nodes, entry) { 4813 void __iomem *base; 4814 int i; 4815 4816 base = its->base; 4817 4818 /* 4819 * Make sure that the ITS is disabled. If it fails to quiesce, 4820 * don't restore it since writing to CBASER or BASER<n> 4821 * registers is undefined according to the GIC v3 ITS 4822 * Specification. 4823 * 4824 * Firmware resuming with the ITS enabled is terminally broken. 4825 */ 4826 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE); 4827 ret = its_force_quiescent(base); 4828 if (ret) { 4829 pr_err("ITS@%pa: failed to quiesce on resume: %d\n", 4830 &its->phys_base, ret); 4831 continue; 4832 } 4833 4834 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); 4835 4836 /* 4837 * Writing CBASER resets CREADR to 0, so make CWRITER and 4838 * cmd_write line up with it. 4839 */ 4840 its->cmd_write = its->cmd_base; 4841 gits_write_cwriter(0, base + GITS_CWRITER); 4842 4843 /* Restore GITS_BASER from the value cache. */ 4844 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 4845 struct its_baser *baser = &its->tables[i]; 4846 4847 if (!(baser->val & GITS_BASER_VALID)) 4848 continue; 4849 4850 its_write_baser(its, baser, baser->val); 4851 } 4852 writel_relaxed(its->ctlr_save, base + GITS_CTLR); 4853 4854 /* 4855 * Reinit the collection if it's stored in the ITS. This is 4856 * indicated by the col_id being less than the HCC field. 4857 * CID < HCC as specified in the GIC v3 Documentation. 4858 */ 4859 if (its->collections[smp_processor_id()].col_id < 4860 GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER))) 4861 its_cpu_init_collection(its); 4862 } 4863 raw_spin_unlock(&its_lock); 4864 } 4865 4866 static struct syscore_ops its_syscore_ops = { 4867 .suspend = its_save_disable, 4868 .resume = its_restore_enable, 4869 }; 4870 4871 static void __init __iomem *its_map_one(struct resource *res, int *err) 4872 { 4873 void __iomem *its_base; 4874 u32 val; 4875 4876 its_base = ioremap(res->start, SZ_64K); 4877 if (!its_base) { 4878 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); 4879 *err = -ENOMEM; 4880 return NULL; 4881 } 4882 4883 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 4884 if (val != 0x30 && val != 0x40) { 4885 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); 4886 *err = -ENODEV; 4887 goto out_unmap; 4888 } 4889 4890 *err = its_force_quiescent(its_base); 4891 if (*err) { 4892 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); 4893 goto out_unmap; 4894 } 4895 4896 return its_base; 4897 4898 out_unmap: 4899 iounmap(its_base); 4900 return NULL; 4901 } 4902 4903 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) 4904 { 4905 struct irq_domain *inner_domain; 4906 struct msi_domain_info *info; 4907 4908 info = kzalloc(sizeof(*info), GFP_KERNEL); 4909 if (!info) 4910 return -ENOMEM; 4911 4912 inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its); 4913 if (!inner_domain) { 4914 kfree(info); 4915 return -ENOMEM; 4916 } 4917 4918 inner_domain->parent = its_parent; 4919 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); 4920 inner_domain->flags |= its->msi_domain_flags; 4921 info->ops = &its_msi_domain_ops; 4922 info->data = its; 4923 inner_domain->host_data = info; 4924 4925 return 0; 4926 } 4927 4928 static int its_init_vpe_domain(void) 4929 { 4930 struct its_node *its; 4931 u32 devid; 4932 int entries; 4933 4934 if (gic_rdists->has_direct_lpi) { 4935 pr_info("ITS: Using DirectLPI for VPE invalidation\n"); 4936 return 0; 4937 } 4938 4939 /* Any ITS will do, even if not v4 */ 4940 its = list_first_entry(&its_nodes, struct its_node, entry); 4941 4942 entries = roundup_pow_of_two(nr_cpu_ids); 4943 vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes), 4944 GFP_KERNEL); 4945 if (!vpe_proxy.vpes) 4946 return -ENOMEM; 4947 4948 /* Use the last possible DevID */ 4949 devid = GENMASK(device_ids(its) - 1, 0); 4950 vpe_proxy.dev = its_create_device(its, devid, entries, false); 4951 if (!vpe_proxy.dev) { 4952 kfree(vpe_proxy.vpes); 4953 pr_err("ITS: Can't allocate GICv4 proxy device\n"); 4954 return -ENOMEM; 4955 } 4956 4957 BUG_ON(entries > vpe_proxy.dev->nr_ites); 4958 4959 raw_spin_lock_init(&vpe_proxy.lock); 4960 vpe_proxy.next_victim = 0; 4961 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n", 4962 devid, vpe_proxy.dev->nr_ites); 4963 4964 return 0; 4965 } 4966 4967 static int __init its_compute_its_list_map(struct resource *res, 4968 void __iomem *its_base) 4969 { 4970 int its_number; 4971 u32 ctlr; 4972 4973 /* 4974 * This is assumed to be done early enough that we're 4975 * guaranteed to be single-threaded, hence no 4976 * locking. Should this change, we should address 4977 * this. 4978 */ 4979 its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX); 4980 if (its_number >= GICv4_ITS_LIST_MAX) { 4981 pr_err("ITS@%pa: No ITSList entry available!\n", 4982 &res->start); 4983 return -EINVAL; 4984 } 4985 4986 ctlr = readl_relaxed(its_base + GITS_CTLR); 4987 ctlr &= ~GITS_CTLR_ITS_NUMBER; 4988 ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT; 4989 writel_relaxed(ctlr, its_base + GITS_CTLR); 4990 ctlr = readl_relaxed(its_base + GITS_CTLR); 4991 if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) { 4992 its_number = ctlr & GITS_CTLR_ITS_NUMBER; 4993 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT; 4994 } 4995 4996 if (test_and_set_bit(its_number, &its_list_map)) { 4997 pr_err("ITS@%pa: Duplicate ITSList entry %d\n", 4998 &res->start, its_number); 4999 return -EINVAL; 5000 } 5001 5002 return its_number; 5003 } 5004 5005 static int __init its_probe_one(struct resource *res, 5006 struct fwnode_handle *handle, int numa_node) 5007 { 5008 struct its_node *its; 5009 void __iomem *its_base; 5010 u64 baser, tmp, typer; 5011 struct page *page; 5012 u32 ctlr; 5013 int err; 5014 5015 its_base = its_map_one(res, &err); 5016 if (!its_base) 5017 return err; 5018 5019 pr_info("ITS %pR\n", res); 5020 5021 its = kzalloc(sizeof(*its), GFP_KERNEL); 5022 if (!its) { 5023 err = -ENOMEM; 5024 goto out_unmap; 5025 } 5026 5027 raw_spin_lock_init(&its->lock); 5028 mutex_init(&its->dev_alloc_lock); 5029 INIT_LIST_HEAD(&its->entry); 5030 INIT_LIST_HEAD(&its->its_device_list); 5031 typer = gic_read_typer(its_base + GITS_TYPER); 5032 its->typer = typer; 5033 its->base = its_base; 5034 its->phys_base = res->start; 5035 if (is_v4(its)) { 5036 if (!(typer & GITS_TYPER_VMOVP)) { 5037 err = its_compute_its_list_map(res, its_base); 5038 if (err < 0) 5039 goto out_free_its; 5040 5041 its->list_nr = err; 5042 5043 pr_info("ITS@%pa: Using ITS number %d\n", 5044 &res->start, err); 5045 } else { 5046 pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); 5047 } 5048 5049 if (is_v4_1(its)) { 5050 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); 5051 5052 its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K); 5053 if (!its->sgir_base) { 5054 err = -ENOMEM; 5055 goto out_free_its; 5056 } 5057 5058 its->mpidr = readl_relaxed(its_base + GITS_MPIDR); 5059 5060 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", 5061 &res->start, its->mpidr, svpet); 5062 } 5063 } 5064 5065 its->numa_node = numa_node; 5066 5067 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, 5068 get_order(ITS_CMD_QUEUE_SZ)); 5069 if (!page) { 5070 err = -ENOMEM; 5071 goto out_unmap_sgir; 5072 } 5073 its->cmd_base = (void *)page_address(page); 5074 its->cmd_write = its->cmd_base; 5075 its->fwnode_handle = handle; 5076 its->get_msi_base = its_irq_get_msi_base; 5077 its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP; 5078 5079 its_enable_quirks(its); 5080 5081 err = its_alloc_tables(its); 5082 if (err) 5083 goto out_free_cmd; 5084 5085 err = its_alloc_collections(its); 5086 if (err) 5087 goto out_free_tables; 5088 5089 baser = (virt_to_phys(its->cmd_base) | 5090 GITS_CBASER_RaWaWb | 5091 GITS_CBASER_InnerShareable | 5092 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 5093 GITS_CBASER_VALID); 5094 5095 gits_write_cbaser(baser, its->base + GITS_CBASER); 5096 tmp = gits_read_cbaser(its->base + GITS_CBASER); 5097 5098 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 5099 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 5100 /* 5101 * The HW reports non-shareable, we must 5102 * remove the cacheability attributes as 5103 * well. 5104 */ 5105 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 5106 GITS_CBASER_CACHEABILITY_MASK); 5107 baser |= GITS_CBASER_nC; 5108 gits_write_cbaser(baser, its->base + GITS_CBASER); 5109 } 5110 pr_info("ITS: using cache flushing for cmd queue\n"); 5111 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 5112 } 5113 5114 gits_write_cwriter(0, its->base + GITS_CWRITER); 5115 ctlr = readl_relaxed(its->base + GITS_CTLR); 5116 ctlr |= GITS_CTLR_ENABLE; 5117 if (is_v4(its)) 5118 ctlr |= GITS_CTLR_ImDe; 5119 writel_relaxed(ctlr, its->base + GITS_CTLR); 5120 5121 err = its_init_domain(handle, its); 5122 if (err) 5123 goto out_free_tables; 5124 5125 raw_spin_lock(&its_lock); 5126 list_add(&its->entry, &its_nodes); 5127 raw_spin_unlock(&its_lock); 5128 5129 return 0; 5130 5131 out_free_tables: 5132 its_free_tables(its); 5133 out_free_cmd: 5134 free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); 5135 out_unmap_sgir: 5136 if (its->sgir_base) 5137 iounmap(its->sgir_base); 5138 out_free_its: 5139 kfree(its); 5140 out_unmap: 5141 iounmap(its_base); 5142 pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err); 5143 return err; 5144 } 5145 5146 static bool gic_rdists_supports_plpis(void) 5147 { 5148 return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 5149 } 5150 5151 static int redist_disable_lpis(void) 5152 { 5153 void __iomem *rbase = gic_data_rdist_rd_base(); 5154 u64 timeout = USEC_PER_SEC; 5155 u64 val; 5156 5157 if (!gic_rdists_supports_plpis()) { 5158 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 5159 return -ENXIO; 5160 } 5161 5162 val = readl_relaxed(rbase + GICR_CTLR); 5163 if (!(val & GICR_CTLR_ENABLE_LPIS)) 5164 return 0; 5165 5166 /* 5167 * If coming via a CPU hotplug event, we don't need to disable 5168 * LPIs before trying to re-enable them. They are already 5169 * configured and all is well in the world. 5170 * 5171 * If running with preallocated tables, there is nothing to do. 5172 */ 5173 if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) || 5174 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) 5175 return 0; 5176 5177 /* 5178 * From that point on, we only try to do some damage control. 5179 */ 5180 pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n", 5181 smp_processor_id()); 5182 add_taint(TAINT_CRAP, LOCKDEP_STILL_OK); 5183 5184 /* Disable LPIs */ 5185 val &= ~GICR_CTLR_ENABLE_LPIS; 5186 writel_relaxed(val, rbase + GICR_CTLR); 5187 5188 /* Make sure any change to GICR_CTLR is observable by the GIC */ 5189 dsb(sy); 5190 5191 /* 5192 * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs 5193 * from 1 to 0 before programming GICR_PEND{PROP}BASER registers. 5194 * Error out if we time out waiting for RWP to clear. 5195 */ 5196 while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) { 5197 if (!timeout) { 5198 pr_err("CPU%d: Timeout while disabling LPIs\n", 5199 smp_processor_id()); 5200 return -ETIMEDOUT; 5201 } 5202 udelay(1); 5203 timeout--; 5204 } 5205 5206 /* 5207 * After it has been written to 1, it is IMPLEMENTATION 5208 * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be 5209 * cleared to 0. Error out if clearing the bit failed. 5210 */ 5211 if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) { 5212 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id()); 5213 return -EBUSY; 5214 } 5215 5216 return 0; 5217 } 5218 5219 int its_cpu_init(void) 5220 { 5221 if (!list_empty(&its_nodes)) { 5222 int ret; 5223 5224 ret = redist_disable_lpis(); 5225 if (ret) 5226 return ret; 5227 5228 its_cpu_init_lpis(); 5229 its_cpu_init_collections(); 5230 } 5231 5232 return 0; 5233 } 5234 5235 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work) 5236 { 5237 cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); 5238 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; 5239 } 5240 5241 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work, 5242 rdist_memreserve_cpuhp_cleanup_workfn); 5243 5244 static int its_cpu_memreserve_lpi(unsigned int cpu) 5245 { 5246 struct page *pend_page; 5247 int ret = 0; 5248 5249 /* This gets to run exactly once per CPU */ 5250 if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE) 5251 return 0; 5252 5253 pend_page = gic_data_rdist()->pend_page; 5254 if (WARN_ON(!pend_page)) { 5255 ret = -ENOMEM; 5256 goto out; 5257 } 5258 /* 5259 * If the pending table was pre-programmed, free the memory we 5260 * preemptively allocated. Otherwise, reserve that memory for 5261 * later kexecs. 5262 */ 5263 if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) { 5264 its_free_pending_table(pend_page); 5265 gic_data_rdist()->pend_page = NULL; 5266 } else { 5267 phys_addr_t paddr = page_to_phys(pend_page); 5268 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ)); 5269 } 5270 5271 out: 5272 /* Last CPU being brought up gets to issue the cleanup */ 5273 if (!IS_ENABLED(CONFIG_SMP) || 5274 cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask)) 5275 schedule_work(&rdist_memreserve_cpuhp_cleanup_work); 5276 5277 gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; 5278 return ret; 5279 } 5280 5281 /* Mark all the BASER registers as invalid before they get reprogrammed */ 5282 static int __init its_reset_one(struct resource *res) 5283 { 5284 void __iomem *its_base; 5285 int err, i; 5286 5287 its_base = its_map_one(res, &err); 5288 if (!its_base) 5289 return err; 5290 5291 for (i = 0; i < GITS_BASER_NR_REGS; i++) 5292 gits_write_baser(0, its_base + GITS_BASER + (i << 3)); 5293 5294 iounmap(its_base); 5295 return 0; 5296 } 5297 5298 static const struct of_device_id its_device_id[] = { 5299 { .compatible = "arm,gic-v3-its", }, 5300 {}, 5301 }; 5302 5303 static int __init its_of_probe(struct device_node *node) 5304 { 5305 struct device_node *np; 5306 struct resource res; 5307 5308 /* 5309 * Make sure *all* the ITS are reset before we probe any, as 5310 * they may be sharing memory. If any of the ITS fails to 5311 * reset, don't even try to go any further, as this could 5312 * result in something even worse. 5313 */ 5314 for (np = of_find_matching_node(node, its_device_id); np; 5315 np = of_find_matching_node(np, its_device_id)) { 5316 int err; 5317 5318 if (!of_device_is_available(np) || 5319 !of_property_read_bool(np, "msi-controller") || 5320 of_address_to_resource(np, 0, &res)) 5321 continue; 5322 5323 err = its_reset_one(&res); 5324 if (err) 5325 return err; 5326 } 5327 5328 for (np = of_find_matching_node(node, its_device_id); np; 5329 np = of_find_matching_node(np, its_device_id)) { 5330 if (!of_device_is_available(np)) 5331 continue; 5332 if (!of_property_read_bool(np, "msi-controller")) { 5333 pr_warn("%pOF: no msi-controller property, ITS ignored\n", 5334 np); 5335 continue; 5336 } 5337 5338 if (of_address_to_resource(np, 0, &res)) { 5339 pr_warn("%pOF: no regs?\n", np); 5340 continue; 5341 } 5342 5343 its_probe_one(&res, &np->fwnode, of_node_to_nid(np)); 5344 } 5345 return 0; 5346 } 5347 5348 #ifdef CONFIG_ACPI 5349 5350 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K) 5351 5352 #ifdef CONFIG_ACPI_NUMA 5353 struct its_srat_map { 5354 /* numa node id */ 5355 u32 numa_node; 5356 /* GIC ITS ID */ 5357 u32 its_id; 5358 }; 5359 5360 static struct its_srat_map *its_srat_maps __initdata; 5361 static int its_in_srat __initdata; 5362 5363 static int __init acpi_get_its_numa_node(u32 its_id) 5364 { 5365 int i; 5366 5367 for (i = 0; i < its_in_srat; i++) { 5368 if (its_id == its_srat_maps[i].its_id) 5369 return its_srat_maps[i].numa_node; 5370 } 5371 return NUMA_NO_NODE; 5372 } 5373 5374 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header, 5375 const unsigned long end) 5376 { 5377 return 0; 5378 } 5379 5380 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header, 5381 const unsigned long end) 5382 { 5383 int node; 5384 struct acpi_srat_gic_its_affinity *its_affinity; 5385 5386 its_affinity = (struct acpi_srat_gic_its_affinity *)header; 5387 if (!its_affinity) 5388 return -EINVAL; 5389 5390 if (its_affinity->header.length < sizeof(*its_affinity)) { 5391 pr_err("SRAT: Invalid header length %d in ITS affinity\n", 5392 its_affinity->header.length); 5393 return -EINVAL; 5394 } 5395 5396 /* 5397 * Note that in theory a new proximity node could be created by this 5398 * entry as it is an SRAT resource allocation structure. 5399 * We do not currently support doing so. 5400 */ 5401 node = pxm_to_node(its_affinity->proximity_domain); 5402 5403 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) { 5404 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); 5405 return 0; 5406 } 5407 5408 its_srat_maps[its_in_srat].numa_node = node; 5409 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; 5410 its_in_srat++; 5411 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", 5412 its_affinity->proximity_domain, its_affinity->its_id, node); 5413 5414 return 0; 5415 } 5416 5417 static void __init acpi_table_parse_srat_its(void) 5418 { 5419 int count; 5420 5421 count = acpi_table_parse_entries(ACPI_SIG_SRAT, 5422 sizeof(struct acpi_table_srat), 5423 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 5424 gic_acpi_match_srat_its, 0); 5425 if (count <= 0) 5426 return; 5427 5428 its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map), 5429 GFP_KERNEL); 5430 if (!its_srat_maps) 5431 return; 5432 5433 acpi_table_parse_entries(ACPI_SIG_SRAT, 5434 sizeof(struct acpi_table_srat), 5435 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY, 5436 gic_acpi_parse_srat_its, 0); 5437 } 5438 5439 /* free the its_srat_maps after ITS probing */ 5440 static void __init acpi_its_srat_maps_free(void) 5441 { 5442 kfree(its_srat_maps); 5443 } 5444 #else 5445 static void __init acpi_table_parse_srat_its(void) { } 5446 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; } 5447 static void __init acpi_its_srat_maps_free(void) { } 5448 #endif 5449 5450 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, 5451 const unsigned long end) 5452 { 5453 struct acpi_madt_generic_translator *its_entry; 5454 struct fwnode_handle *dom_handle; 5455 struct resource res; 5456 int err; 5457 5458 its_entry = (struct acpi_madt_generic_translator *)header; 5459 memset(&res, 0, sizeof(res)); 5460 res.start = its_entry->base_address; 5461 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; 5462 res.flags = IORESOURCE_MEM; 5463 5464 dom_handle = irq_domain_alloc_fwnode(&res.start); 5465 if (!dom_handle) { 5466 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n", 5467 &res.start); 5468 return -ENOMEM; 5469 } 5470 5471 err = iort_register_domain_token(its_entry->translation_id, res.start, 5472 dom_handle); 5473 if (err) { 5474 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", 5475 &res.start, its_entry->translation_id); 5476 goto dom_err; 5477 } 5478 5479 err = its_probe_one(&res, dom_handle, 5480 acpi_get_its_numa_node(its_entry->translation_id)); 5481 if (!err) 5482 return 0; 5483 5484 iort_deregister_domain_token(its_entry->translation_id); 5485 dom_err: 5486 irq_domain_free_fwnode(dom_handle); 5487 return err; 5488 } 5489 5490 static int __init its_acpi_reset(union acpi_subtable_headers *header, 5491 const unsigned long end) 5492 { 5493 struct acpi_madt_generic_translator *its_entry; 5494 struct resource res; 5495 5496 its_entry = (struct acpi_madt_generic_translator *)header; 5497 res = (struct resource) { 5498 .start = its_entry->base_address, 5499 .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1, 5500 .flags = IORESOURCE_MEM, 5501 }; 5502 5503 return its_reset_one(&res); 5504 } 5505 5506 static void __init its_acpi_probe(void) 5507 { 5508 acpi_table_parse_srat_its(); 5509 /* 5510 * Make sure *all* the ITS are reset before we probe any, as 5511 * they may be sharing memory. If any of the ITS fails to 5512 * reset, don't even try to go any further, as this could 5513 * result in something even worse. 5514 */ 5515 if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 5516 its_acpi_reset, 0) > 0) 5517 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 5518 gic_acpi_parse_madt_its, 0); 5519 acpi_its_srat_maps_free(); 5520 } 5521 #else 5522 static void __init its_acpi_probe(void) { } 5523 #endif 5524 5525 int __init its_lpi_memreserve_init(void) 5526 { 5527 int state; 5528 5529 if (!efi_enabled(EFI_CONFIG_TABLES)) 5530 return 0; 5531 5532 if (list_empty(&its_nodes)) 5533 return 0; 5534 5535 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; 5536 state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 5537 "irqchip/arm/gicv3/memreserve:online", 5538 its_cpu_memreserve_lpi, 5539 NULL); 5540 if (state < 0) 5541 return state; 5542 5543 gic_rdists->cpuhp_memreserve_state = state; 5544 5545 return 0; 5546 } 5547 5548 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, 5549 struct irq_domain *parent_domain) 5550 { 5551 struct device_node *of_node; 5552 struct its_node *its; 5553 bool has_v4 = false; 5554 bool has_v4_1 = false; 5555 int err; 5556 5557 gic_rdists = rdists; 5558 5559 its_parent = parent_domain; 5560 of_node = to_of_node(handle); 5561 if (of_node) 5562 its_of_probe(of_node); 5563 else 5564 its_acpi_probe(); 5565 5566 if (list_empty(&its_nodes)) { 5567 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 5568 return -ENXIO; 5569 } 5570 5571 err = allocate_lpi_tables(); 5572 if (err) 5573 return err; 5574 5575 list_for_each_entry(its, &its_nodes, entry) { 5576 has_v4 |= is_v4(its); 5577 has_v4_1 |= is_v4_1(its); 5578 } 5579 5580 /* Don't bother with inconsistent systems */ 5581 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) 5582 rdists->has_rvpeid = false; 5583 5584 if (has_v4 & rdists->has_vlpis) { 5585 const struct irq_domain_ops *sgi_ops; 5586 5587 if (has_v4_1) 5588 sgi_ops = &its_sgi_domain_ops; 5589 else 5590 sgi_ops = NULL; 5591 5592 if (its_init_vpe_domain() || 5593 its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) { 5594 rdists->has_vlpis = false; 5595 pr_err("ITS: Disabling GICv4 support\n"); 5596 } 5597 } 5598 5599 register_syscore_ops(&its_syscore_ops); 5600 5601 return 0; 5602 } 5603