15ef3166eSFrederic Barrat // SPDX-License-Identifier: GPL-2.0+ 25ef3166eSFrederic Barrat // Copyright 2017 IBM Corp. 35ef3166eSFrederic Barrat #include <linux/sched/mm.h> 45ef3166eSFrederic Barrat #include <linux/mutex.h> 550a7ca3cSSouptick Joarder #include <linux/mm_types.h> 65ef3166eSFrederic Barrat #include <linux/mmu_context.h> 75ef3166eSFrederic Barrat #include <asm/copro.h> 85ef3166eSFrederic Barrat #include <asm/pnv-ocxl.h> 9dde6f18aSFrederic Barrat #include <asm/xive.h> 10280b983cSFrederic Barrat #include <misc/ocxl.h> 115ef3166eSFrederic Barrat #include "ocxl_internal.h" 1292add22eSFrederic Barrat #include "trace.h" 135ef3166eSFrederic Barrat 145ef3166eSFrederic Barrat 155ef3166eSFrederic Barrat #define SPA_PASID_BITS 15 165ef3166eSFrederic Barrat #define SPA_PASID_MAX ((1 << SPA_PASID_BITS) - 1) 175ef3166eSFrederic Barrat #define SPA_PE_MASK SPA_PASID_MAX 185ef3166eSFrederic Barrat #define SPA_SPA_SIZE_LOG 22 /* Each SPA is 4 Mb */ 195ef3166eSFrederic Barrat 205ef3166eSFrederic Barrat #define SPA_CFG_SF (1ull << (63-0)) 215ef3166eSFrederic Barrat #define SPA_CFG_TA (1ull << (63-1)) 225ef3166eSFrederic Barrat #define SPA_CFG_HV (1ull << (63-3)) 235ef3166eSFrederic Barrat #define SPA_CFG_UV (1ull << (63-4)) 245ef3166eSFrederic Barrat #define SPA_CFG_XLAT_hpt (0ull << (63-6)) /* Hashed page table (HPT) mode */ 255ef3166eSFrederic Barrat #define SPA_CFG_XLAT_roh (2ull << (63-6)) /* Radix on HPT mode */ 265ef3166eSFrederic Barrat #define SPA_CFG_XLAT_ror (3ull << (63-6)) /* Radix on Radix mode */ 275ef3166eSFrederic Barrat #define SPA_CFG_PR (1ull << (63-49)) 285ef3166eSFrederic Barrat #define SPA_CFG_TC (1ull << (63-54)) 295ef3166eSFrederic Barrat #define SPA_CFG_DR (1ull << (63-59)) 305ef3166eSFrederic Barrat 315ef3166eSFrederic Barrat #define SPA_XSL_TF (1ull << (63-3)) /* Translation fault */ 325ef3166eSFrederic Barrat #define SPA_XSL_S (1ull << (63-38)) /* Store operation */ 335ef3166eSFrederic Barrat 345ef3166eSFrederic Barrat #define SPA_PE_VALID 0x80000000 355ef3166eSFrederic Barrat 365ef3166eSFrederic Barrat 375ef3166eSFrederic Barrat struct pe_data { 385ef3166eSFrederic Barrat struct mm_struct *mm; 395ef3166eSFrederic Barrat /* callback to trigger when a translation fault occurs */ 405ef3166eSFrederic Barrat void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr); 415ef3166eSFrederic Barrat /* opaque pointer to be passed to the above callback */ 425ef3166eSFrederic Barrat void *xsl_err_data; 435ef3166eSFrederic Barrat struct rcu_head rcu; 445ef3166eSFrederic Barrat }; 455ef3166eSFrederic Barrat 465ef3166eSFrederic Barrat struct spa { 475ef3166eSFrederic Barrat struct ocxl_process_element *spa_mem; 485ef3166eSFrederic Barrat int spa_order; 495ef3166eSFrederic Barrat struct mutex spa_lock; 505ef3166eSFrederic Barrat struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */ 515ef3166eSFrederic Barrat char *irq_name; 525ef3166eSFrederic Barrat int virq; 535ef3166eSFrederic Barrat void __iomem *reg_dsisr; 545ef3166eSFrederic Barrat void __iomem *reg_dar; 555ef3166eSFrederic Barrat void __iomem *reg_tfc; 565ef3166eSFrederic Barrat void __iomem *reg_pe_handle; 575ef3166eSFrederic Barrat /* 585ef3166eSFrederic Barrat * The following field are used by the memory fault 595ef3166eSFrederic Barrat * interrupt handler. We can only have one interrupt at a 605ef3166eSFrederic Barrat * time. The NPU won't raise another interrupt until the 615ef3166eSFrederic Barrat * previous one has been ack'd by writing to the TFC register 625ef3166eSFrederic Barrat */ 635ef3166eSFrederic Barrat struct xsl_fault { 645ef3166eSFrederic Barrat struct work_struct fault_work; 655ef3166eSFrederic Barrat u64 pe; 665ef3166eSFrederic Barrat u64 dsisr; 675ef3166eSFrederic Barrat u64 dar; 685ef3166eSFrederic Barrat struct pe_data pe_data; 695ef3166eSFrederic Barrat } xsl_fault; 705ef3166eSFrederic Barrat }; 715ef3166eSFrederic Barrat 725ef3166eSFrederic Barrat /* 735ef3166eSFrederic Barrat * A opencapi link can be used be by several PCI functions. We have 745ef3166eSFrederic Barrat * one link per device slot. 755ef3166eSFrederic Barrat * 765ef3166eSFrederic Barrat * A linked list of opencapi links should suffice, as there's a 775ef3166eSFrederic Barrat * limited number of opencapi slots on a system and lookup is only 785ef3166eSFrederic Barrat * done when the device is probed 795ef3166eSFrederic Barrat */ 809c4ae064SAlastair D'Silva struct ocxl_link { 815ef3166eSFrederic Barrat struct list_head list; 825ef3166eSFrederic Barrat struct kref ref; 835ef3166eSFrederic Barrat int domain; 845ef3166eSFrederic Barrat int bus; 855ef3166eSFrederic Barrat int dev; 865ef3166eSFrederic Barrat atomic_t irq_available; 875ef3166eSFrederic Barrat struct spa *spa; 885ef3166eSFrederic Barrat void *platform_data; 895ef3166eSFrederic Barrat }; 905ef3166eSFrederic Barrat static struct list_head links_list = LIST_HEAD_INIT(links_list); 915ef3166eSFrederic Barrat static DEFINE_MUTEX(links_list_lock); 925ef3166eSFrederic Barrat 935ef3166eSFrederic Barrat enum xsl_response { 945ef3166eSFrederic Barrat CONTINUE, 955ef3166eSFrederic Barrat ADDRESS_ERROR, 965ef3166eSFrederic Barrat RESTART, 975ef3166eSFrederic Barrat }; 985ef3166eSFrederic Barrat 995ef3166eSFrederic Barrat 1005ef3166eSFrederic Barrat static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe) 1015ef3166eSFrederic Barrat { 1025ef3166eSFrederic Barrat u64 reg; 1035ef3166eSFrederic Barrat 1045ef3166eSFrederic Barrat *dsisr = in_be64(spa->reg_dsisr); 1055ef3166eSFrederic Barrat *dar = in_be64(spa->reg_dar); 1065ef3166eSFrederic Barrat reg = in_be64(spa->reg_pe_handle); 1075ef3166eSFrederic Barrat *pe = reg & SPA_PE_MASK; 1085ef3166eSFrederic Barrat } 1095ef3166eSFrederic Barrat 1105ef3166eSFrederic Barrat static void ack_irq(struct spa *spa, enum xsl_response r) 1115ef3166eSFrederic Barrat { 1125ef3166eSFrederic Barrat u64 reg = 0; 1135ef3166eSFrederic Barrat 1145ef3166eSFrederic Barrat /* continue is not supported */ 1155ef3166eSFrederic Barrat if (r == RESTART) 1165ef3166eSFrederic Barrat reg = PPC_BIT(31); 1175ef3166eSFrederic Barrat else if (r == ADDRESS_ERROR) 1185ef3166eSFrederic Barrat reg = PPC_BIT(30); 1195ef3166eSFrederic Barrat else 1205ef3166eSFrederic Barrat WARN(1, "Invalid irq response %d\n", r); 1215ef3166eSFrederic Barrat 12292add22eSFrederic Barrat if (reg) { 12392add22eSFrederic Barrat trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe, 12492add22eSFrederic Barrat spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg); 1255ef3166eSFrederic Barrat out_be64(spa->reg_tfc, reg); 1265ef3166eSFrederic Barrat } 12792add22eSFrederic Barrat } 1285ef3166eSFrederic Barrat 1295ef3166eSFrederic Barrat static void xsl_fault_handler_bh(struct work_struct *fault_work) 1305ef3166eSFrederic Barrat { 13150a7ca3cSSouptick Joarder vm_fault_t flt = 0; 1325ef3166eSFrederic Barrat unsigned long access, flags, inv_flags = 0; 1335ef3166eSFrederic Barrat enum xsl_response r; 1345ef3166eSFrederic Barrat struct xsl_fault *fault = container_of(fault_work, struct xsl_fault, 1355ef3166eSFrederic Barrat fault_work); 1365ef3166eSFrederic Barrat struct spa *spa = container_of(fault, struct spa, xsl_fault); 1375ef3166eSFrederic Barrat 1385ef3166eSFrederic Barrat int rc; 1395ef3166eSFrederic Barrat 1405ef3166eSFrederic Barrat /* 141d497ebf5SFrederic Barrat * We must release a reference on mm_users whenever exiting this 1425ef3166eSFrederic Barrat * function (taken in the memory fault interrupt handler) 1435ef3166eSFrederic Barrat */ 1445ef3166eSFrederic Barrat rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr, 1455ef3166eSFrederic Barrat &flt); 1465ef3166eSFrederic Barrat if (rc) { 1475ef3166eSFrederic Barrat pr_debug("copro_handle_mm_fault failed: %d\n", rc); 1485ef3166eSFrederic Barrat if (fault->pe_data.xsl_err_cb) { 1495ef3166eSFrederic Barrat fault->pe_data.xsl_err_cb( 1505ef3166eSFrederic Barrat fault->pe_data.xsl_err_data, 1515ef3166eSFrederic Barrat fault->dar, fault->dsisr); 1525ef3166eSFrederic Barrat } 1535ef3166eSFrederic Barrat r = ADDRESS_ERROR; 1545ef3166eSFrederic Barrat goto ack; 1555ef3166eSFrederic Barrat } 1565ef3166eSFrederic Barrat 1575ef3166eSFrederic Barrat if (!radix_enabled()) { 1585ef3166eSFrederic Barrat /* 1595ef3166eSFrederic Barrat * update_mmu_cache() will not have loaded the hash 1605ef3166eSFrederic Barrat * since current->trap is not a 0x400 or 0x300, so 1615ef3166eSFrederic Barrat * just call hash_page_mm() here. 1625ef3166eSFrederic Barrat */ 1635ef3166eSFrederic Barrat access = _PAGE_PRESENT | _PAGE_READ; 1645ef3166eSFrederic Barrat if (fault->dsisr & SPA_XSL_S) 1655ef3166eSFrederic Barrat access |= _PAGE_WRITE; 1665ef3166eSFrederic Barrat 1670034d395SAneesh Kumar K.V if (get_region_id(fault->dar) != USER_REGION_ID) 1685ef3166eSFrederic Barrat access |= _PAGE_PRIVILEGED; 1695ef3166eSFrederic Barrat 1705ef3166eSFrederic Barrat local_irq_save(flags); 1715ef3166eSFrederic Barrat hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300, 1725ef3166eSFrederic Barrat inv_flags); 1735ef3166eSFrederic Barrat local_irq_restore(flags); 1745ef3166eSFrederic Barrat } 1755ef3166eSFrederic Barrat r = RESTART; 1765ef3166eSFrederic Barrat ack: 177d497ebf5SFrederic Barrat mmput(fault->pe_data.mm); 1785ef3166eSFrederic Barrat ack_irq(spa, r); 1795ef3166eSFrederic Barrat } 1805ef3166eSFrederic Barrat 1815ef3166eSFrederic Barrat static irqreturn_t xsl_fault_handler(int irq, void *data) 1825ef3166eSFrederic Barrat { 1839c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) data; 1845ef3166eSFrederic Barrat struct spa *spa = link->spa; 1855ef3166eSFrederic Barrat u64 dsisr, dar, pe_handle; 1865ef3166eSFrederic Barrat struct pe_data *pe_data; 1875ef3166eSFrederic Barrat struct ocxl_process_element *pe; 18832eeb561SYueHaibing int pid; 189d497ebf5SFrederic Barrat bool schedule = false; 1905ef3166eSFrederic Barrat 1915ef3166eSFrederic Barrat read_irq(spa, &dsisr, &dar, &pe_handle); 19292add22eSFrederic Barrat trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1); 1935ef3166eSFrederic Barrat 1945ef3166eSFrederic Barrat WARN_ON(pe_handle > SPA_PE_MASK); 1955ef3166eSFrederic Barrat pe = spa->spa_mem + pe_handle; 1965ef3166eSFrederic Barrat pid = be32_to_cpu(pe->pid); 1975ef3166eSFrederic Barrat /* We could be reading all null values here if the PE is being 1985ef3166eSFrederic Barrat * removed while an interrupt kicks in. It's not supposed to 1995ef3166eSFrederic Barrat * happen if the driver notified the AFU to terminate the 2005ef3166eSFrederic Barrat * PASID, and the AFU waited for pending operations before 2015ef3166eSFrederic Barrat * acknowledging. But even if it happens, we won't find a 2025ef3166eSFrederic Barrat * memory context below and fail silently, so it should be ok. 2035ef3166eSFrederic Barrat */ 2045ef3166eSFrederic Barrat if (!(dsisr & SPA_XSL_TF)) { 2055ef3166eSFrederic Barrat WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr); 2065ef3166eSFrederic Barrat ack_irq(spa, ADDRESS_ERROR); 2075ef3166eSFrederic Barrat return IRQ_HANDLED; 2085ef3166eSFrederic Barrat } 2095ef3166eSFrederic Barrat 2105ef3166eSFrederic Barrat rcu_read_lock(); 2115ef3166eSFrederic Barrat pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle); 2125ef3166eSFrederic Barrat if (!pe_data) { 2135ef3166eSFrederic Barrat /* 2145ef3166eSFrederic Barrat * Could only happen if the driver didn't notify the 2155ef3166eSFrederic Barrat * AFU about PASID termination before removing the PE, 2165ef3166eSFrederic Barrat * or the AFU didn't wait for all memory access to 2175ef3166eSFrederic Barrat * have completed. 2185ef3166eSFrederic Barrat * 2195ef3166eSFrederic Barrat * Either way, we fail early, but we shouldn't log an 2205ef3166eSFrederic Barrat * error message, as it is a valid (if unexpected) 2215ef3166eSFrederic Barrat * scenario 2225ef3166eSFrederic Barrat */ 2235ef3166eSFrederic Barrat rcu_read_unlock(); 2245ef3166eSFrederic Barrat pr_debug("Unknown mm context for xsl interrupt\n"); 2255ef3166eSFrederic Barrat ack_irq(spa, ADDRESS_ERROR); 2265ef3166eSFrederic Barrat return IRQ_HANDLED; 2275ef3166eSFrederic Barrat } 22860e8523eSAlastair D'Silva 22960e8523eSAlastair D'Silva if (!pe_data->mm) { 23060e8523eSAlastair D'Silva /* 23160e8523eSAlastair D'Silva * translation fault from a kernel context - an OpenCAPI 23260e8523eSAlastair D'Silva * device tried to access a bad kernel address 23360e8523eSAlastair D'Silva */ 23460e8523eSAlastair D'Silva rcu_read_unlock(); 23560e8523eSAlastair D'Silva pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n"); 23660e8523eSAlastair D'Silva ack_irq(spa, ADDRESS_ERROR); 23760e8523eSAlastair D'Silva return IRQ_HANDLED; 23860e8523eSAlastair D'Silva } 2395ef3166eSFrederic Barrat WARN_ON(pe_data->mm->context.id != pid); 2405ef3166eSFrederic Barrat 241d497ebf5SFrederic Barrat if (mmget_not_zero(pe_data->mm)) { 2425ef3166eSFrederic Barrat spa->xsl_fault.pe = pe_handle; 2435ef3166eSFrederic Barrat spa->xsl_fault.dar = dar; 2445ef3166eSFrederic Barrat spa->xsl_fault.dsisr = dsisr; 2455ef3166eSFrederic Barrat spa->xsl_fault.pe_data = *pe_data; 246d497ebf5SFrederic Barrat schedule = true; 247d497ebf5SFrederic Barrat /* mm_users count released by bottom half */ 248d497ebf5SFrederic Barrat } 2495ef3166eSFrederic Barrat rcu_read_unlock(); 250d497ebf5SFrederic Barrat if (schedule) 2515ef3166eSFrederic Barrat schedule_work(&spa->xsl_fault.fault_work); 252d497ebf5SFrederic Barrat else 253d497ebf5SFrederic Barrat ack_irq(spa, ADDRESS_ERROR); 2545ef3166eSFrederic Barrat return IRQ_HANDLED; 2555ef3166eSFrederic Barrat } 2565ef3166eSFrederic Barrat 2575ef3166eSFrederic Barrat static void unmap_irq_registers(struct spa *spa) 2585ef3166eSFrederic Barrat { 2595ef3166eSFrederic Barrat pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc, 2605ef3166eSFrederic Barrat spa->reg_pe_handle); 2615ef3166eSFrederic Barrat } 2625ef3166eSFrederic Barrat 2635ef3166eSFrederic Barrat static int map_irq_registers(struct pci_dev *dev, struct spa *spa) 2645ef3166eSFrederic Barrat { 2655ef3166eSFrederic Barrat return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar, 2665ef3166eSFrederic Barrat &spa->reg_tfc, &spa->reg_pe_handle); 2675ef3166eSFrederic Barrat } 2685ef3166eSFrederic Barrat 2699c4ae064SAlastair D'Silva static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link) 2705ef3166eSFrederic Barrat { 2715ef3166eSFrederic Barrat struct spa *spa = link->spa; 2725ef3166eSFrederic Barrat int rc; 2735ef3166eSFrederic Barrat int hwirq; 2745ef3166eSFrederic Barrat 2755ef3166eSFrederic Barrat rc = pnv_ocxl_get_xsl_irq(dev, &hwirq); 2765ef3166eSFrederic Barrat if (rc) 2775ef3166eSFrederic Barrat return rc; 2785ef3166eSFrederic Barrat 2795ef3166eSFrederic Barrat rc = map_irq_registers(dev, spa); 2805ef3166eSFrederic Barrat if (rc) 2815ef3166eSFrederic Barrat return rc; 2825ef3166eSFrederic Barrat 2835ef3166eSFrederic Barrat spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x", 2845ef3166eSFrederic Barrat link->domain, link->bus, link->dev); 2855ef3166eSFrederic Barrat if (!spa->irq_name) { 2865ef3166eSFrederic Barrat dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n"); 287759bc015SGreg Kurz rc = -ENOMEM; 288759bc015SGreg Kurz goto err_xsl; 2895ef3166eSFrederic Barrat } 2905ef3166eSFrederic Barrat /* 2915ef3166eSFrederic Barrat * At some point, we'll need to look into allowing a higher 2925ef3166eSFrederic Barrat * number of interrupts. Could we have an IRQ domain per link? 2935ef3166eSFrederic Barrat */ 2945ef3166eSFrederic Barrat spa->virq = irq_create_mapping(NULL, hwirq); 2955ef3166eSFrederic Barrat if (!spa->virq) { 2965ef3166eSFrederic Barrat dev_err(&dev->dev, 2975ef3166eSFrederic Barrat "irq_create_mapping failed for translation interrupt\n"); 298759bc015SGreg Kurz rc = -EINVAL; 299759bc015SGreg Kurz goto err_name; 3005ef3166eSFrederic Barrat } 3015ef3166eSFrederic Barrat 3025ef3166eSFrederic Barrat dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq); 3035ef3166eSFrederic Barrat 3045ef3166eSFrederic Barrat rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name, 3055ef3166eSFrederic Barrat link); 3065ef3166eSFrederic Barrat if (rc) { 3075ef3166eSFrederic Barrat dev_err(&dev->dev, 3085ef3166eSFrederic Barrat "request_irq failed for translation interrupt: %d\n", 3095ef3166eSFrederic Barrat rc); 310759bc015SGreg Kurz rc = -EINVAL; 311759bc015SGreg Kurz goto err_mapping; 3125ef3166eSFrederic Barrat } 3135ef3166eSFrederic Barrat return 0; 314759bc015SGreg Kurz 315759bc015SGreg Kurz err_mapping: 316759bc015SGreg Kurz irq_dispose_mapping(spa->virq); 317759bc015SGreg Kurz err_name: 318759bc015SGreg Kurz kfree(spa->irq_name); 319759bc015SGreg Kurz err_xsl: 320759bc015SGreg Kurz unmap_irq_registers(spa); 321759bc015SGreg Kurz return rc; 3225ef3166eSFrederic Barrat } 3235ef3166eSFrederic Barrat 3249c4ae064SAlastair D'Silva static void release_xsl_irq(struct ocxl_link *link) 3255ef3166eSFrederic Barrat { 3265ef3166eSFrederic Barrat struct spa *spa = link->spa; 3275ef3166eSFrederic Barrat 3285ef3166eSFrederic Barrat if (spa->virq) { 3295ef3166eSFrederic Barrat free_irq(spa->virq, link); 3305ef3166eSFrederic Barrat irq_dispose_mapping(spa->virq); 3315ef3166eSFrederic Barrat } 3325ef3166eSFrederic Barrat kfree(spa->irq_name); 3335ef3166eSFrederic Barrat unmap_irq_registers(spa); 3345ef3166eSFrederic Barrat } 3355ef3166eSFrederic Barrat 3369c4ae064SAlastair D'Silva static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link) 3375ef3166eSFrederic Barrat { 3385ef3166eSFrederic Barrat struct spa *spa; 3395ef3166eSFrederic Barrat 3405ef3166eSFrederic Barrat spa = kzalloc(sizeof(struct spa), GFP_KERNEL); 3415ef3166eSFrederic Barrat if (!spa) 3425ef3166eSFrederic Barrat return -ENOMEM; 3435ef3166eSFrederic Barrat 3445ef3166eSFrederic Barrat mutex_init(&spa->spa_lock); 3455ef3166eSFrederic Barrat INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL); 3465ef3166eSFrederic Barrat INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh); 3475ef3166eSFrederic Barrat 3485ef3166eSFrederic Barrat spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT; 3495ef3166eSFrederic Barrat spa->spa_mem = (struct ocxl_process_element *) 3505ef3166eSFrederic Barrat __get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order); 3515ef3166eSFrederic Barrat if (!spa->spa_mem) { 3525ef3166eSFrederic Barrat dev_err(&dev->dev, "Can't allocate Shared Process Area\n"); 3535ef3166eSFrederic Barrat kfree(spa); 3545ef3166eSFrederic Barrat return -ENOMEM; 3555ef3166eSFrederic Barrat } 3565ef3166eSFrederic Barrat pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus, 3575ef3166eSFrederic Barrat link->dev, spa->spa_mem); 3585ef3166eSFrederic Barrat 3595ef3166eSFrederic Barrat link->spa = spa; 3605ef3166eSFrederic Barrat return 0; 3615ef3166eSFrederic Barrat } 3625ef3166eSFrederic Barrat 3639c4ae064SAlastair D'Silva static void free_spa(struct ocxl_link *link) 3645ef3166eSFrederic Barrat { 3655ef3166eSFrederic Barrat struct spa *spa = link->spa; 3665ef3166eSFrederic Barrat 3675ef3166eSFrederic Barrat pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus, 3685ef3166eSFrederic Barrat link->dev); 3695ef3166eSFrederic Barrat 3705ef3166eSFrederic Barrat if (spa && spa->spa_mem) { 3715ef3166eSFrederic Barrat free_pages((unsigned long) spa->spa_mem, spa->spa_order); 3725ef3166eSFrederic Barrat kfree(spa); 3735ef3166eSFrederic Barrat link->spa = NULL; 3745ef3166eSFrederic Barrat } 3755ef3166eSFrederic Barrat } 3765ef3166eSFrederic Barrat 3779c4ae064SAlastair D'Silva static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link) 3785ef3166eSFrederic Barrat { 3799c4ae064SAlastair D'Silva struct ocxl_link *link; 3805ef3166eSFrederic Barrat int rc; 3815ef3166eSFrederic Barrat 3829c4ae064SAlastair D'Silva link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL); 3835ef3166eSFrederic Barrat if (!link) 3845ef3166eSFrederic Barrat return -ENOMEM; 3855ef3166eSFrederic Barrat 3865ef3166eSFrederic Barrat kref_init(&link->ref); 3875ef3166eSFrederic Barrat link->domain = pci_domain_nr(dev->bus); 3885ef3166eSFrederic Barrat link->bus = dev->bus->number; 3895ef3166eSFrederic Barrat link->dev = PCI_SLOT(dev->devfn); 3905ef3166eSFrederic Barrat atomic_set(&link->irq_available, MAX_IRQ_PER_LINK); 3915ef3166eSFrederic Barrat 3925ef3166eSFrederic Barrat rc = alloc_spa(dev, link); 3935ef3166eSFrederic Barrat if (rc) 3945ef3166eSFrederic Barrat goto err_free; 3955ef3166eSFrederic Barrat 3965ef3166eSFrederic Barrat rc = setup_xsl_irq(dev, link); 3975ef3166eSFrederic Barrat if (rc) 3985ef3166eSFrederic Barrat goto err_spa; 3995ef3166eSFrederic Barrat 4005ef3166eSFrederic Barrat /* platform specific hook */ 4015ef3166eSFrederic Barrat rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask, 4025ef3166eSFrederic Barrat &link->platform_data); 4035ef3166eSFrederic Barrat if (rc) 4045ef3166eSFrederic Barrat goto err_xsl_irq; 4055ef3166eSFrederic Barrat 4065ef3166eSFrederic Barrat *out_link = link; 4075ef3166eSFrederic Barrat return 0; 4085ef3166eSFrederic Barrat 4095ef3166eSFrederic Barrat err_xsl_irq: 4105ef3166eSFrederic Barrat release_xsl_irq(link); 4115ef3166eSFrederic Barrat err_spa: 4125ef3166eSFrederic Barrat free_spa(link); 4135ef3166eSFrederic Barrat err_free: 4145ef3166eSFrederic Barrat kfree(link); 4155ef3166eSFrederic Barrat return rc; 4165ef3166eSFrederic Barrat } 4175ef3166eSFrederic Barrat 4189c4ae064SAlastair D'Silva static void free_link(struct ocxl_link *link) 4195ef3166eSFrederic Barrat { 4205ef3166eSFrederic Barrat release_xsl_irq(link); 4215ef3166eSFrederic Barrat free_spa(link); 4225ef3166eSFrederic Barrat kfree(link); 4235ef3166eSFrederic Barrat } 4245ef3166eSFrederic Barrat 4255ef3166eSFrederic Barrat int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle) 4265ef3166eSFrederic Barrat { 4275ef3166eSFrederic Barrat int rc = 0; 4289c4ae064SAlastair D'Silva struct ocxl_link *link; 4295ef3166eSFrederic Barrat 4305ef3166eSFrederic Barrat mutex_lock(&links_list_lock); 4315ef3166eSFrederic Barrat list_for_each_entry(link, &links_list, list) { 4325ef3166eSFrederic Barrat /* The functions of a device all share the same link */ 4335ef3166eSFrederic Barrat if (link->domain == pci_domain_nr(dev->bus) && 4345ef3166eSFrederic Barrat link->bus == dev->bus->number && 4355ef3166eSFrederic Barrat link->dev == PCI_SLOT(dev->devfn)) { 4365ef3166eSFrederic Barrat kref_get(&link->ref); 4375ef3166eSFrederic Barrat *link_handle = link; 4385ef3166eSFrederic Barrat goto unlock; 4395ef3166eSFrederic Barrat } 4405ef3166eSFrederic Barrat } 4415ef3166eSFrederic Barrat rc = alloc_link(dev, PE_mask, &link); 4425ef3166eSFrederic Barrat if (rc) 4435ef3166eSFrederic Barrat goto unlock; 4445ef3166eSFrederic Barrat 4455ef3166eSFrederic Barrat list_add(&link->list, &links_list); 4465ef3166eSFrederic Barrat *link_handle = link; 4475ef3166eSFrederic Barrat unlock: 4485ef3166eSFrederic Barrat mutex_unlock(&links_list_lock); 4495ef3166eSFrederic Barrat return rc; 4505ef3166eSFrederic Barrat } 451280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_setup); 4525ef3166eSFrederic Barrat 4535ef3166eSFrederic Barrat static void release_xsl(struct kref *ref) 4545ef3166eSFrederic Barrat { 4559c4ae064SAlastair D'Silva struct ocxl_link *link = container_of(ref, struct ocxl_link, ref); 4565ef3166eSFrederic Barrat 4575ef3166eSFrederic Barrat list_del(&link->list); 4585ef3166eSFrederic Barrat /* call platform code before releasing data */ 4595ef3166eSFrederic Barrat pnv_ocxl_spa_release(link->platform_data); 4605ef3166eSFrederic Barrat free_link(link); 4615ef3166eSFrederic Barrat } 4625ef3166eSFrederic Barrat 4635ef3166eSFrederic Barrat void ocxl_link_release(struct pci_dev *dev, void *link_handle) 4645ef3166eSFrederic Barrat { 4659c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 4665ef3166eSFrederic Barrat 4675ef3166eSFrederic Barrat mutex_lock(&links_list_lock); 4685ef3166eSFrederic Barrat kref_put(&link->ref, release_xsl); 4695ef3166eSFrederic Barrat mutex_unlock(&links_list_lock); 4705ef3166eSFrederic Barrat } 471280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_release); 4725ef3166eSFrederic Barrat 4735ef3166eSFrederic Barrat static u64 calculate_cfg_state(bool kernel) 4745ef3166eSFrederic Barrat { 4755ef3166eSFrederic Barrat u64 state; 4765ef3166eSFrederic Barrat 4775ef3166eSFrederic Barrat state = SPA_CFG_DR; 4785ef3166eSFrederic Barrat if (mfspr(SPRN_LPCR) & LPCR_TC) 4795ef3166eSFrederic Barrat state |= SPA_CFG_TC; 4805ef3166eSFrederic Barrat if (radix_enabled()) 4815ef3166eSFrederic Barrat state |= SPA_CFG_XLAT_ror; 4825ef3166eSFrederic Barrat else 4835ef3166eSFrederic Barrat state |= SPA_CFG_XLAT_hpt; 4845ef3166eSFrederic Barrat state |= SPA_CFG_HV; 4855ef3166eSFrederic Barrat if (kernel) { 4865ef3166eSFrederic Barrat if (mfmsr() & MSR_SF) 4875ef3166eSFrederic Barrat state |= SPA_CFG_SF; 4885ef3166eSFrederic Barrat } else { 4895ef3166eSFrederic Barrat state |= SPA_CFG_PR; 4905ef3166eSFrederic Barrat if (!test_tsk_thread_flag(current, TIF_32BIT)) 4915ef3166eSFrederic Barrat state |= SPA_CFG_SF; 4925ef3166eSFrederic Barrat } 4935ef3166eSFrederic Barrat return state; 4945ef3166eSFrederic Barrat } 4955ef3166eSFrederic Barrat 4965ef3166eSFrederic Barrat int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr, 497*d731feeaSChristophe Lombard u64 amr, u16 bdf, struct mm_struct *mm, 4985ef3166eSFrederic Barrat void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr), 4995ef3166eSFrederic Barrat void *xsl_err_data) 5005ef3166eSFrederic Barrat { 5019c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 5025ef3166eSFrederic Barrat struct spa *spa = link->spa; 5035ef3166eSFrederic Barrat struct ocxl_process_element *pe; 5045ef3166eSFrederic Barrat int pe_handle, rc = 0; 5055ef3166eSFrederic Barrat struct pe_data *pe_data; 5065ef3166eSFrederic Barrat 5075ef3166eSFrederic Barrat BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128); 5085ef3166eSFrederic Barrat if (pasid > SPA_PASID_MAX) 5095ef3166eSFrederic Barrat return -EINVAL; 5105ef3166eSFrederic Barrat 5115ef3166eSFrederic Barrat mutex_lock(&spa->spa_lock); 5125ef3166eSFrederic Barrat pe_handle = pasid & SPA_PE_MASK; 5135ef3166eSFrederic Barrat pe = spa->spa_mem + pe_handle; 5145ef3166eSFrederic Barrat 5155ef3166eSFrederic Barrat if (pe->software_state) { 5165ef3166eSFrederic Barrat rc = -EBUSY; 5175ef3166eSFrederic Barrat goto unlock; 5185ef3166eSFrederic Barrat } 5195ef3166eSFrederic Barrat 5205ef3166eSFrederic Barrat pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL); 5215ef3166eSFrederic Barrat if (!pe_data) { 5225ef3166eSFrederic Barrat rc = -ENOMEM; 5235ef3166eSFrederic Barrat goto unlock; 5245ef3166eSFrederic Barrat } 5255ef3166eSFrederic Barrat 5265ef3166eSFrederic Barrat pe_data->mm = mm; 5275ef3166eSFrederic Barrat pe_data->xsl_err_cb = xsl_err_cb; 5285ef3166eSFrederic Barrat pe_data->xsl_err_data = xsl_err_data; 5295ef3166eSFrederic Barrat 5305ef3166eSFrederic Barrat memset(pe, 0, sizeof(struct ocxl_process_element)); 5315ef3166eSFrederic Barrat pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0)); 532*d731feeaSChristophe Lombard pe->pasid = cpu_to_be32(pasid << (31 - 19)); 533*d731feeaSChristophe Lombard pe->bdf = cpu_to_be16(bdf); 5345ef3166eSFrederic Barrat pe->lpid = cpu_to_be32(mfspr(SPRN_LPID)); 5355ef3166eSFrederic Barrat pe->pid = cpu_to_be32(pidr); 5365ef3166eSFrederic Barrat pe->tid = cpu_to_be32(tidr); 5375ef3166eSFrederic Barrat pe->amr = cpu_to_be64(amr); 5385ef3166eSFrederic Barrat pe->software_state = cpu_to_be32(SPA_PE_VALID); 5395ef3166eSFrederic Barrat 54060e8523eSAlastair D'Silva /* 54160e8523eSAlastair D'Silva * For user contexts, register a copro so that TLBIs are seen 54260e8523eSAlastair D'Silva * by the nest MMU. If we have a kernel context, TLBIs are 54360e8523eSAlastair D'Silva * already global. 54460e8523eSAlastair D'Silva */ 54560e8523eSAlastair D'Silva if (mm) 5465ef3166eSFrederic Barrat mm_context_add_copro(mm); 5475ef3166eSFrederic Barrat /* 5485ef3166eSFrederic Barrat * Barrier is to make sure PE is visible in the SPA before it 5495ef3166eSFrederic Barrat * is used by the device. It also helps with the global TLBI 5505ef3166eSFrederic Barrat * invalidation 5515ef3166eSFrederic Barrat */ 5525ef3166eSFrederic Barrat mb(); 5535ef3166eSFrederic Barrat radix_tree_insert(&spa->pe_tree, pe_handle, pe_data); 5545ef3166eSFrederic Barrat 5555ef3166eSFrederic Barrat /* 5565ef3166eSFrederic Barrat * The mm must stay valid for as long as the device uses it. We 5575ef3166eSFrederic Barrat * lower the count when the context is removed from the SPA. 5585ef3166eSFrederic Barrat * 5595ef3166eSFrederic Barrat * We grab mm_count (and not mm_users), as we don't want to 5605ef3166eSFrederic Barrat * end up in a circular dependency if a process mmaps its 5615ef3166eSFrederic Barrat * mmio, therefore incrementing the file ref count when 5625ef3166eSFrederic Barrat * calling mmap(), and forgets to unmap before exiting. In 5635ef3166eSFrederic Barrat * that scenario, when the kernel handles the death of the 5645ef3166eSFrederic Barrat * process, the file is not cleaned because unmap was not 5655ef3166eSFrederic Barrat * called, and the mm wouldn't be freed because we would still 5665ef3166eSFrederic Barrat * have a reference on mm_users. Incrementing mm_count solves 5675ef3166eSFrederic Barrat * the problem. 5685ef3166eSFrederic Barrat */ 56960e8523eSAlastair D'Silva if (mm) 5705ef3166eSFrederic Barrat mmgrab(mm); 57192add22eSFrederic Barrat trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr); 5725ef3166eSFrederic Barrat unlock: 5735ef3166eSFrederic Barrat mutex_unlock(&spa->spa_lock); 5745ef3166eSFrederic Barrat return rc; 5755ef3166eSFrederic Barrat } 576280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_add_pe); 5775ef3166eSFrederic Barrat 578e948e06fSAlastair D'Silva int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid) 579e948e06fSAlastair D'Silva { 5809c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 581e948e06fSAlastair D'Silva struct spa *spa = link->spa; 582e948e06fSAlastair D'Silva struct ocxl_process_element *pe; 583e948e06fSAlastair D'Silva int pe_handle, rc; 584e948e06fSAlastair D'Silva 585e948e06fSAlastair D'Silva if (pasid > SPA_PASID_MAX) 586e948e06fSAlastair D'Silva return -EINVAL; 587e948e06fSAlastair D'Silva 588e948e06fSAlastair D'Silva pe_handle = pasid & SPA_PE_MASK; 589e948e06fSAlastair D'Silva pe = spa->spa_mem + pe_handle; 590e948e06fSAlastair D'Silva 591e948e06fSAlastair D'Silva mutex_lock(&spa->spa_lock); 592e948e06fSAlastair D'Silva 593e1e71e20SGreg Kurz pe->tid = cpu_to_be32(tid); 594e948e06fSAlastair D'Silva 595e948e06fSAlastair D'Silva /* 596e948e06fSAlastair D'Silva * The barrier makes sure the PE is updated 597e948e06fSAlastair D'Silva * before we clear the NPU context cache below, so that the 598e948e06fSAlastair D'Silva * old PE cannot be reloaded erroneously. 599e948e06fSAlastair D'Silva */ 600e948e06fSAlastair D'Silva mb(); 601e948e06fSAlastair D'Silva 602e948e06fSAlastair D'Silva /* 603e948e06fSAlastair D'Silva * hook to platform code 604e948e06fSAlastair D'Silva * On powerpc, the entry needs to be cleared from the context 605e948e06fSAlastair D'Silva * cache of the NPU. 606e948e06fSAlastair D'Silva */ 607e948e06fSAlastair D'Silva rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle); 608e948e06fSAlastair D'Silva WARN_ON(rc); 609e948e06fSAlastair D'Silva 610e948e06fSAlastair D'Silva mutex_unlock(&spa->spa_lock); 611e948e06fSAlastair D'Silva return rc; 612e948e06fSAlastair D'Silva } 613e948e06fSAlastair D'Silva 6145ef3166eSFrederic Barrat int ocxl_link_remove_pe(void *link_handle, int pasid) 6155ef3166eSFrederic Barrat { 6169c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 6175ef3166eSFrederic Barrat struct spa *spa = link->spa; 6185ef3166eSFrederic Barrat struct ocxl_process_element *pe; 6195ef3166eSFrederic Barrat struct pe_data *pe_data; 6205ef3166eSFrederic Barrat int pe_handle, rc; 6215ef3166eSFrederic Barrat 6225ef3166eSFrederic Barrat if (pasid > SPA_PASID_MAX) 6235ef3166eSFrederic Barrat return -EINVAL; 6245ef3166eSFrederic Barrat 6255ef3166eSFrederic Barrat /* 6265ef3166eSFrederic Barrat * About synchronization with our memory fault handler: 6275ef3166eSFrederic Barrat * 6285ef3166eSFrederic Barrat * Before removing the PE, the driver is supposed to have 6295ef3166eSFrederic Barrat * notified the AFU, which should have cleaned up and make 6305ef3166eSFrederic Barrat * sure the PASID is no longer in use, including pending 6315ef3166eSFrederic Barrat * interrupts. However, there's no way to be sure... 6325ef3166eSFrederic Barrat * 6335ef3166eSFrederic Barrat * We clear the PE and remove the context from our radix 6345ef3166eSFrederic Barrat * tree. From that point on, any new interrupt for that 6355ef3166eSFrederic Barrat * context will fail silently, which is ok. As mentioned 6365ef3166eSFrederic Barrat * above, that's not expected, but it could happen if the 6375ef3166eSFrederic Barrat * driver or AFU didn't do the right thing. 6385ef3166eSFrederic Barrat * 6395ef3166eSFrederic Barrat * There could still be a bottom half running, but we don't 6405ef3166eSFrederic Barrat * need to wait/flush, as it is managing a reference count on 6415ef3166eSFrederic Barrat * the mm it reads from the radix tree. 6425ef3166eSFrederic Barrat */ 6435ef3166eSFrederic Barrat pe_handle = pasid & SPA_PE_MASK; 6445ef3166eSFrederic Barrat pe = spa->spa_mem + pe_handle; 6455ef3166eSFrederic Barrat 6465ef3166eSFrederic Barrat mutex_lock(&spa->spa_lock); 6475ef3166eSFrederic Barrat 6485ef3166eSFrederic Barrat if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) { 6495ef3166eSFrederic Barrat rc = -EINVAL; 6505ef3166eSFrederic Barrat goto unlock; 6515ef3166eSFrederic Barrat } 6525ef3166eSFrederic Barrat 65392add22eSFrederic Barrat trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid, 65492add22eSFrederic Barrat be32_to_cpu(pe->pid), be32_to_cpu(pe->tid)); 65592add22eSFrederic Barrat 6565ef3166eSFrederic Barrat memset(pe, 0, sizeof(struct ocxl_process_element)); 6575ef3166eSFrederic Barrat /* 6585ef3166eSFrederic Barrat * The barrier makes sure the PE is removed from the SPA 6595ef3166eSFrederic Barrat * before we clear the NPU context cache below, so that the 6605ef3166eSFrederic Barrat * old PE cannot be reloaded erroneously. 6615ef3166eSFrederic Barrat */ 6625ef3166eSFrederic Barrat mb(); 6635ef3166eSFrederic Barrat 6645ef3166eSFrederic Barrat /* 6655ef3166eSFrederic Barrat * hook to platform code 6665ef3166eSFrederic Barrat * On powerpc, the entry needs to be cleared from the context 6675ef3166eSFrederic Barrat * cache of the NPU. 6685ef3166eSFrederic Barrat */ 66919df3958SAlastair D'Silva rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle); 6705ef3166eSFrederic Barrat WARN_ON(rc); 6715ef3166eSFrederic Barrat 6725ef3166eSFrederic Barrat pe_data = radix_tree_delete(&spa->pe_tree, pe_handle); 6735ef3166eSFrederic Barrat if (!pe_data) { 6745ef3166eSFrederic Barrat WARN(1, "Couldn't find pe data when removing PE\n"); 6755ef3166eSFrederic Barrat } else { 67660e8523eSAlastair D'Silva if (pe_data->mm) { 6775ef3166eSFrederic Barrat mm_context_remove_copro(pe_data->mm); 6785ef3166eSFrederic Barrat mmdrop(pe_data->mm); 67960e8523eSAlastair D'Silva } 6805ef3166eSFrederic Barrat kfree_rcu(pe_data, rcu); 6815ef3166eSFrederic Barrat } 6825ef3166eSFrederic Barrat unlock: 6835ef3166eSFrederic Barrat mutex_unlock(&spa->spa_lock); 6845ef3166eSFrederic Barrat return rc; 6855ef3166eSFrederic Barrat } 686280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_remove_pe); 687aeddad17SFrederic Barrat 688dde6f18aSFrederic Barrat int ocxl_link_irq_alloc(void *link_handle, int *hw_irq) 689aeddad17SFrederic Barrat { 6909c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 691dde6f18aSFrederic Barrat int irq; 692aeddad17SFrederic Barrat 693aeddad17SFrederic Barrat if (atomic_dec_if_positive(&link->irq_available) < 0) 694aeddad17SFrederic Barrat return -ENOSPC; 695aeddad17SFrederic Barrat 696dde6f18aSFrederic Barrat irq = xive_native_alloc_irq(); 697dde6f18aSFrederic Barrat if (!irq) { 698aeddad17SFrederic Barrat atomic_inc(&link->irq_available); 699dde6f18aSFrederic Barrat return -ENXIO; 700aeddad17SFrederic Barrat } 701aeddad17SFrederic Barrat 702aeddad17SFrederic Barrat *hw_irq = irq; 703aeddad17SFrederic Barrat return 0; 704aeddad17SFrederic Barrat } 705280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc); 706aeddad17SFrederic Barrat 707aeddad17SFrederic Barrat void ocxl_link_free_irq(void *link_handle, int hw_irq) 708aeddad17SFrederic Barrat { 7099c4ae064SAlastair D'Silva struct ocxl_link *link = (struct ocxl_link *) link_handle; 710aeddad17SFrederic Barrat 711dde6f18aSFrederic Barrat xive_native_free_irq(hw_irq); 712aeddad17SFrederic Barrat atomic_inc(&link->irq_available); 713aeddad17SFrederic Barrat } 714280b983cSFrederic Barrat EXPORT_SYMBOL_GPL(ocxl_link_free_irq); 715