1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 24eafad7fSAlexey Ishchuk /* 34eafad7fSAlexey Ishchuk * Access to PCI I/O memory from user space programs. 44eafad7fSAlexey Ishchuk * 54eafad7fSAlexey Ishchuk * Copyright IBM Corp. 2014 64eafad7fSAlexey Ishchuk * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> 74eafad7fSAlexey Ishchuk */ 84eafad7fSAlexey Ishchuk #include <linux/kernel.h> 94eafad7fSAlexey Ishchuk #include <linux/syscalls.h> 104eafad7fSAlexey Ishchuk #include <linux/init.h> 114eafad7fSAlexey Ishchuk #include <linux/mm.h> 124eafad7fSAlexey Ishchuk #include <linux/errno.h> 134eafad7fSAlexey Ishchuk #include <linux/pci.h> 14d09a307fSHeiko Carstens #include <asm/asm-extable.h> 15f058599eSNiklas Schnelle #include <asm/pci_io.h> 16f058599eSNiklas Schnelle #include <asm/pci_debug.h> 17f058599eSNiklas Schnelle 18f058599eSNiklas Schnelle static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) 19f058599eSNiklas Schnelle { 20f058599eSNiklas Schnelle struct { 21f058599eSNiklas Schnelle u64 offset; 22f058599eSNiklas Schnelle u8 cc; 23f058599eSNiklas Schnelle u8 status; 24f058599eSNiklas Schnelle } data = {offset, cc, status}; 25f058599eSNiklas Schnelle 26f058599eSNiklas Schnelle zpci_err_hex(&data, sizeof(data)); 27f058599eSNiklas Schnelle } 28f058599eSNiklas Schnelle 29f058599eSNiklas Schnelle static inline int __pcistb_mio_inuser( 30f058599eSNiklas Schnelle void __iomem *ioaddr, const void __user *src, 31f058599eSNiklas Schnelle u64 len, u8 *status) 32f058599eSNiklas Schnelle { 33f058599eSNiklas Schnelle int cc = -ENXIO; 34f058599eSNiklas Schnelle 35f058599eSNiklas Schnelle asm volatile ( 36f058599eSNiklas Schnelle " sacf 256\n" 37f058599eSNiklas Schnelle "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" 38f058599eSNiklas Schnelle "1: ipm %[cc]\n" 39f058599eSNiklas Schnelle " srl %[cc],28\n" 40f058599eSNiklas Schnelle "2: sacf 768\n" 41f058599eSNiklas Schnelle EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) 42f058599eSNiklas Schnelle : [cc] "+d" (cc), [len] "+d" (len) 43f058599eSNiklas Schnelle : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) 44f058599eSNiklas Schnelle : "cc", "memory"); 45f058599eSNiklas Schnelle *status = len >> 24 & 0xff; 46f058599eSNiklas Schnelle return cc; 47f058599eSNiklas Schnelle } 48f058599eSNiklas Schnelle 49f058599eSNiklas Schnelle static inline int __pcistg_mio_inuser( 50f058599eSNiklas Schnelle void __iomem *ioaddr, const void __user *src, 51f058599eSNiklas Schnelle u64 ulen, u8 *status) 52f058599eSNiklas Schnelle { 53d66a4c7fSNiklas Schnelle union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; 54f058599eSNiklas Schnelle int cc = -ENXIO; 55f058599eSNiklas Schnelle u64 val = 0; 56f058599eSNiklas Schnelle u64 cnt = ulen; 57f058599eSNiklas Schnelle u8 tmp; 58f058599eSNiklas Schnelle 59f058599eSNiklas Schnelle /* 60f058599eSNiklas Schnelle * copy 0 < @len <= 8 bytes from @src into the right most bytes of 61f058599eSNiklas Schnelle * a register, then store it to PCI at @ioaddr while in secondary 62f058599eSNiklas Schnelle * address space. pcistg then uses the user mappings. 63f058599eSNiklas Schnelle */ 64f058599eSNiklas Schnelle asm volatile ( 65f058599eSNiklas Schnelle " sacf 256\n" 66f058599eSNiklas Schnelle "0: llgc %[tmp],0(%[src])\n" 676ec80302SHeiko Carstens "4: sllg %[val],%[val],8\n" 68f058599eSNiklas Schnelle " aghi %[src],1\n" 69f058599eSNiklas Schnelle " ogr %[val],%[tmp]\n" 70f058599eSNiklas Schnelle " brctg %[cnt],0b\n" 71d66a4c7fSNiklas Schnelle "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" 72f058599eSNiklas Schnelle "2: ipm %[cc]\n" 73f058599eSNiklas Schnelle " srl %[cc],28\n" 74f058599eSNiklas Schnelle "3: sacf 768\n" 756ec80302SHeiko Carstens EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) 76f058599eSNiklas Schnelle : 77f058599eSNiklas Schnelle [src] "+a" (src), [cnt] "+d" (cnt), 78f058599eSNiklas Schnelle [val] "+d" (val), [tmp] "=d" (tmp), 79d66a4c7fSNiklas Schnelle [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) 80f058599eSNiklas Schnelle :: "cc", "memory"); 81d66a4c7fSNiklas Schnelle *status = ioaddr_len.odd >> 24 & 0xff; 82f058599eSNiklas Schnelle 83f058599eSNiklas Schnelle /* did we read everything from user memory? */ 84f058599eSNiklas Schnelle if (!cc && cnt != 0) 85f058599eSNiklas Schnelle cc = -EFAULT; 86f058599eSNiklas Schnelle 87f058599eSNiklas Schnelle return cc; 88f058599eSNiklas Schnelle } 89f058599eSNiklas Schnelle 90f058599eSNiklas Schnelle static inline int __memcpy_toio_inuser(void __iomem *dst, 91f058599eSNiklas Schnelle const void __user *src, size_t n) 92f058599eSNiklas Schnelle { 93f058599eSNiklas Schnelle int size, rc = 0; 94f058599eSNiklas Schnelle u8 status = 0; 95f058599eSNiklas Schnelle 96f058599eSNiklas Schnelle if (!src) 97f058599eSNiklas Schnelle return -EINVAL; 98f058599eSNiklas Schnelle 99f058599eSNiklas Schnelle while (n > 0) { 100*80df7d6aSNiklas Schnelle size = zpci_get_max_io_size((u64 __force) dst, 101f058599eSNiklas Schnelle (u64 __force) src, n, 102f058599eSNiklas Schnelle ZPCI_MAX_WRITE_SIZE); 103f058599eSNiklas Schnelle if (size > 8) /* main path */ 104f058599eSNiklas Schnelle rc = __pcistb_mio_inuser(dst, src, size, &status); 105f058599eSNiklas Schnelle else 106f058599eSNiklas Schnelle rc = __pcistg_mio_inuser(dst, src, size, &status); 107f058599eSNiklas Schnelle if (rc) 108f058599eSNiklas Schnelle break; 109f058599eSNiklas Schnelle src += size; 110f058599eSNiklas Schnelle dst += size; 111f058599eSNiklas Schnelle n -= size; 112f058599eSNiklas Schnelle } 113f058599eSNiklas Schnelle if (rc) 114f058599eSNiklas Schnelle zpci_err_mmio(rc, status, (__force u64) dst); 115f058599eSNiklas Schnelle return rc; 116f058599eSNiklas Schnelle } 1174eafad7fSAlexey Ishchuk 1184eafad7fSAlexey Ishchuk SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, 1194eafad7fSAlexey Ishchuk const void __user *, user_buffer, size_t, length) 1204eafad7fSAlexey Ishchuk { 1214eafad7fSAlexey Ishchuk u8 local_buf[64]; 1224eafad7fSAlexey Ishchuk void __iomem *io_addr; 1234eafad7fSAlexey Ishchuk void *buf; 124a67a88b0SDaniel Vetter struct vm_area_struct *vma; 125a67a88b0SDaniel Vetter pte_t *ptep; 126a67a88b0SDaniel Vetter spinlock_t *ptl; 1274eafad7fSAlexey Ishchuk long ret; 1284eafad7fSAlexey Ishchuk 1294eafad7fSAlexey Ishchuk if (!zpci_is_enabled()) 1304eafad7fSAlexey Ishchuk return -ENODEV; 1314eafad7fSAlexey Ishchuk 1324eafad7fSAlexey Ishchuk if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 1334eafad7fSAlexey Ishchuk return -EINVAL; 134f058599eSNiklas Schnelle 135f058599eSNiklas Schnelle /* 1364631f3caSNiklas Schnelle * We only support write access to MIO capable devices if we are on 1374631f3caSNiklas Schnelle * a MIO enabled system. Otherwise we would have to check for every 1384631f3caSNiklas Schnelle * address if it is a special ZPCI_ADDR and would have to do 139a67a88b0SDaniel Vetter * a pfn lookup which we don't need for MIO capable devices. Currently 1404631f3caSNiklas Schnelle * ISM devices are the only devices without MIO support and there is no 1414631f3caSNiklas Schnelle * known need for accessing these from userspace. 142f058599eSNiklas Schnelle */ 143f058599eSNiklas Schnelle if (static_branch_likely(&have_mio)) { 144f058599eSNiklas Schnelle ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, 145f058599eSNiklas Schnelle user_buffer, 146f058599eSNiklas Schnelle length); 147f058599eSNiklas Schnelle return ret; 148f058599eSNiklas Schnelle } 149f058599eSNiklas Schnelle 1504eafad7fSAlexey Ishchuk if (length > 64) { 1514eafad7fSAlexey Ishchuk buf = kmalloc(length, GFP_KERNEL); 1524eafad7fSAlexey Ishchuk if (!buf) 1534eafad7fSAlexey Ishchuk return -ENOMEM; 1544eafad7fSAlexey Ishchuk } else 1554eafad7fSAlexey Ishchuk buf = local_buf; 1564eafad7fSAlexey Ishchuk 157a67a88b0SDaniel Vetter ret = -EFAULT; 158a67a88b0SDaniel Vetter if (copy_from_user(buf, user_buffer, length)) 159a67a88b0SDaniel Vetter goto out_free; 160a67a88b0SDaniel Vetter 161a67a88b0SDaniel Vetter mmap_read_lock(current->mm); 162a67a88b0SDaniel Vetter ret = -EINVAL; 163a8b92b8cSDavid Hildenbrand vma = vma_lookup(current->mm, mmio_addr); 164a67a88b0SDaniel Vetter if (!vma) 165a67a88b0SDaniel Vetter goto out_unlock_mmap; 166a67a88b0SDaniel Vetter if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 167a67a88b0SDaniel Vetter goto out_unlock_mmap; 168a67a88b0SDaniel Vetter ret = -EACCES; 169a67a88b0SDaniel Vetter if (!(vma->vm_flags & VM_WRITE)) 170a67a88b0SDaniel Vetter goto out_unlock_mmap; 171a67a88b0SDaniel Vetter 1729fd6dad1SPaolo Bonzini ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); 1734eafad7fSAlexey Ishchuk if (ret) 174a67a88b0SDaniel Vetter goto out_unlock_mmap; 175a67a88b0SDaniel Vetter 176a67a88b0SDaniel Vetter io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | 177f058599eSNiklas Schnelle (mmio_addr & ~PAGE_MASK)); 1784eafad7fSAlexey Ishchuk 1794eafad7fSAlexey Ishchuk if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) 180a67a88b0SDaniel Vetter goto out_unlock_pt; 1814eafad7fSAlexey Ishchuk 182f0483044SSebastian Ott ret = zpci_memcpy_toio(io_addr, buf, length); 183a67a88b0SDaniel Vetter out_unlock_pt: 184a67a88b0SDaniel Vetter pte_unmap_unlock(ptep, ptl); 185a67a88b0SDaniel Vetter out_unlock_mmap: 186a67a88b0SDaniel Vetter mmap_read_unlock(current->mm); 187a67a88b0SDaniel Vetter out_free: 1884eafad7fSAlexey Ishchuk if (buf != local_buf) 1894eafad7fSAlexey Ishchuk kfree(buf); 1904eafad7fSAlexey Ishchuk return ret; 1914eafad7fSAlexey Ishchuk } 1924eafad7fSAlexey Ishchuk 193f058599eSNiklas Schnelle static inline int __pcilg_mio_inuser( 194f058599eSNiklas Schnelle void __user *dst, const void __iomem *ioaddr, 195f058599eSNiklas Schnelle u64 ulen, u8 *status) 196f058599eSNiklas Schnelle { 197d66a4c7fSNiklas Schnelle union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; 198f058599eSNiklas Schnelle u64 cnt = ulen; 199f058599eSNiklas Schnelle int shift = ulen * 8; 200f058599eSNiklas Schnelle int cc = -ENXIO; 201f058599eSNiklas Schnelle u64 val, tmp; 202f058599eSNiklas Schnelle 203f058599eSNiklas Schnelle /* 204f058599eSNiklas Schnelle * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in 205f058599eSNiklas Schnelle * user space) into a register using pcilg then store these bytes at 206f058599eSNiklas Schnelle * user address @dst 207f058599eSNiklas Schnelle */ 208f058599eSNiklas Schnelle asm volatile ( 209f058599eSNiklas Schnelle " sacf 256\n" 210d66a4c7fSNiklas Schnelle "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" 211f058599eSNiklas Schnelle "1: ipm %[cc]\n" 212f058599eSNiklas Schnelle " srl %[cc],28\n" 213f058599eSNiklas Schnelle " ltr %[cc],%[cc]\n" 214f058599eSNiklas Schnelle " jne 4f\n" 215f058599eSNiklas Schnelle "2: ahi %[shift],-8\n" 216f058599eSNiklas Schnelle " srlg %[tmp],%[val],0(%[shift])\n" 217f058599eSNiklas Schnelle "3: stc %[tmp],0(%[dst])\n" 2186ec80302SHeiko Carstens "5: aghi %[dst],1\n" 219f058599eSNiklas Schnelle " brctg %[cnt],2b\n" 220f058599eSNiklas Schnelle "4: sacf 768\n" 2216ec80302SHeiko Carstens EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) 222f058599eSNiklas Schnelle : 223d66a4c7fSNiklas Schnelle [ioaddr_len] "+&d" (ioaddr_len.pair), 224d66a4c7fSNiklas Schnelle [cc] "+d" (cc), [val] "=d" (val), 225f058599eSNiklas Schnelle [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), 226f058599eSNiklas Schnelle [shift] "+d" (shift) 227d66a4c7fSNiklas Schnelle :: "cc", "memory"); 228f058599eSNiklas Schnelle 229f058599eSNiklas Schnelle /* did we write everything to the user space buffer? */ 230f058599eSNiklas Schnelle if (!cc && cnt != 0) 231f058599eSNiklas Schnelle cc = -EFAULT; 232f058599eSNiklas Schnelle 233d66a4c7fSNiklas Schnelle *status = ioaddr_len.odd >> 24 & 0xff; 234f058599eSNiklas Schnelle return cc; 235f058599eSNiklas Schnelle } 236f058599eSNiklas Schnelle 237f058599eSNiklas Schnelle static inline int __memcpy_fromio_inuser(void __user *dst, 238f058599eSNiklas Schnelle const void __iomem *src, 239f058599eSNiklas Schnelle unsigned long n) 240f058599eSNiklas Schnelle { 241f058599eSNiklas Schnelle int size, rc = 0; 242f058599eSNiklas Schnelle u8 status; 243f058599eSNiklas Schnelle 244f058599eSNiklas Schnelle while (n > 0) { 245*80df7d6aSNiklas Schnelle size = zpci_get_max_io_size((u64 __force) src, 246f058599eSNiklas Schnelle (u64 __force) dst, n, 247f058599eSNiklas Schnelle ZPCI_MAX_READ_SIZE); 248f058599eSNiklas Schnelle rc = __pcilg_mio_inuser(dst, src, size, &status); 249f058599eSNiklas Schnelle if (rc) 250f058599eSNiklas Schnelle break; 251f058599eSNiklas Schnelle src += size; 252f058599eSNiklas Schnelle dst += size; 253f058599eSNiklas Schnelle n -= size; 254f058599eSNiklas Schnelle } 255f058599eSNiklas Schnelle if (rc) 256f058599eSNiklas Schnelle zpci_err_mmio(rc, status, (__force u64) dst); 257f058599eSNiklas Schnelle return rc; 258f058599eSNiklas Schnelle } 259f058599eSNiklas Schnelle 2604eafad7fSAlexey Ishchuk SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, 2614eafad7fSAlexey Ishchuk void __user *, user_buffer, size_t, length) 2624eafad7fSAlexey Ishchuk { 2634eafad7fSAlexey Ishchuk u8 local_buf[64]; 2644eafad7fSAlexey Ishchuk void __iomem *io_addr; 2654eafad7fSAlexey Ishchuk void *buf; 266a67a88b0SDaniel Vetter struct vm_area_struct *vma; 267a67a88b0SDaniel Vetter pte_t *ptep; 268a67a88b0SDaniel Vetter spinlock_t *ptl; 2694eafad7fSAlexey Ishchuk long ret; 2704eafad7fSAlexey Ishchuk 2714eafad7fSAlexey Ishchuk if (!zpci_is_enabled()) 2724eafad7fSAlexey Ishchuk return -ENODEV; 2734eafad7fSAlexey Ishchuk 2744eafad7fSAlexey Ishchuk if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 2754eafad7fSAlexey Ishchuk return -EINVAL; 276f058599eSNiklas Schnelle 277f058599eSNiklas Schnelle /* 2784631f3caSNiklas Schnelle * We only support read access to MIO capable devices if we are on 2794631f3caSNiklas Schnelle * a MIO enabled system. Otherwise we would have to check for every 2804631f3caSNiklas Schnelle * address if it is a special ZPCI_ADDR and would have to do 281a67a88b0SDaniel Vetter * a pfn lookup which we don't need for MIO capable devices. Currently 2824631f3caSNiklas Schnelle * ISM devices are the only devices without MIO support and there is no 2834631f3caSNiklas Schnelle * known need for accessing these from userspace. 284f058599eSNiklas Schnelle */ 285f058599eSNiklas Schnelle if (static_branch_likely(&have_mio)) { 286f058599eSNiklas Schnelle ret = __memcpy_fromio_inuser( 287f058599eSNiklas Schnelle user_buffer, (const void __iomem *)mmio_addr, 288f058599eSNiklas Schnelle length); 289f058599eSNiklas Schnelle return ret; 290f058599eSNiklas Schnelle } 291f058599eSNiklas Schnelle 2924eafad7fSAlexey Ishchuk if (length > 64) { 2934eafad7fSAlexey Ishchuk buf = kmalloc(length, GFP_KERNEL); 2944eafad7fSAlexey Ishchuk if (!buf) 2954eafad7fSAlexey Ishchuk return -ENOMEM; 296f058599eSNiklas Schnelle } else { 2974eafad7fSAlexey Ishchuk buf = local_buf; 298f058599eSNiklas Schnelle } 2994eafad7fSAlexey Ishchuk 300a67a88b0SDaniel Vetter mmap_read_lock(current->mm); 301a67a88b0SDaniel Vetter ret = -EINVAL; 302a8b92b8cSDavid Hildenbrand vma = vma_lookup(current->mm, mmio_addr); 303a67a88b0SDaniel Vetter if (!vma) 304a67a88b0SDaniel Vetter goto out_unlock_mmap; 305a67a88b0SDaniel Vetter if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 306a67a88b0SDaniel Vetter goto out_unlock_mmap; 307a67a88b0SDaniel Vetter ret = -EACCES; 308a67a88b0SDaniel Vetter if (!(vma->vm_flags & VM_WRITE)) 309a67a88b0SDaniel Vetter goto out_unlock_mmap; 310a67a88b0SDaniel Vetter 3119fd6dad1SPaolo Bonzini ret = follow_pte(vma->vm_mm, mmio_addr, &ptep, &ptl); 3124eafad7fSAlexey Ishchuk if (ret) 313a67a88b0SDaniel Vetter goto out_unlock_mmap; 314a67a88b0SDaniel Vetter 315a67a88b0SDaniel Vetter io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | 316a67a88b0SDaniel Vetter (mmio_addr & ~PAGE_MASK)); 3174eafad7fSAlexey Ishchuk 318f0483044SSebastian Ott if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { 3194eafad7fSAlexey Ishchuk ret = -EFAULT; 320a67a88b0SDaniel Vetter goto out_unlock_pt; 321f0483044SSebastian Ott } 322f0483044SSebastian Ott ret = zpci_memcpy_fromio(buf, io_addr, length); 323a67a88b0SDaniel Vetter 324a67a88b0SDaniel Vetter out_unlock_pt: 325a67a88b0SDaniel Vetter pte_unmap_unlock(ptep, ptl); 326a67a88b0SDaniel Vetter out_unlock_mmap: 327a67a88b0SDaniel Vetter mmap_read_unlock(current->mm); 328a67a88b0SDaniel Vetter 329a67a88b0SDaniel Vetter if (!ret && copy_to_user(user_buffer, buf, length)) 330f0483044SSebastian Ott ret = -EFAULT; 3314eafad7fSAlexey Ishchuk 3324eafad7fSAlexey Ishchuk if (buf != local_buf) 3334eafad7fSAlexey Ishchuk kfree(buf); 3344eafad7fSAlexey Ishchuk return ret; 3354eafad7fSAlexey Ishchuk } 336