1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Access to PCI I/O memory from user space programs. 4 * 5 * Copyright IBM Corp. 2014 6 * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> 7 */ 8 #include <linux/kernel.h> 9 #include <linux/syscalls.h> 10 #include <linux/init.h> 11 #include <linux/mm.h> 12 #include <linux/errno.h> 13 #include <linux/pci.h> 14 #include <asm/asm-extable.h> 15 #include <asm/pci_io.h> 16 #include <asm/pci_debug.h> 17 #include <asm/asm.h> 18 19 static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) 20 { 21 struct { 22 u64 offset; 23 u8 cc; 24 u8 status; 25 } data = {offset, cc, status}; 26 27 zpci_err_hex(&data, sizeof(data)); 28 } 29 30 static inline int __pcistb_mio_inuser( 31 void __iomem *ioaddr, const void __user *src, 32 u64 len, u8 *status) 33 { 34 int cc, exception; 35 36 exception = 1; 37 asm_inline volatile ( 38 " sacf 256\n" 39 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" 40 "1: lhi %[exc],0\n" 41 "2: sacf 768\n" 42 CC_IPM(cc) 43 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) 44 : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception) 45 : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) 46 : CC_CLOBBER_LIST("memory")); 47 *status = len >> 24 & 0xff; 48 return exception ? -ENXIO : CC_TRANSFORM(cc); 49 } 50 51 static inline int __pcistg_mio_inuser( 52 void __iomem *ioaddr, const void __user *src, 53 u64 ulen, u8 *status) 54 { 55 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; 56 int cc, exception; 57 u64 val = 0; 58 u64 cnt = ulen; 59 u8 tmp; 60 61 /* 62 * copy 0 < @len <= 8 bytes from @src into the right most bytes of 63 * a register, then store it to PCI at @ioaddr while in secondary 64 * address space. pcistg then uses the user mappings. 65 */ 66 exception = 1; 67 asm_inline volatile ( 68 " sacf 256\n" 69 "0: llgc %[tmp],0(%[src])\n" 70 "4: sllg %[val],%[val],8\n" 71 " aghi %[src],1\n" 72 " ogr %[val],%[tmp]\n" 73 " brctg %[cnt],0b\n" 74 "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" 75 "2: lhi %[exc],0\n" 76 "3: sacf 768\n" 77 CC_IPM(cc) 78 EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) 79 : [src] "+a" (src), [cnt] "+d" (cnt), 80 [val] "+d" (val), [tmp] "=d" (tmp), [exc] "+d" (exception), 81 CC_OUT(cc, cc), [ioaddr_len] "+&d" (ioaddr_len.pair) 82 : 83 : CC_CLOBBER_LIST("memory")); 84 *status = ioaddr_len.odd >> 24 & 0xff; 85 86 cc = exception ? -ENXIO : CC_TRANSFORM(cc); 87 /* did we read everything from user memory? */ 88 if (!cc && cnt != 0) 89 cc = -EFAULT; 90 91 return cc; 92 } 93 94 static inline int __memcpy_toio_inuser(void __iomem *dst, 95 const void __user *src, size_t n) 96 { 97 int size, rc = 0; 98 u8 status = 0; 99 100 if (!src) 101 return -EINVAL; 102 103 while (n > 0) { 104 size = zpci_get_max_io_size((u64 __force) dst, 105 (u64 __force) src, n, 106 ZPCI_MAX_WRITE_SIZE); 107 if (size > 8) /* main path */ 108 rc = __pcistb_mio_inuser(dst, src, size, &status); 109 else 110 rc = __pcistg_mio_inuser(dst, src, size, &status); 111 if (rc) 112 break; 113 src += size; 114 dst += size; 115 n -= size; 116 } 117 if (rc) 118 zpci_err_mmio(rc, status, (__force u64) dst); 119 return rc; 120 } 121 122 SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, 123 const void __user *, user_buffer, size_t, length) 124 { 125 struct follow_pfnmap_args args = { }; 126 u8 local_buf[64]; 127 void __iomem *io_addr; 128 void *buf; 129 struct vm_area_struct *vma; 130 long ret; 131 132 if (!zpci_is_enabled()) 133 return -ENODEV; 134 135 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 136 return -EINVAL; 137 138 /* 139 * We only support write access to MIO capable devices if we are on 140 * a MIO enabled system. Otherwise we would have to check for every 141 * address if it is a special ZPCI_ADDR and would have to do 142 * a pfn lookup which we don't need for MIO capable devices. Currently 143 * ISM devices are the only devices without MIO support and there is no 144 * known need for accessing these from userspace. 145 */ 146 if (static_branch_likely(&have_mio)) { 147 ret = __memcpy_toio_inuser((void __iomem *) mmio_addr, 148 user_buffer, 149 length); 150 return ret; 151 } 152 153 if (length > 64) { 154 buf = kmalloc(length, GFP_KERNEL); 155 if (!buf) 156 return -ENOMEM; 157 } else 158 buf = local_buf; 159 160 ret = -EFAULT; 161 if (copy_from_user(buf, user_buffer, length)) 162 goto out_free; 163 164 mmap_read_lock(current->mm); 165 ret = -EINVAL; 166 vma = vma_lookup(current->mm, mmio_addr); 167 if (!vma) 168 goto out_unlock_mmap; 169 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 170 goto out_unlock_mmap; 171 ret = -EACCES; 172 if (!(vma->vm_flags & VM_WRITE)) 173 goto out_unlock_mmap; 174 175 args.address = mmio_addr; 176 args.vma = vma; 177 ret = follow_pfnmap_start(&args); 178 if (ret) { 179 fixup_user_fault(current->mm, mmio_addr, FAULT_FLAG_WRITE, NULL); 180 ret = follow_pfnmap_start(&args); 181 if (ret) 182 goto out_unlock_mmap; 183 } 184 185 io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | 186 (mmio_addr & ~PAGE_MASK)); 187 188 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) 189 goto out_unlock_pt; 190 191 ret = zpci_memcpy_toio(io_addr, buf, length); 192 out_unlock_pt: 193 follow_pfnmap_end(&args); 194 out_unlock_mmap: 195 mmap_read_unlock(current->mm); 196 out_free: 197 if (buf != local_buf) 198 kfree(buf); 199 return ret; 200 } 201 202 static inline int __pcilg_mio_inuser( 203 void __user *dst, const void __iomem *ioaddr, 204 u64 ulen, u8 *status) 205 { 206 union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; 207 u64 cnt = ulen; 208 int shift = ulen * 8; 209 int cc, exception; 210 u64 val, tmp; 211 212 /* 213 * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in 214 * user space) into a register using pcilg then store these bytes at 215 * user address @dst 216 */ 217 exception = 1; 218 asm_inline volatile ( 219 " sacf 256\n" 220 "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" 221 "1: lhi %[exc],0\n" 222 " jne 4f\n" 223 "2: ahi %[shift],-8\n" 224 " srlg %[tmp],%[val],0(%[shift])\n" 225 "3: stc %[tmp],0(%[dst])\n" 226 "5: aghi %[dst],1\n" 227 " brctg %[cnt],2b\n" 228 /* 229 * Use xr to clear exc and set condition code to zero 230 * to ensure flag output is correct for this branch. 231 */ 232 " xr %[exc],%[exc]\n" 233 "4: sacf 768\n" 234 CC_IPM(cc) 235 EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) 236 : [ioaddr_len] "+&d" (ioaddr_len.pair), [exc] "+d" (exception), 237 CC_OUT(cc, cc), [val] "=d" (val), 238 [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), 239 [shift] "+d" (shift) 240 : 241 : CC_CLOBBER_LIST("memory")); 242 243 cc = exception ? -ENXIO : CC_TRANSFORM(cc); 244 /* did we write everything to the user space buffer? */ 245 if (!cc && cnt != 0) 246 cc = -EFAULT; 247 248 *status = ioaddr_len.odd >> 24 & 0xff; 249 return cc; 250 } 251 252 static inline int __memcpy_fromio_inuser(void __user *dst, 253 const void __iomem *src, 254 unsigned long n) 255 { 256 int size, rc = 0; 257 u8 status; 258 259 while (n > 0) { 260 size = zpci_get_max_io_size((u64 __force) src, 261 (u64 __force) dst, n, 262 ZPCI_MAX_READ_SIZE); 263 rc = __pcilg_mio_inuser(dst, src, size, &status); 264 if (rc) 265 break; 266 src += size; 267 dst += size; 268 n -= size; 269 } 270 if (rc) 271 zpci_err_mmio(rc, status, (__force u64) dst); 272 return rc; 273 } 274 275 SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, 276 void __user *, user_buffer, size_t, length) 277 { 278 struct follow_pfnmap_args args = { }; 279 u8 local_buf[64]; 280 void __iomem *io_addr; 281 void *buf; 282 struct vm_area_struct *vma; 283 long ret; 284 285 if (!zpci_is_enabled()) 286 return -ENODEV; 287 288 if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) 289 return -EINVAL; 290 291 /* 292 * We only support read access to MIO capable devices if we are on 293 * a MIO enabled system. Otherwise we would have to check for every 294 * address if it is a special ZPCI_ADDR and would have to do 295 * a pfn lookup which we don't need for MIO capable devices. Currently 296 * ISM devices are the only devices without MIO support and there is no 297 * known need for accessing these from userspace. 298 */ 299 if (static_branch_likely(&have_mio)) { 300 ret = __memcpy_fromio_inuser( 301 user_buffer, (const void __iomem *)mmio_addr, 302 length); 303 return ret; 304 } 305 306 if (length > 64) { 307 buf = kmalloc(length, GFP_KERNEL); 308 if (!buf) 309 return -ENOMEM; 310 } else { 311 buf = local_buf; 312 } 313 314 mmap_read_lock(current->mm); 315 ret = -EINVAL; 316 vma = vma_lookup(current->mm, mmio_addr); 317 if (!vma) 318 goto out_unlock_mmap; 319 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 320 goto out_unlock_mmap; 321 ret = -EACCES; 322 if (!(vma->vm_flags & VM_READ)) 323 goto out_unlock_mmap; 324 325 args.vma = vma; 326 args.address = mmio_addr; 327 ret = follow_pfnmap_start(&args); 328 if (ret) { 329 fixup_user_fault(current->mm, mmio_addr, 0, NULL); 330 ret = follow_pfnmap_start(&args); 331 if (ret) 332 goto out_unlock_mmap; 333 } 334 335 io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | 336 (mmio_addr & ~PAGE_MASK)); 337 338 if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { 339 ret = -EFAULT; 340 goto out_unlock_pt; 341 } 342 ret = zpci_memcpy_fromio(buf, io_addr, length); 343 344 out_unlock_pt: 345 follow_pfnmap_end(&args); 346 out_unlock_mmap: 347 mmap_read_unlock(current->mm); 348 349 if (!ret && copy_to_user(user_buffer, buf, length)) 350 ret = -EFAULT; 351 352 if (buf != local_buf) 353 kfree(buf); 354 return ret; 355 } 356