1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * s390 specific pci instructions
4 *
5 * Copyright IBM Corp. 2013
6 */
7
8 #include <linux/export.h>
9 #include <linux/errno.h>
10 #include <linux/delay.h>
11 #include <linux/jump_label.h>
12 #include <asm/asm-extable.h>
13 #include <asm/facility.h>
14 #include <asm/pci_insn.h>
15 #include <asm/pci_debug.h>
16 #include <asm/pci_io.h>
17 #include <asm/processor.h>
18 #include <asm/asm.h>
19
20 #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */
21 #define ZPCI_ERR_INSN_DATA_SIZE 32
22
23 struct zpci_err_insn_data {
24 u8 insn;
25 u8 cc;
26 u8 status;
27 u8 pad0[5];
28 union {
29 struct {
30 u64 req;
31 union {
32 /* PCILG/PCISTG/MPCIFC */
33 struct {
34 u64 offset;
35 u64 pad1;
36 };
37 /* RPCIT */
38 struct {
39 u64 iova;
40 u64 range;
41 };
42 };
43 };
44 /* PCILGI/PCISTGI */
45 struct {
46 u64 addr;
47 u64 len;
48 u64 pad2;
49 };
50 };
51 };
52
53 static_assert(sizeof(struct zpci_err_insn_data) <= ZPCI_ERR_INSN_DATA_SIZE);
54
zpci_err_insn_rpcit(int lvl,u8 insn,u8 cc,u8 status,u64 req,u64 iova,u64 range)55 static inline void zpci_err_insn_rpcit(int lvl, u8 insn, u8 cc, u8 status,
56 u64 req, u64 iova, u64 range)
57 {
58 struct zpci_err_insn_data data = {
59 .insn = insn, .cc = cc, .status = status,
60 .req = req, .iova = iova, .range = range};
61
62 zpci_err_hex_level(lvl, &data, sizeof(data));
63 }
64
zpci_err_insn_req(int lvl,u8 insn,u8 cc,u8 status,u64 req,u64 offset)65 static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status,
66 u64 req, u64 offset)
67 {
68 struct zpci_err_insn_data data = {
69 .insn = insn, .cc = cc, .status = status,
70 .req = req, .offset = offset};
71
72 zpci_err_hex_level(lvl, &data, sizeof(data));
73 }
74
zpci_err_insn_addr(int lvl,u8 insn,u8 cc,u8 status,u64 addr,u64 len)75 static inline void zpci_err_insn_addr(int lvl, u8 insn, u8 cc, u8 status,
76 u64 addr, u64 len)
77 {
78 struct zpci_err_insn_data data = {
79 .insn = insn, .cc = cc, .status = status,
80 .addr = addr, .len = len};
81
82 zpci_err_hex_level(lvl, &data, sizeof(data));
83 }
84
85 /* Modify PCI Function Controls */
__mpcifc(u64 req,struct zpci_fib * fib,u8 * status)86 static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status)
87 {
88 int cc;
89
90 asm volatile (
91 " .insn rxy,0xe300000000d0,%[req],%[fib]\n"
92 CC_IPM(cc)
93 : CC_OUT(cc, cc), [req] "+d" (req), [fib] "+Q" (*fib)
94 :
95 : CC_CLOBBER);
96 *status = req >> 24 & 0xff;
97 return CC_TRANSFORM(cc);
98 }
99
zpci_mod_fc(u64 req,struct zpci_fib * fib,u8 * status)100 u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status)
101 {
102 bool retried = false;
103 u8 cc;
104
105 do {
106 cc = __mpcifc(req, fib, status);
107 if (cc == 2) {
108 msleep(ZPCI_INSN_BUSY_DELAY);
109 if (!retried) {
110 zpci_err_insn_req(1, 'M', cc, *status, req, 0);
111 retried = true;
112 }
113 }
114 } while (cc == 2);
115
116 if (cc)
117 zpci_err_insn_req(0, 'M', cc, *status, req, 0);
118 else if (retried)
119 zpci_err_insn_req(1, 'M', cc, *status, req, 0);
120
121 return cc;
122 }
123 EXPORT_SYMBOL_GPL(zpci_mod_fc);
124
125 /* Refresh PCI Translations */
__rpcit(u64 fn,u64 addr,u64 range,u8 * status)126 static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status)
127 {
128 union register_pair addr_range = {.even = addr, .odd = range};
129 int cc;
130
131 asm volatile (
132 " .insn rre,0xb9d30000,%[fn],%[addr_range]\n"
133 CC_IPM(cc)
134 : CC_OUT(cc, cc), [fn] "+d" (fn)
135 : [addr_range] "d" (addr_range.pair)
136 : CC_CLOBBER);
137 *status = fn >> 24 & 0xff;
138 return CC_TRANSFORM(cc);
139 }
140
zpci_refresh_trans(u32 fh,u64 addr,u64 range)141 int zpci_refresh_trans(u32 fh, u64 addr, u64 range)
142 {
143 u64 req = ZPCI_CREATE_REQ(fh, 0, 0);
144 bool retried = false;
145 u8 cc, status;
146
147 do {
148 cc = __rpcit(req, addr, range, &status);
149 if (cc == 2) {
150 udelay(ZPCI_INSN_BUSY_DELAY);
151 if (!retried) {
152 zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range);
153 retried = true;
154 }
155 }
156 } while (cc == 2);
157
158 if (cc)
159 zpci_err_insn_rpcit(0, 'R', cc, status, req, addr, range);
160 else if (retried)
161 zpci_err_insn_rpcit(1, 'R', cc, status, req, addr, range);
162
163 if (cc == 1 && (status == 4 || status == 16))
164 return -ENOMEM;
165
166 return (cc) ? -EIO : 0;
167 }
168
169 /* Set Interruption Controls */
zpci_set_irq_ctrl(u16 ctl,u8 isc,union zpci_sic_iib * iib)170 int zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib)
171 {
172 if (!test_facility(72))
173 return -EIO;
174
175 asm volatile(
176 ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]"
177 : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib));
178
179 return 0;
180 }
181 EXPORT_SYMBOL_GPL(zpci_set_irq_ctrl);
182
183 /* PCI Load */
____pcilg(u64 * data,u64 req,u64 offset,u8 * status)184 static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status)
185 {
186 union register_pair req_off = {.even = req, .odd = offset};
187 int cc, exception;
188 u64 __data;
189
190 exception = 1;
191 asm_inline volatile (
192 "0: .insn rre,0xb9d20000,%[data],%[req_off]\n"
193 " lhi %[exc],0\n"
194 "1:\n"
195 CC_IPM(cc)
196 EX_TABLE_INSN(0b, 1b)
197 : CC_OUT(cc, cc), [data] "=d" (__data),
198 [req_off] "+d" (req_off.pair), [exc] "+d" (exception)
199 :
200 : CC_CLOBBER);
201 *status = req_off.even >> 24 & 0xff;
202 *data = __data;
203 return exception ? -ENXIO : CC_TRANSFORM(cc);
204 }
205
__pcilg(u64 * data,u64 req,u64 offset,u8 * status)206 static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status)
207 {
208 u64 __data;
209 int cc;
210
211 cc = ____pcilg(&__data, req, offset, status);
212 if (!cc)
213 *data = __data;
214
215 return cc;
216 }
217
__zpci_load(u64 * data,u64 req,u64 offset)218 int __zpci_load(u64 *data, u64 req, u64 offset)
219 {
220 bool retried = false;
221 u8 status;
222 int cc;
223
224 do {
225 cc = __pcilg(data, req, offset, &status);
226 if (cc == 2) {
227 udelay(ZPCI_INSN_BUSY_DELAY);
228 if (!retried) {
229 zpci_err_insn_req(1, 'l', cc, status, req, offset);
230 retried = true;
231 }
232 }
233 } while (cc == 2);
234
235 if (cc)
236 zpci_err_insn_req(0, 'l', cc, status, req, offset);
237 else if (retried)
238 zpci_err_insn_req(1, 'l', cc, status, req, offset);
239
240 return (cc > 0) ? -EIO : cc;
241 }
242 EXPORT_SYMBOL_GPL(__zpci_load);
243
zpci_load_fh(u64 * data,const volatile void __iomem * addr,unsigned long len)244 static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr,
245 unsigned long len)
246 {
247 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)];
248 u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len);
249
250 return __zpci_load(data, req, ZPCI_OFFSET(addr));
251 }
252
__pcilg_mio(u64 * data,u64 ioaddr,u64 len,u8 * status)253 static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status)
254 {
255 union register_pair ioaddr_len = {.even = ioaddr, .odd = len};
256 int cc, exception;
257 u64 __data;
258
259 exception = 1;
260 asm_inline volatile (
261 "0: .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n"
262 " lhi %[exc],0\n"
263 "1:\n"
264 CC_IPM(cc)
265 EX_TABLE_INSN(0b, 1b)
266 : CC_OUT(cc, cc), [data] "=d" (__data),
267 [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception)
268 :
269 : CC_CLOBBER);
270 *status = ioaddr_len.odd >> 24 & 0xff;
271 *data = __data;
272 return exception ? -ENXIO : CC_TRANSFORM(cc);
273 }
274
zpci_load(u64 * data,const volatile void __iomem * addr,unsigned long len)275 int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len)
276 {
277 u8 status;
278 int cc;
279
280 if (!static_branch_unlikely(&have_mio))
281 return zpci_load_fh(data, addr, len);
282
283 cc = __pcilg_mio(data, (__force u64) addr, len, &status);
284 if (cc)
285 zpci_err_insn_addr(0, 'L', cc, status, (__force u64) addr, len);
286
287 return (cc > 0) ? -EIO : cc;
288 }
289 EXPORT_SYMBOL_GPL(zpci_load);
290
291 /* PCI Store */
__pcistg(u64 data,u64 req,u64 offset,u8 * status)292 static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status)
293 {
294 union register_pair req_off = {.even = req, .odd = offset};
295 int cc, exception;
296
297 exception = 1;
298 asm_inline volatile (
299 "0: .insn rre,0xb9d00000,%[data],%[req_off]\n"
300 " lhi %[exc],0\n"
301 "1:\n"
302 CC_IPM(cc)
303 EX_TABLE_INSN(0b, 1b)
304 : CC_OUT(cc, cc), [req_off] "+d" (req_off.pair), [exc] "+d" (exception)
305 : [data] "d" (data)
306 : CC_CLOBBER);
307 *status = req_off.even >> 24 & 0xff;
308 return exception ? -ENXIO : CC_TRANSFORM(cc);
309 }
310
__zpci_store(u64 data,u64 req,u64 offset)311 int __zpci_store(u64 data, u64 req, u64 offset)
312 {
313 bool retried = false;
314 u8 status;
315 int cc;
316
317 do {
318 cc = __pcistg(data, req, offset, &status);
319 if (cc == 2) {
320 udelay(ZPCI_INSN_BUSY_DELAY);
321 if (!retried) {
322 zpci_err_insn_req(1, 's', cc, status, req, offset);
323 retried = true;
324 }
325 }
326 } while (cc == 2);
327
328 if (cc)
329 zpci_err_insn_req(0, 's', cc, status, req, offset);
330 else if (retried)
331 zpci_err_insn_req(1, 's', cc, status, req, offset);
332
333 return (cc > 0) ? -EIO : cc;
334 }
335 EXPORT_SYMBOL_GPL(__zpci_store);
336
zpci_store_fh(const volatile void __iomem * addr,u64 data,unsigned long len)337 static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data,
338 unsigned long len)
339 {
340 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)];
341 u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len);
342
343 return __zpci_store(data, req, ZPCI_OFFSET(addr));
344 }
345
__pcistg_mio(u64 data,u64 ioaddr,u64 len,u8 * status)346 static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status)
347 {
348 union register_pair ioaddr_len = {.even = ioaddr, .odd = len};
349 int cc, exception;
350
351 exception = 1;
352 asm_inline volatile (
353 "0: .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n"
354 " lhi %[exc],0\n"
355 "1:\n"
356 CC_IPM(cc)
357 EX_TABLE_INSN(0b, 1b)
358 : CC_OUT(cc, cc), [ioaddr_len] "+d" (ioaddr_len.pair), [exc] "+d" (exception)
359 : [data] "d" (data)
360 : CC_CLOBBER_LIST("memory"));
361 *status = ioaddr_len.odd >> 24 & 0xff;
362 return exception ? -ENXIO : CC_TRANSFORM(cc);
363 }
364
zpci_store(const volatile void __iomem * addr,u64 data,unsigned long len)365 int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len)
366 {
367 u8 status;
368 int cc;
369
370 if (!static_branch_unlikely(&have_mio))
371 return zpci_store_fh(addr, data, len);
372
373 cc = __pcistg_mio(data, (__force u64) addr, len, &status);
374 if (cc)
375 zpci_err_insn_addr(0, 'S', cc, status, (__force u64) addr, len);
376
377 return (cc > 0) ? -EIO : cc;
378 }
379 EXPORT_SYMBOL_GPL(zpci_store);
380
381 /* PCI Store Block */
__pcistb(const u64 * data,u64 req,u64 offset,u8 * status)382 static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status)
383 {
384 int cc, exception;
385
386 exception = 1;
387 asm_inline volatile (
388 "0: .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n"
389 " lhi %[exc],0\n"
390 "1:\n"
391 CC_IPM(cc)
392 EX_TABLE_INSN(0b, 1b)
393 : CC_OUT(cc, cc), [req] "+d" (req), [exc] "+d" (exception)
394 : [offset] "d" (offset), [data] "Q" (*data)
395 : CC_CLOBBER);
396 *status = req >> 24 & 0xff;
397 return exception ? -ENXIO : CC_TRANSFORM(cc);
398 }
399
__zpci_store_block(const u64 * data,u64 req,u64 offset)400 int __zpci_store_block(const u64 *data, u64 req, u64 offset)
401 {
402 bool retried = false;
403 u8 status;
404 int cc;
405
406 do {
407 cc = __pcistb(data, req, offset, &status);
408 if (cc == 2) {
409 udelay(ZPCI_INSN_BUSY_DELAY);
410 if (!retried) {
411 zpci_err_insn_req(0, 'b', cc, status, req, offset);
412 retried = true;
413 }
414 }
415 } while (cc == 2);
416
417 if (cc)
418 zpci_err_insn_req(0, 'b', cc, status, req, offset);
419 else if (retried)
420 zpci_err_insn_req(1, 'b', cc, status, req, offset);
421
422 return (cc > 0) ? -EIO : cc;
423 }
424 EXPORT_SYMBOL_GPL(__zpci_store_block);
425
zpci_write_block_fh(volatile void __iomem * dst,const void * src,unsigned long len)426 static inline int zpci_write_block_fh(volatile void __iomem *dst,
427 const void *src, unsigned long len)
428 {
429 struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)];
430 u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len);
431 u64 offset = ZPCI_OFFSET(dst);
432
433 return __zpci_store_block(src, req, offset);
434 }
435
__pcistb_mio(const u64 * data,u64 ioaddr,u64 len,u8 * status)436 static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status)
437 {
438 int cc, exception;
439
440 exception = 1;
441 asm_inline volatile (
442 "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n"
443 " lhi %[exc],0\n"
444 "1:\n"
445 CC_IPM(cc)
446 EX_TABLE_INSN(0b, 1b)
447 : CC_OUT(cc, cc), [len] "+d" (len), [exc] "+d" (exception)
448 : [ioaddr] "d" (ioaddr), [data] "Q" (*data)
449 : CC_CLOBBER);
450 *status = len >> 24 & 0xff;
451 return exception ? -ENXIO : CC_TRANSFORM(cc);
452 }
453
zpci_write_block(volatile void __iomem * dst,const void * src,unsigned long len)454 int zpci_write_block(volatile void __iomem *dst,
455 const void *src, unsigned long len)
456 {
457 u8 status;
458 int cc;
459
460 if (!static_branch_unlikely(&have_mio))
461 return zpci_write_block_fh(dst, src, len);
462
463 cc = __pcistb_mio(src, (__force u64) dst, len, &status);
464 if (cc)
465 zpci_err_insn_addr(0, 'B', cc, status, (__force u64) dst, len);
466
467 return (cc > 0) ? -EIO : cc;
468 }
469 EXPORT_SYMBOL_GPL(zpci_write_block);
470
__pciwb_mio(void)471 static inline void __pciwb_mio(void)
472 {
473 asm volatile (".insn rre,0xb9d50000,0,0");
474 }
475
zpci_barrier(void)476 void zpci_barrier(void)
477 {
478 if (static_branch_likely(&have_mio))
479 __pciwb_mio();
480 }
481 EXPORT_SYMBOL_GPL(zpci_barrier);
482