xref: /linux/drivers/s390/cio/qdio_main.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Linux for s390 qdio support, buffer handling, qdio API and module support.
4  *
5  * Copyright IBM Corp. 2000, 2008
6  * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
7  *	      Jan Glauber <jang@linux.vnet.ibm.com>
8  * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
9  */
10 #include <linux/module.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/kmemleak.h>
14 #include <linux/delay.h>
15 #include <linux/gfp.h>
16 #include <linux/io.h>
17 #include <linux/atomic.h>
18 #include <asm/debug.h>
19 #include <asm/qdio.h>
20 #include <asm/ipl.h>
21 
22 #include "cio.h"
23 #include "css.h"
24 #include "device.h"
25 #include "qdio.h"
26 #include "qdio_debug.h"
27 
28 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
29 	"Jan Glauber <jang@linux.vnet.ibm.com>");
30 MODULE_DESCRIPTION("QDIO base support");
31 MODULE_LICENSE("GPL");
32 
do_siga_sync(unsigned long schid,unsigned long out_mask,unsigned long in_mask,unsigned int fc)33 static inline int do_siga_sync(unsigned long schid,
34 			       unsigned long out_mask, unsigned long in_mask,
35 			       unsigned int fc)
36 {
37 	int cc;
38 
39 	asm volatile(
40 		"	lgr	0,%[fc]\n"
41 		"	lgr	1,%[schid]\n"
42 		"	lgr	2,%[out]\n"
43 		"	lgr	3,%[in]\n"
44 		"	siga	0\n"
45 		"	ipm	%[cc]\n"
46 		"	srl	%[cc],28\n"
47 		: [cc] "=&d" (cc)
48 		: [fc] "d" (fc), [schid] "d" (schid),
49 		  [out] "d" (out_mask), [in] "d" (in_mask)
50 		: "cc", "0", "1", "2", "3");
51 	return cc;
52 }
53 
do_siga_input(unsigned long schid,unsigned long mask,unsigned long fc)54 static inline int do_siga_input(unsigned long schid, unsigned long mask,
55 				unsigned long fc)
56 {
57 	int cc;
58 
59 	asm volatile(
60 		"	lgr	0,%[fc]\n"
61 		"	lgr	1,%[schid]\n"
62 		"	lgr	2,%[mask]\n"
63 		"	siga	0\n"
64 		"	ipm	%[cc]\n"
65 		"	srl	%[cc],28\n"
66 		: [cc] "=&d" (cc)
67 		: [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask)
68 		: "cc", "0", "1", "2");
69 	return cc;
70 }
71 
72 /**
73  * do_siga_output - perform SIGA-w/wt function
74  * @schid: subchannel id or in case of QEBSM the subchannel token
75  * @mask: which output queues to process
76  * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
77  * @fc: function code to perform
78  * @aob: asynchronous operation block
79  *
80  * Returns condition code.
81  * Note: For IQDC unicast queues only the highest priority queue is processed.
82  */
do_siga_output(unsigned long schid,unsigned long mask,unsigned int * bb,unsigned long fc,dma64_t aob)83 static inline int do_siga_output(unsigned long schid, unsigned long mask,
84 				 unsigned int *bb, unsigned long fc,
85 				 dma64_t aob)
86 {
87 	int cc;
88 
89 	asm volatile(
90 		"	lgr	0,%[fc]\n"
91 		"	lgr	1,%[schid]\n"
92 		"	lgr	2,%[mask]\n"
93 		"	lgr	3,%[aob]\n"
94 		"	siga	0\n"
95 		"	lgr	%[fc],0\n"
96 		"	ipm	%[cc]\n"
97 		"	srl	%[cc],28\n"
98 		: [cc] "=&d" (cc), [fc] "+&d" (fc)
99 		: [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob)
100 		: "cc", "0", "1", "2", "3");
101 	*bb = fc >> 31;
102 	return cc;
103 }
104 
105 /**
106  * qdio_do_eqbs - extract buffer states for QEBSM
107  * @q: queue to manipulate
108  * @state: state of the extracted buffers
109  * @start: buffer number to start at
110  * @count: count of buffers to examine
111  * @auto_ack: automatically acknowledge buffers
112  *
113  * Returns the number of successfully extracted equal buffer states.
114  * Stops processing if a state is different from the last buffers state.
115  */
qdio_do_eqbs(struct qdio_q * q,unsigned char * state,int start,int count,int auto_ack)116 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
117 			int start, int count, int auto_ack)
118 {
119 	int tmp_count = count, tmp_start = start, nr = q->nr;
120 	unsigned int ccq = 0;
121 
122 	qperf_inc(q, eqbs);
123 
124 	if (!q->is_input_q)
125 		nr += q->irq_ptr->nr_input_qs;
126 again:
127 	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
128 		      auto_ack);
129 
130 	switch (ccq) {
131 	case 0:
132 	case 32:
133 		/* all done, or next buffer state different */
134 		return count - tmp_count;
135 	case 96:
136 		/* not all buffers processed */
137 		qperf_inc(q, eqbs_partial);
138 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",
139 			tmp_count);
140 		return count - tmp_count;
141 	case 97:
142 		/* no buffer processed */
143 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
144 		goto again;
145 	default:
146 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
147 		DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
148 		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
149 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,
150 			   q->first_to_check, count, q->irq_ptr->int_parm);
151 		return 0;
152 	}
153 }
154 
155 /**
156  * qdio_do_sqbs - set buffer states for QEBSM
157  * @q: queue to manipulate
158  * @state: new state of the buffers
159  * @start: first buffer number to change
160  * @count: how many buffers to change
161  *
162  * Returns the number of successfully changed buffers.
163  * Does retrying until the specified count of buffer states is set or an
164  * error occurs.
165  */
qdio_do_sqbs(struct qdio_q * q,unsigned char state,int start,int count)166 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
167 			int count)
168 {
169 	unsigned int ccq = 0;
170 	int tmp_count = count, tmp_start = start;
171 	int nr = q->nr;
172 
173 	qperf_inc(q, sqbs);
174 
175 	if (!q->is_input_q)
176 		nr += q->irq_ptr->nr_input_qs;
177 again:
178 	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
179 
180 	switch (ccq) {
181 	case 0:
182 	case 32:
183 		/* all done, or active buffer adapter-owned */
184 		WARN_ON_ONCE(tmp_count);
185 		return count - tmp_count;
186 	case 96:
187 		/* not all buffers processed */
188 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
189 		qperf_inc(q, sqbs_partial);
190 		goto again;
191 	default:
192 		DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
193 		DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
194 		DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
195 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,
196 			   q->first_to_check, count, q->irq_ptr->int_parm);
197 		return 0;
198 	}
199 }
200 
201 /*
202  * Returns number of examined buffers and their common state in *state.
203  * Requested number of buffers-to-examine must be > 0.
204  */
get_buf_states(struct qdio_q * q,unsigned int bufnr,unsigned char * state,unsigned int count,int auto_ack)205 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
206 				 unsigned char *state, unsigned int count,
207 				 int auto_ack)
208 {
209 	unsigned char __state = 0;
210 	int i = 1;
211 
212 	if (is_qebsm(q))
213 		return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
214 
215 	/* get initial state: */
216 	__state = q->slsb.val[bufnr];
217 
218 	/* Bail out early if there is no work on the queue: */
219 	if (__state & SLSB_OWNER_CU)
220 		goto out;
221 
222 	for (; i < count; i++) {
223 		bufnr = next_buf(bufnr);
224 
225 		/* stop if next state differs from initial state: */
226 		if (q->slsb.val[bufnr] != __state)
227 			break;
228 	}
229 
230 out:
231 	*state = __state;
232 	return i;
233 }
234 
get_buf_state(struct qdio_q * q,unsigned int bufnr,unsigned char * state,int auto_ack)235 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
236 				unsigned char *state, int auto_ack)
237 {
238 	return get_buf_states(q, bufnr, state, 1, auto_ack);
239 }
240 
241 /* wrap-around safe setting of slsb states, returns number of changed buffers */
set_buf_states(struct qdio_q * q,int bufnr,unsigned char state,int count)242 static inline int set_buf_states(struct qdio_q *q, int bufnr,
243 				 unsigned char state, int count)
244 {
245 	int i;
246 
247 	if (is_qebsm(q))
248 		return qdio_do_sqbs(q, state, bufnr, count);
249 
250 	/* Ensure that all preceding changes to the SBALs are visible: */
251 	mb();
252 
253 	for (i = 0; i < count; i++) {
254 		WRITE_ONCE(q->slsb.val[bufnr], state);
255 		bufnr = next_buf(bufnr);
256 	}
257 
258 	/* Make our SLSB changes visible: */
259 	mb();
260 
261 	return count;
262 }
263 
set_buf_state(struct qdio_q * q,int bufnr,unsigned char state)264 static inline int set_buf_state(struct qdio_q *q, int bufnr,
265 				unsigned char state)
266 {
267 	return set_buf_states(q, bufnr, state, 1);
268 }
269 
270 /* set slsb states to initial state */
qdio_init_buf_states(struct qdio_irq * irq_ptr)271 static void qdio_init_buf_states(struct qdio_irq *irq_ptr)
272 {
273 	struct qdio_q *q;
274 	int i;
275 
276 	for_each_input_queue(irq_ptr, q, i)
277 		set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
278 			       QDIO_MAX_BUFFERS_PER_Q);
279 	for_each_output_queue(irq_ptr, q, i)
280 		set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
281 			       QDIO_MAX_BUFFERS_PER_Q);
282 }
283 
qdio_siga_sync(struct qdio_q * q,unsigned int output,unsigned int input)284 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
285 			  unsigned int input)
286 {
287 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
288 	unsigned int fc = QDIO_SIGA_SYNC;
289 	int cc;
290 
291 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
292 	qperf_inc(q, siga_sync);
293 
294 	if (is_qebsm(q)) {
295 		schid = q->irq_ptr->sch_token;
296 		fc |= QDIO_SIGA_QEBSM_FLAG;
297 	}
298 
299 	cc = do_siga_sync(schid, output, input, fc);
300 	if (unlikely(cc))
301 		DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
302 	return (cc) ? -EIO : 0;
303 }
304 
qdio_sync_input_queue(struct qdio_q * q)305 static inline int qdio_sync_input_queue(struct qdio_q *q)
306 {
307 	return qdio_siga_sync(q, 0, q->mask);
308 }
309 
qdio_sync_output_queue(struct qdio_q * q)310 static inline int qdio_sync_output_queue(struct qdio_q *q)
311 {
312 	return qdio_siga_sync(q, q->mask, 0);
313 }
314 
qdio_siga_sync_q(struct qdio_q * q)315 static inline int qdio_siga_sync_q(struct qdio_q *q)
316 {
317 	if (q->is_input_q)
318 		return qdio_sync_input_queue(q);
319 	else
320 		return qdio_sync_output_queue(q);
321 }
322 
qdio_siga_output(struct qdio_q * q,unsigned int count,unsigned int * busy_bit,dma64_t aob)323 static int qdio_siga_output(struct qdio_q *q, unsigned int count,
324 			    unsigned int *busy_bit, dma64_t aob)
325 {
326 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
327 	unsigned int fc = QDIO_SIGA_WRITE;
328 	u64 start_time = 0;
329 	int retries = 0, cc;
330 
331 	if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) {
332 		if (count > 1)
333 			fc = QDIO_SIGA_WRITEM;
334 		else if (aob)
335 			fc = QDIO_SIGA_WRITEQ;
336 	}
337 
338 	if (is_qebsm(q)) {
339 		schid = q->irq_ptr->sch_token;
340 		fc |= QDIO_SIGA_QEBSM_FLAG;
341 	}
342 again:
343 	cc = do_siga_output(schid, q->mask, busy_bit, fc, aob);
344 
345 	/* hipersocket busy condition */
346 	if (unlikely(*busy_bit)) {
347 		retries++;
348 
349 		if (!start_time) {
350 			start_time = get_tod_clock_fast();
351 			goto again;
352 		}
353 		if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE)
354 			goto again;
355 	}
356 	if (retries) {
357 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
358 			      "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
359 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
360 	}
361 	return cc;
362 }
363 
qdio_siga_input(struct qdio_q * q)364 static inline int qdio_siga_input(struct qdio_q *q)
365 {
366 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);
367 	unsigned int fc = QDIO_SIGA_READ;
368 	int cc;
369 
370 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
371 	qperf_inc(q, siga_read);
372 
373 	if (is_qebsm(q)) {
374 		schid = q->irq_ptr->sch_token;
375 		fc |= QDIO_SIGA_QEBSM_FLAG;
376 	}
377 
378 	cc = do_siga_input(schid, q->mask, fc);
379 	if (unlikely(cc))
380 		DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
381 	return (cc) ? -EIO : 0;
382 }
383 
debug_get_buf_state(struct qdio_q * q,unsigned int bufnr,unsigned char * state)384 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
385 			unsigned char *state)
386 {
387 	if (qdio_need_siga_sync(q->irq_ptr))
388 		qdio_siga_sync_q(q);
389 	return get_buf_state(q, bufnr, state, 0);
390 }
391 
qdio_stop_polling(struct qdio_q * q)392 static inline void qdio_stop_polling(struct qdio_q *q)
393 {
394 	if (!q->u.in.batch_count)
395 		return;
396 
397 	qperf_inc(q, stop_polling);
398 
399 	/* show the card that we are not polling anymore */
400 	set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT,
401 		       q->u.in.batch_count);
402 	q->u.in.batch_count = 0;
403 }
404 
account_sbals(struct qdio_q * q,unsigned int count)405 static inline void account_sbals(struct qdio_q *q, unsigned int count)
406 {
407 	q->q_stats.nr_sbal_total += count;
408 	q->q_stats.nr_sbals[ilog2(count)]++;
409 }
410 
process_buffer_error(struct qdio_q * q,unsigned int start,int count)411 static void process_buffer_error(struct qdio_q *q, unsigned int start,
412 				 int count)
413 {
414 	/* special handling for no target buffer empty */
415 	if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q &&
416 	    q->sbal[start]->element[15].sflags == 0x10) {
417 		qperf_inc(q, target_full);
418 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);
419 		return;
420 	}
421 
422 	DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
423 	DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
424 	DBF_ERROR("FTC:%3d C:%3d", start, count);
425 	DBF_ERROR("F14:%2x F15:%2x",
426 		  q->sbal[start]->element[14].sflags,
427 		  q->sbal[start]->element[15].sflags);
428 }
429 
inbound_handle_work(struct qdio_q * q,unsigned int start,int count,bool auto_ack)430 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start,
431 				       int count, bool auto_ack)
432 {
433 	/* ACK the newest SBAL: */
434 	if (!auto_ack)
435 		set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK);
436 
437 	if (!q->u.in.batch_count)
438 		q->u.in.batch_start = start;
439 	q->u.in.batch_count += count;
440 }
441 
get_inbound_buffer_frontier(struct qdio_q * q,unsigned int start,unsigned int * error)442 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start,
443 				       unsigned int *error)
444 {
445 	unsigned char state = 0;
446 	int count;
447 
448 	q->timestamp = get_tod_clock_fast();
449 
450 	count = atomic_read(&q->nr_buf_used);
451 	if (!count)
452 		return 0;
453 
454 	if (qdio_need_siga_sync(q->irq_ptr))
455 		qdio_sync_input_queue(q);
456 
457 	count = get_buf_states(q, start, &state, count, 1);
458 	if (!count)
459 		return 0;
460 
461 	switch (state) {
462 	case SLSB_P_INPUT_PRIMED:
463 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,
464 			      count);
465 
466 		inbound_handle_work(q, start, count, is_qebsm(q));
467 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
468 			qperf_inc(q, inbound_queue_full);
469 		if (q->irq_ptr->perf_stat_enabled)
470 			account_sbals(q, count);
471 		return count;
472 	case SLSB_P_INPUT_ERROR:
473 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,
474 			      count);
475 
476 		*error = QDIO_ERROR_SLSB_STATE;
477 		process_buffer_error(q, start, count);
478 		inbound_handle_work(q, start, count, false);
479 		if (atomic_sub_return(count, &q->nr_buf_used) == 0)
480 			qperf_inc(q, inbound_queue_full);
481 		if (q->irq_ptr->perf_stat_enabled)
482 			account_sbals_error(q, count);
483 		return count;
484 	case SLSB_CU_INPUT_EMPTY:
485 		if (q->irq_ptr->perf_stat_enabled)
486 			q->q_stats.nr_sbal_nop++;
487 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",
488 			      q->nr, start);
489 		return 0;
490 	case SLSB_P_INPUT_NOT_INIT:
491 	case SLSB_P_INPUT_ACK:
492 		/* We should never see this state, throw a WARN: */
493 	default:
494 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
495 			      "found state %#x at index %u on queue %u\n",
496 			      state, start, q->nr);
497 		return 0;
498 	}
499 }
500 
qdio_inspect_input_queue(struct ccw_device * cdev,unsigned int nr,unsigned int * bufnr,unsigned int * error)501 int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr,
502 			     unsigned int *bufnr, unsigned int *error)
503 {
504 	struct qdio_irq *irq = cdev->private->qdio_data;
505 	unsigned int start;
506 	struct qdio_q *q;
507 	int count;
508 
509 	if (!irq)
510 		return -ENODEV;
511 
512 	q = irq->input_qs[nr];
513 	start = q->first_to_check;
514 	*error = 0;
515 
516 	count = get_inbound_buffer_frontier(q, start, error);
517 	if (count == 0)
518 		return 0;
519 
520 	*bufnr = start;
521 	q->first_to_check = add_buf(start, count);
522 	return count;
523 }
524 EXPORT_SYMBOL_GPL(qdio_inspect_input_queue);
525 
qdio_inbound_q_done(struct qdio_q * q,unsigned int start)526 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start)
527 {
528 	unsigned char state = 0;
529 
530 	if (!atomic_read(&q->nr_buf_used))
531 		return 1;
532 
533 	if (qdio_need_siga_sync(q->irq_ptr))
534 		qdio_sync_input_queue(q);
535 	get_buf_state(q, start, &state, 0);
536 
537 	if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
538 		/* more work coming */
539 		return 0;
540 
541 	return 1;
542 }
543 
get_outbound_buffer_frontier(struct qdio_q * q,unsigned int start,unsigned int * error)544 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start,
545 					unsigned int *error)
546 {
547 	unsigned char state = 0;
548 	int count;
549 
550 	q->timestamp = get_tod_clock_fast();
551 
552 	count = atomic_read(&q->nr_buf_used);
553 	if (!count)
554 		return 0;
555 
556 	if (qdio_need_siga_sync(q->irq_ptr))
557 		qdio_sync_output_queue(q);
558 
559 	count = get_buf_states(q, start, &state, count, 0);
560 	if (!count)
561 		return 0;
562 
563 	switch (state) {
564 	case SLSB_P_OUTPUT_PENDING:
565 		*error = QDIO_ERROR_SLSB_PENDING;
566 		fallthrough;
567 	case SLSB_P_OUTPUT_EMPTY:
568 		/* the adapter got it */
569 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
570 			"out empty:%1d %02x", q->nr, count);
571 
572 		atomic_sub(count, &q->nr_buf_used);
573 		if (q->irq_ptr->perf_stat_enabled)
574 			account_sbals(q, count);
575 		return count;
576 	case SLSB_P_OUTPUT_ERROR:
577 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",
578 			      q->nr, count);
579 
580 		*error = QDIO_ERROR_SLSB_STATE;
581 		process_buffer_error(q, start, count);
582 		atomic_sub(count, &q->nr_buf_used);
583 		if (q->irq_ptr->perf_stat_enabled)
584 			account_sbals_error(q, count);
585 		return count;
586 	case SLSB_CU_OUTPUT_PRIMED:
587 		/* the adapter has not fetched the output yet */
588 		if (q->irq_ptr->perf_stat_enabled)
589 			q->q_stats.nr_sbal_nop++;
590 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
591 			      q->nr);
592 		return 0;
593 	case SLSB_P_OUTPUT_HALTED:
594 		return 0;
595 	case SLSB_P_OUTPUT_NOT_INIT:
596 		/* We should never see this state, throw a WARN: */
597 	default:
598 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,
599 			      "found state %#x at index %u on queue %u\n",
600 			      state, start, q->nr);
601 		return 0;
602 	}
603 }
604 
qdio_inspect_output_queue(struct ccw_device * cdev,unsigned int nr,unsigned int * bufnr,unsigned int * error)605 int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr,
606 			      unsigned int *bufnr, unsigned int *error)
607 {
608 	struct qdio_irq *irq = cdev->private->qdio_data;
609 	unsigned int start;
610 	struct qdio_q *q;
611 	int count;
612 
613 	if (!irq)
614 		return -ENODEV;
615 
616 	q = irq->output_qs[nr];
617 	start = q->first_to_check;
618 	*error = 0;
619 
620 	count = get_outbound_buffer_frontier(q, start, error);
621 	if (count == 0)
622 		return 0;
623 
624 	*bufnr = start;
625 	q->first_to_check = add_buf(start, count);
626 	return count;
627 }
628 EXPORT_SYMBOL_GPL(qdio_inspect_output_queue);
629 
qdio_kick_outbound_q(struct qdio_q * q,unsigned int count,dma64_t aob)630 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count,
631 				dma64_t aob)
632 {
633 	int retries = 0, cc;
634 	unsigned int busy_bit;
635 
636 	if (!qdio_need_siga_out(q->irq_ptr))
637 		return 0;
638 
639 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
640 retry:
641 	qperf_inc(q, siga_write);
642 
643 	cc = qdio_siga_output(q, count, &busy_bit, aob);
644 	switch (cc) {
645 	case 0:
646 		break;
647 	case 2:
648 		if (busy_bit) {
649 			while (++retries < QDIO_BUSY_BIT_RETRIES) {
650 				mdelay(QDIO_BUSY_BIT_RETRY_DELAY);
651 				goto retry;
652 			}
653 			DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
654 			cc = -EBUSY;
655 		} else {
656 			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
657 			cc = -ENOBUFS;
658 		}
659 		break;
660 	case 1:
661 	case 3:
662 		DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
663 		cc = -EIO;
664 		break;
665 	}
666 	if (retries) {
667 		DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
668 		DBF_ERROR("count:%u", retries);
669 	}
670 	return cc;
671 }
672 
qdio_set_state(struct qdio_irq * irq_ptr,enum qdio_irq_states state)673 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
674 				  enum qdio_irq_states state)
675 {
676 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
677 
678 	irq_ptr->state = state;
679 	mb();
680 }
681 
qdio_irq_check_sense(struct qdio_irq * irq_ptr,struct irb * irb)682 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
683 {
684 	if (irb->esw.esw0.erw.cons) {
685 		DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
686 		DBF_ERROR_HEX(irb, 64);
687 		DBF_ERROR_HEX(irb->ecw, 64);
688 	}
689 }
690 
691 /* PCI interrupt handler */
qdio_int_handler_pci(struct qdio_irq * irq_ptr)692 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
693 {
694 	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
695 		return;
696 
697 	qdio_deliver_irq(irq_ptr);
698 	irq_ptr->last_data_irq_time = get_lowcore()->int_clock;
699 }
700 
qdio_handle_activate_check(struct qdio_irq * irq_ptr,unsigned long intparm,int cstat,int dstat)701 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,
702 				       unsigned long intparm, int cstat,
703 				       int dstat)
704 {
705 	unsigned int first_to_check = 0;
706 
707 	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
708 	DBF_ERROR("intp :%lx", intparm);
709 	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
710 
711 	/* zfcp wants this: */
712 	if (irq_ptr->nr_input_qs)
713 		first_to_check = irq_ptr->input_qs[0]->first_to_check;
714 
715 	irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0,
716 			       first_to_check, 0, irq_ptr->int_parm);
717 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
718 	/*
719 	 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen.
720 	 * Therefore we call the LGR detection function here.
721 	 */
722 	lgr_info_log();
723 }
724 
qdio_establish_handle_irq(struct qdio_irq * irq_ptr,int cstat,int dstat,int dcc)725 static int qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,
726 				     int dstat, int dcc)
727 {
728 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
729 
730 	if (cstat)
731 		goto error;
732 	if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
733 		goto error;
734 	if (dcc == 1)
735 		return -EAGAIN;
736 	if (!(dstat & DEV_STAT_DEV_END))
737 		goto error;
738 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
739 	return 0;
740 
741 error:
742 	DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
743 	DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
744 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
745 	return -EIO;
746 }
747 
748 /* qdio interrupt handler */
qdio_int_handler(struct ccw_device * cdev,unsigned long intparm,struct irb * irb)749 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
750 		      struct irb *irb)
751 {
752 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
753 	struct subchannel_id schid;
754 	int cstat, dstat, rc, dcc;
755 
756 	if (!intparm || !irq_ptr) {
757 		ccw_device_get_schid(cdev, &schid);
758 		DBF_ERROR("qint:%4x", schid.sch_no);
759 		return;
760 	}
761 
762 	if (irq_ptr->perf_stat_enabled)
763 		irq_ptr->perf_stat.qdio_int++;
764 
765 	if (IS_ERR(irb)) {
766 		DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
767 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
768 		wake_up(&cdev->private->wait_q);
769 		return;
770 	}
771 	qdio_irq_check_sense(irq_ptr, irb);
772 	cstat = irb->scsw.cmd.cstat;
773 	dstat = irb->scsw.cmd.dstat;
774 	dcc   = scsw_cmd_is_valid_cc(&irb->scsw) ? irb->scsw.cmd.cc : 0;
775 	rc    = 0;
776 
777 	switch (irq_ptr->state) {
778 	case QDIO_IRQ_STATE_INACTIVE:
779 		rc = qdio_establish_handle_irq(irq_ptr, cstat, dstat, dcc);
780 		break;
781 	case QDIO_IRQ_STATE_CLEANUP:
782 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
783 		break;
784 	case QDIO_IRQ_STATE_ESTABLISHED:
785 	case QDIO_IRQ_STATE_ACTIVE:
786 		if (cstat & SCHN_STAT_PCI) {
787 			qdio_int_handler_pci(irq_ptr);
788 			return;
789 		}
790 		if (cstat || dstat)
791 			qdio_handle_activate_check(irq_ptr, intparm, cstat,
792 						   dstat);
793 		else if (dcc == 1)
794 			rc = -EAGAIN;
795 		break;
796 	case QDIO_IRQ_STATE_STOPPED:
797 		break;
798 	default:
799 		WARN_ON_ONCE(1);
800 	}
801 
802 	if (rc == -EAGAIN) {
803 		DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qint retry");
804 		rc = ccw_device_start(cdev, irq_ptr->ccw, intparm, 0, 0);
805 		if (!rc)
806 			return;
807 		DBF_ERROR("%4x RETRY ERR", irq_ptr->schid.sch_no);
808 		DBF_ERROR("rc:%4x", rc);
809 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
810 	}
811 
812 	wake_up(&cdev->private->wait_q);
813 }
814 
815 /**
816  * qdio_get_ssqd_desc - get qdio subchannel description
817  * @cdev: ccw device to get description for
818  * @data: where to store the ssqd
819  *
820  * Returns 0 or an error code. The results of the chsc are stored in the
821  * specified structure.
822  */
qdio_get_ssqd_desc(struct ccw_device * cdev,struct qdio_ssqd_desc * data)823 int qdio_get_ssqd_desc(struct ccw_device *cdev,
824 		       struct qdio_ssqd_desc *data)
825 {
826 	struct subchannel_id schid;
827 
828 	if (!cdev || !cdev->private)
829 		return -EINVAL;
830 
831 	ccw_device_get_schid(cdev, &schid);
832 	DBF_EVENT("get ssqd:%4x", schid.sch_no);
833 	return qdio_setup_get_ssqd(NULL, &schid, data);
834 }
835 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
836 
qdio_cancel_ccw(struct qdio_irq * irq,int how)837 static int qdio_cancel_ccw(struct qdio_irq *irq, int how)
838 {
839 	struct ccw_device *cdev = irq->cdev;
840 	long timeout;
841 	int rc;
842 
843 	spin_lock_irq(get_ccwdev_lock(cdev));
844 	qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP);
845 	if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
846 		rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
847 	else
848 		/* default behaviour is halt */
849 		rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
850 	spin_unlock_irq(get_ccwdev_lock(cdev));
851 	if (rc) {
852 		DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no);
853 		DBF_ERROR("rc:%4d", rc);
854 		return rc;
855 	}
856 
857 	timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
858 						   irq->state == QDIO_IRQ_STATE_INACTIVE ||
859 						   irq->state == QDIO_IRQ_STATE_ERR,
860 						   10 * HZ);
861 	if (timeout <= 0)
862 		rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
863 
864 	return rc;
865 }
866 
867 /**
868  * qdio_shutdown - shut down a qdio subchannel
869  * @cdev: associated ccw device
870  * @how: use halt or clear to shutdown
871  */
qdio_shutdown(struct ccw_device * cdev,int how)872 int qdio_shutdown(struct ccw_device *cdev, int how)
873 {
874 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
875 	struct subchannel_id schid;
876 	int rc;
877 
878 	if (!irq_ptr)
879 		return -ENODEV;
880 
881 	WARN_ON_ONCE(irqs_disabled());
882 	ccw_device_get_schid(cdev, &schid);
883 	DBF_EVENT("qshutdown:%4x", schid.sch_no);
884 
885 	mutex_lock(&irq_ptr->setup_mutex);
886 	/*
887 	 * Subchannel was already shot down. We cannot prevent being called
888 	 * twice since cio may trigger a shutdown asynchronously.
889 	 */
890 	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
891 		mutex_unlock(&irq_ptr->setup_mutex);
892 		return 0;
893 	}
894 
895 	/*
896 	 * Indicate that the device is going down.
897 	 */
898 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
899 
900 	qdio_shutdown_debug_entries(irq_ptr);
901 
902 	rc = qdio_cancel_ccw(irq_ptr, how);
903 	qdio_shutdown_thinint(irq_ptr);
904 	qdio_shutdown_irq(irq_ptr);
905 
906 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
907 	mutex_unlock(&irq_ptr->setup_mutex);
908 	if (rc)
909 		return rc;
910 	return 0;
911 }
912 EXPORT_SYMBOL_GPL(qdio_shutdown);
913 
914 /**
915  * qdio_free - free data structures for a qdio subchannel
916  * @cdev: associated ccw device
917  */
qdio_free(struct ccw_device * cdev)918 int qdio_free(struct ccw_device *cdev)
919 {
920 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
921 	struct subchannel_id schid;
922 
923 	if (!irq_ptr)
924 		return -ENODEV;
925 
926 	ccw_device_get_schid(cdev, &schid);
927 	DBF_EVENT("qfree:%4x", schid.sch_no);
928 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned");
929 	mutex_lock(&irq_ptr->setup_mutex);
930 
931 	irq_ptr->debug_area = NULL;
932 	cdev->private->qdio_data = NULL;
933 	mutex_unlock(&irq_ptr->setup_mutex);
934 
935 	qdio_free_queues(irq_ptr);
936 	free_page((unsigned long) irq_ptr->qdr);
937 	free_page(irq_ptr->chsc_page);
938 	kfree(irq_ptr->ccw);
939 	free_page((unsigned long) irq_ptr);
940 	return 0;
941 }
942 EXPORT_SYMBOL_GPL(qdio_free);
943 
944 /**
945  * qdio_allocate - allocate qdio queues and associated data
946  * @cdev: associated ccw device
947  * @no_input_qs: allocate this number of Input Queues
948  * @no_output_qs: allocate this number of Output Queues
949  */
qdio_allocate(struct ccw_device * cdev,unsigned int no_input_qs,unsigned int no_output_qs)950 int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs,
951 		  unsigned int no_output_qs)
952 {
953 	struct subchannel_id schid;
954 	struct qdio_irq *irq_ptr;
955 	int rc = -ENOMEM;
956 
957 	ccw_device_get_schid(cdev, &schid);
958 	DBF_EVENT("qallocate:%4x", schid.sch_no);
959 
960 	if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ ||
961 	    no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)
962 		return -EINVAL;
963 
964 	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL);
965 	if (!irq_ptr)
966 		return -ENOMEM;
967 
968 	irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA);
969 	if (!irq_ptr->ccw)
970 		goto err_ccw;
971 
972 	/* kmemleak doesn't scan the page-allocated irq_ptr: */
973 	kmemleak_not_leak(irq_ptr->ccw);
974 
975 	irq_ptr->cdev = cdev;
976 	mutex_init(&irq_ptr->setup_mutex);
977 	if (qdio_allocate_dbf(irq_ptr))
978 		goto err_dbf;
979 
980 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,
981 		      no_output_qs);
982 
983 	/*
984 	 * Allocate a page for the chsc calls in qdio_establish.
985 	 * Must be pre-allocated since a zfcp recovery will call
986 	 * qdio_establish. In case of low memory and swap on a zfcp disk
987 	 * we may not be able to allocate memory otherwise.
988 	 */
989 	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
990 	if (!irq_ptr->chsc_page)
991 		goto err_chsc;
992 
993 	/* qdr is used in ccw1.cda which is u32 */
994 	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
995 	if (!irq_ptr->qdr)
996 		goto err_qdr;
997 
998 	rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);
999 	if (rc)
1000 		goto err_queues;
1001 
1002 	cdev->private->qdio_data = irq_ptr;
1003 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1004 	return 0;
1005 
1006 err_queues:
1007 	free_page((unsigned long) irq_ptr->qdr);
1008 err_qdr:
1009 	free_page(irq_ptr->chsc_page);
1010 err_chsc:
1011 err_dbf:
1012 	kfree(irq_ptr->ccw);
1013 err_ccw:
1014 	free_page((unsigned long) irq_ptr);
1015 	return rc;
1016 }
1017 EXPORT_SYMBOL_GPL(qdio_allocate);
1018 
qdio_trace_init_data(struct qdio_irq * irq,struct qdio_initialize * data)1019 static void qdio_trace_init_data(struct qdio_irq *irq,
1020 				 struct qdio_initialize *data)
1021 {
1022 	DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format);
1023 	DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format);
1024 	DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR);
1025 	DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs,
1026 		      data->no_output_qs);
1027 	DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR);
1028 	DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR);
1029 	DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR);
1030 	DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR);
1031 	DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *),
1032 		    DBF_ERR);
1033 }
1034 
1035 /**
1036  * qdio_establish - establish queues on a qdio subchannel
1037  * @cdev: associated ccw device
1038  * @init_data: initialization data
1039  */
qdio_establish(struct ccw_device * cdev,struct qdio_initialize * init_data)1040 int qdio_establish(struct ccw_device *cdev,
1041 		   struct qdio_initialize *init_data)
1042 {
1043 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1044 	struct subchannel_id schid;
1045 	struct ciw *ciw;
1046 	long timeout;
1047 	int rc;
1048 
1049 	ccw_device_get_schid(cdev, &schid);
1050 	DBF_EVENT("qestablish:%4x", schid.sch_no);
1051 
1052 	if (!irq_ptr)
1053 		return -ENODEV;
1054 
1055 	if (init_data->no_input_qs > irq_ptr->max_input_qs ||
1056 	    init_data->no_output_qs > irq_ptr->max_output_qs)
1057 		return -EINVAL;
1058 
1059 	/* Needed as error_handler: */
1060 	if (!init_data->input_handler)
1061 		return -EINVAL;
1062 
1063 	if (init_data->no_output_qs && !init_data->output_handler)
1064 		return -EINVAL;
1065 
1066 	if (!init_data->input_sbal_addr_array ||
1067 	    !init_data->output_sbal_addr_array)
1068 		return -EINVAL;
1069 
1070 	if (!init_data->irq_poll)
1071 		return -EINVAL;
1072 
1073 	ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE);
1074 	if (!ciw) {
1075 		DBF_ERROR("%4x NO EQ", schid.sch_no);
1076 		return -EIO;
1077 	}
1078 
1079 	mutex_lock(&irq_ptr->setup_mutex);
1080 	qdio_trace_init_data(irq_ptr, init_data);
1081 	qdio_setup_irq(irq_ptr, init_data);
1082 
1083 	rc = qdio_establish_thinint(irq_ptr);
1084 	if (rc)
1085 		goto err_thinint;
1086 
1087 	/* establish q */
1088 	irq_ptr->ccw->cmd_code = ciw->cmd;
1089 	irq_ptr->ccw->flags = CCW_FLAG_SLI;
1090 	irq_ptr->ccw->count = ciw->count;
1091 	irq_ptr->ccw->cda = virt_to_dma32(irq_ptr->qdr);
1092 
1093 	spin_lock_irq(get_ccwdev_lock(cdev));
1094 	ccw_device_set_options_mask(cdev, 0);
1095 
1096 	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1097 	spin_unlock_irq(get_ccwdev_lock(cdev));
1098 	if (rc) {
1099 		DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1100 		DBF_ERROR("rc:%4x", rc);
1101 		goto err_ccw_start;
1102 	}
1103 
1104 	timeout = wait_event_interruptible_timeout(cdev->private->wait_q,
1105 						   irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1106 						   irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1107 	if (timeout <= 0) {
1108 		rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME;
1109 		goto err_ccw_timeout;
1110 	}
1111 
1112 	if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1113 		rc = -EIO;
1114 		goto err_ccw_error;
1115 	}
1116 
1117 	qdio_setup_ssqd_info(irq_ptr);
1118 
1119 	/* qebsm is now setup if available, initialize buffer states */
1120 	qdio_init_buf_states(irq_ptr);
1121 
1122 	mutex_unlock(&irq_ptr->setup_mutex);
1123 	qdio_print_subchannel_info(irq_ptr);
1124 	qdio_setup_debug_entries(irq_ptr);
1125 	return 0;
1126 
1127 err_ccw_timeout:
1128 	qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);
1129 err_ccw_error:
1130 err_ccw_start:
1131 	qdio_shutdown_thinint(irq_ptr);
1132 err_thinint:
1133 	qdio_shutdown_irq(irq_ptr);
1134 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1135 	mutex_unlock(&irq_ptr->setup_mutex);
1136 	return rc;
1137 }
1138 EXPORT_SYMBOL_GPL(qdio_establish);
1139 
1140 /**
1141  * qdio_activate - activate queues on a qdio subchannel
1142  * @cdev: associated cdev
1143  */
qdio_activate(struct ccw_device * cdev)1144 int qdio_activate(struct ccw_device *cdev)
1145 {
1146 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1147 	struct subchannel_id schid;
1148 	struct ciw *ciw;
1149 	int rc;
1150 
1151 	ccw_device_get_schid(cdev, &schid);
1152 	DBF_EVENT("qactivate:%4x", schid.sch_no);
1153 
1154 	if (!irq_ptr)
1155 		return -ENODEV;
1156 
1157 	ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE);
1158 	if (!ciw) {
1159 		DBF_ERROR("%4x NO AQ", schid.sch_no);
1160 		return -EIO;
1161 	}
1162 
1163 	mutex_lock(&irq_ptr->setup_mutex);
1164 	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1165 		rc = -EBUSY;
1166 		goto out;
1167 	}
1168 
1169 	irq_ptr->ccw->cmd_code = ciw->cmd;
1170 	irq_ptr->ccw->flags = CCW_FLAG_SLI;
1171 	irq_ptr->ccw->count = ciw->count;
1172 	irq_ptr->ccw->cda = 0;
1173 
1174 	spin_lock_irq(get_ccwdev_lock(cdev));
1175 	ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1176 
1177 	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1178 			      0, DOIO_DENY_PREFETCH);
1179 	spin_unlock_irq(get_ccwdev_lock(cdev));
1180 	if (rc) {
1181 		DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1182 		DBF_ERROR("rc:%4x", rc);
1183 		goto out;
1184 	}
1185 
1186 	/* wait for subchannel to become active */
1187 	msleep(5);
1188 
1189 	switch (irq_ptr->state) {
1190 	case QDIO_IRQ_STATE_STOPPED:
1191 	case QDIO_IRQ_STATE_ERR:
1192 		rc = -EIO;
1193 		break;
1194 	default:
1195 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1196 		rc = 0;
1197 	}
1198 out:
1199 	mutex_unlock(&irq_ptr->setup_mutex);
1200 	return rc;
1201 }
1202 EXPORT_SYMBOL_GPL(qdio_activate);
1203 
1204 /**
1205  * handle_inbound - reset processed input buffers
1206  * @q: queue containing the buffers
1207  * @bufnr: first buffer to process
1208  * @count: how many buffers are emptied
1209  */
handle_inbound(struct qdio_q * q,int bufnr,int count)1210 static int handle_inbound(struct qdio_q *q, int bufnr, int count)
1211 {
1212 	int overlap;
1213 
1214 	qperf_inc(q, inbound_call);
1215 
1216 	/* If any processed SBALs are returned to HW, adjust our tracking: */
1217 	overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr),
1218 			     q->u.in.batch_count);
1219 	if (overlap > 0) {
1220 		q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap);
1221 		q->u.in.batch_count -= overlap;
1222 	}
1223 
1224 	count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1225 	atomic_add(count, &q->nr_buf_used);
1226 
1227 	if (qdio_need_siga_in(q->irq_ptr))
1228 		return qdio_siga_input(q);
1229 
1230 	return 0;
1231 }
1232 
1233 /**
1234  * qdio_add_bufs_to_input_queue - process buffers on an Input Queue
1235  * @cdev: associated ccw_device for the qdio subchannel
1236  * @q_nr: queue number
1237  * @bufnr: buffer number
1238  * @count: how many buffers to process
1239  */
qdio_add_bufs_to_input_queue(struct ccw_device * cdev,unsigned int q_nr,unsigned int bufnr,unsigned int count)1240 int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr,
1241 				 unsigned int bufnr, unsigned int count)
1242 {
1243 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1244 
1245 	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1246 		return -EINVAL;
1247 
1248 	if (!irq_ptr)
1249 		return -ENODEV;
1250 
1251 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count);
1252 
1253 	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1254 		return -EIO;
1255 	if (!count)
1256 		return 0;
1257 
1258 	return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count);
1259 }
1260 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue);
1261 
1262 /**
1263  * handle_outbound - process filled outbound buffers
1264  * @q: queue containing the buffers
1265  * @bufnr: first buffer to process
1266  * @count: how many buffers are filled
1267  * @aob: asynchronous operation block
1268  */
handle_outbound(struct qdio_q * q,unsigned int bufnr,unsigned int count,struct qaob * aob)1269 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count,
1270 			   struct qaob *aob)
1271 {
1272 	unsigned char state = 0;
1273 	int used, rc = 0;
1274 
1275 	qperf_inc(q, outbound_call);
1276 
1277 	count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1278 	used = atomic_add_return(count, &q->nr_buf_used);
1279 
1280 	if (used == QDIO_MAX_BUFFERS_PER_Q)
1281 		qperf_inc(q, outbound_queue_full);
1282 
1283 	if (queue_type(q) == QDIO_IQDIO_QFMT) {
1284 		dma64_t phys_aob = aob ? virt_to_dma64(aob) : 0;
1285 
1286 		WARN_ON_ONCE(!IS_ALIGNED(dma64_to_u64(phys_aob), 256));
1287 		rc = qdio_kick_outbound_q(q, count, phys_aob);
1288 	} else if (qdio_need_siga_sync(q->irq_ptr)) {
1289 		rc = qdio_sync_output_queue(q);
1290 	} else if (count < QDIO_MAX_BUFFERS_PER_Q &&
1291 		   get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 &&
1292 		   state == SLSB_CU_OUTPUT_PRIMED) {
1293 		/* The previous buffer is not processed yet, tack on. */
1294 		qperf_inc(q, fast_requeue);
1295 	} else {
1296 		rc = qdio_kick_outbound_q(q, count, 0);
1297 	}
1298 
1299 	return rc;
1300 }
1301 
1302 /**
1303  * qdio_add_bufs_to_output_queue - process buffers on an Output Queue
1304  * @cdev: associated ccw_device for the qdio subchannel
1305  * @q_nr: queue number
1306  * @bufnr: buffer number
1307  * @count: how many buffers to process
1308  * @aob: asynchronous operation block
1309  */
qdio_add_bufs_to_output_queue(struct ccw_device * cdev,unsigned int q_nr,unsigned int bufnr,unsigned int count,struct qaob * aob)1310 int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr,
1311 				  unsigned int bufnr, unsigned int count,
1312 				  struct qaob *aob)
1313 {
1314 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1315 
1316 	if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1317 		return -EINVAL;
1318 
1319 	if (!irq_ptr)
1320 		return -ENODEV;
1321 
1322 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count);
1323 
1324 	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1325 		return -EIO;
1326 	if (!count)
1327 		return 0;
1328 
1329 	return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob);
1330 }
1331 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue);
1332 
1333 /**
1334  * qdio_start_irq - enable interrupt processing for the device
1335  * @cdev: associated ccw_device for the qdio subchannel
1336  *
1337  * Return codes
1338  *   0 - success
1339  *   1 - irqs not started since new data is available
1340  */
qdio_start_irq(struct ccw_device * cdev)1341 int qdio_start_irq(struct ccw_device *cdev)
1342 {
1343 	struct qdio_q *q;
1344 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1345 	unsigned int i;
1346 
1347 	if (!irq_ptr)
1348 		return -ENODEV;
1349 
1350 	for_each_input_queue(irq_ptr, q, i)
1351 		qdio_stop_polling(q);
1352 
1353 	clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);
1354 
1355 	/*
1356 	 * We need to check again to not lose initiative after
1357 	 * resetting the ACK state.
1358 	 */
1359 	if (test_nonshared_ind(irq_ptr))
1360 		goto rescan;
1361 
1362 	for_each_input_queue(irq_ptr, q, i) {
1363 		if (!qdio_inbound_q_done(q, q->first_to_check))
1364 			goto rescan;
1365 	}
1366 
1367 	return 0;
1368 
1369 rescan:
1370 	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1371 		return 0;
1372 	else
1373 		return 1;
1374 
1375 }
1376 EXPORT_SYMBOL(qdio_start_irq);
1377 
1378 /**
1379  * qdio_stop_irq - disable interrupt processing for the device
1380  * @cdev: associated ccw_device for the qdio subchannel
1381  *
1382  * Return codes
1383  *   0 - interrupts were already disabled
1384  *   1 - interrupts successfully disabled
1385  */
qdio_stop_irq(struct ccw_device * cdev)1386 int qdio_stop_irq(struct ccw_device *cdev)
1387 {
1388 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1389 
1390 	if (!irq_ptr)
1391 		return -ENODEV;
1392 
1393 	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))
1394 		return 0;
1395 	else
1396 		return 1;
1397 }
1398 EXPORT_SYMBOL(qdio_stop_irq);
1399 
init_QDIO(void)1400 static int __init init_QDIO(void)
1401 {
1402 	int rc;
1403 
1404 	rc = qdio_debug_init();
1405 	if (rc)
1406 		return rc;
1407 	rc = qdio_setup_init();
1408 	if (rc)
1409 		goto out_debug;
1410 	rc = qdio_thinint_init();
1411 	if (rc)
1412 		goto out_cache;
1413 	return 0;
1414 
1415 out_cache:
1416 	qdio_setup_exit();
1417 out_debug:
1418 	qdio_debug_exit();
1419 	return rc;
1420 }
1421 
exit_QDIO(void)1422 static void __exit exit_QDIO(void)
1423 {
1424 	qdio_thinint_exit();
1425 	qdio_setup_exit();
1426 	qdio_debug_exit();
1427 }
1428 
1429 module_init(init_QDIO);
1430 module_exit(exit_QDIO);
1431