1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Linux for s390 qdio support, buffer handling, qdio API and module support. 4 * 5 * Copyright IBM Corp. 2000, 2008 6 * Author(s): Utz Bacher <utz.bacher@de.ibm.com> 7 * Jan Glauber <jang@linux.vnet.ibm.com> 8 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com> 9 */ 10 #include <linux/module.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/kmemleak.h> 14 #include <linux/delay.h> 15 #include <linux/gfp.h> 16 #include <linux/io.h> 17 #include <linux/atomic.h> 18 #include <asm/debug.h> 19 #include <asm/qdio.h> 20 #include <asm/asm.h> 21 #include <asm/ipl.h> 22 23 #include "cio.h" 24 #include "css.h" 25 #include "device.h" 26 #include "qdio.h" 27 #include "qdio_debug.h" 28 29 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\ 30 "Jan Glauber <jang@linux.vnet.ibm.com>"); 31 MODULE_DESCRIPTION("QDIO base support"); 32 MODULE_LICENSE("GPL"); 33 34 static inline int do_siga_sync(unsigned long schid, 35 unsigned long out_mask, unsigned long in_mask, 36 unsigned int fc) 37 { 38 int cc; 39 40 asm volatile( 41 " lgr 0,%[fc]\n" 42 " lgr 1,%[schid]\n" 43 " lgr 2,%[out]\n" 44 " lgr 3,%[in]\n" 45 " siga 0\n" 46 CC_IPM(cc) 47 : CC_OUT(cc, cc) 48 : [fc] "d" (fc), [schid] "d" (schid), 49 [out] "d" (out_mask), [in] "d" (in_mask) 50 : CC_CLOBBER_LIST("0", "1", "2", "3")); 51 return CC_TRANSFORM(cc); 52 } 53 54 static inline int do_siga_input(unsigned long schid, unsigned long mask, 55 unsigned long fc) 56 { 57 int cc; 58 59 asm volatile( 60 " lgr 0,%[fc]\n" 61 " lgr 1,%[schid]\n" 62 " lgr 2,%[mask]\n" 63 " siga 0\n" 64 CC_IPM(cc) 65 : CC_OUT(cc, cc) 66 : [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask) 67 : CC_CLOBBER_LIST("0", "1", "2")); 68 return CC_TRANSFORM(cc); 69 } 70 71 /** 72 * do_siga_output - perform SIGA-w/wt function 73 * @schid: subchannel id or in case of QEBSM the subchannel token 74 * @mask: which output queues to process 75 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer 76 * @fc: function code to perform 77 * @aob: asynchronous operation block 78 * 79 * Returns condition code. 80 * Note: For IQDC unicast queues only the highest priority queue is processed. 81 */ 82 static inline int do_siga_output(unsigned long schid, unsigned long mask, 83 unsigned int *bb, unsigned long fc, 84 dma64_t aob) 85 { 86 int cc; 87 88 asm volatile( 89 " lgr 0,%[fc]\n" 90 " lgr 1,%[schid]\n" 91 " lgr 2,%[mask]\n" 92 " lgr 3,%[aob]\n" 93 " siga 0\n" 94 " lgr %[fc],0\n" 95 CC_IPM(cc) 96 : CC_OUT(cc, cc), [fc] "+&d" (fc) 97 : [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob) 98 : CC_CLOBBER_LIST("0", "1", "2", "3")); 99 *bb = fc >> 31; 100 return CC_TRANSFORM(cc); 101 } 102 103 /** 104 * qdio_do_eqbs - extract buffer states for QEBSM 105 * @q: queue to manipulate 106 * @state: state of the extracted buffers 107 * @start: buffer number to start at 108 * @count: count of buffers to examine 109 * @auto_ack: automatically acknowledge buffers 110 * 111 * Returns the number of successfully extracted equal buffer states. 112 * Stops processing if a state is different from the last buffers state. 113 */ 114 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state, 115 int start, int count, int auto_ack) 116 { 117 int tmp_count = count, tmp_start = start, nr = q->nr; 118 unsigned int ccq = 0; 119 120 qperf_inc(q, eqbs); 121 122 if (!q->is_input_q) 123 nr += q->irq_ptr->nr_input_qs; 124 again: 125 ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count, 126 auto_ack); 127 128 switch (ccq) { 129 case 0: 130 case 32: 131 /* all done, or next buffer state different */ 132 return count - tmp_count; 133 case 96: 134 /* not all buffers processed */ 135 qperf_inc(q, eqbs_partial); 136 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x", 137 tmp_count); 138 return count - tmp_count; 139 case 97: 140 /* no buffer processed */ 141 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq); 142 goto again; 143 default: 144 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 145 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q)); 146 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 147 q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr, 148 q->first_to_check, count, q->irq_ptr->int_parm); 149 return 0; 150 } 151 } 152 153 /** 154 * qdio_do_sqbs - set buffer states for QEBSM 155 * @q: queue to manipulate 156 * @state: new state of the buffers 157 * @start: first buffer number to change 158 * @count: how many buffers to change 159 * 160 * Returns the number of successfully changed buffers. 161 * Does retrying until the specified count of buffer states is set or an 162 * error occurs. 163 */ 164 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start, 165 int count) 166 { 167 unsigned int ccq = 0; 168 int tmp_count = count, tmp_start = start; 169 int nr = q->nr; 170 171 qperf_inc(q, sqbs); 172 173 if (!q->is_input_q) 174 nr += q->irq_ptr->nr_input_qs; 175 again: 176 ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count); 177 178 switch (ccq) { 179 case 0: 180 case 32: 181 /* all done, or active buffer adapter-owned */ 182 WARN_ON_ONCE(tmp_count); 183 return count - tmp_count; 184 case 96: 185 /* not all buffers processed */ 186 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq); 187 qperf_inc(q, sqbs_partial); 188 goto again; 189 default: 190 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 191 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q)); 192 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 193 q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr, 194 q->first_to_check, count, q->irq_ptr->int_parm); 195 return 0; 196 } 197 } 198 199 /* 200 * Returns number of examined buffers and their common state in *state. 201 * Requested number of buffers-to-examine must be > 0. 202 */ 203 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, 204 unsigned char *state, unsigned int count, 205 int auto_ack) 206 { 207 unsigned char __state = 0; 208 int i = 1; 209 210 if (is_qebsm(q)) 211 return qdio_do_eqbs(q, state, bufnr, count, auto_ack); 212 213 /* get initial state: */ 214 __state = q->slsb.val[bufnr]; 215 216 /* Bail out early if there is no work on the queue: */ 217 if (__state & SLSB_OWNER_CU) 218 goto out; 219 220 for (; i < count; i++) { 221 bufnr = next_buf(bufnr); 222 223 /* stop if next state differs from initial state: */ 224 if (q->slsb.val[bufnr] != __state) 225 break; 226 } 227 228 out: 229 *state = __state; 230 return i; 231 } 232 233 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr, 234 unsigned char *state, int auto_ack) 235 { 236 return get_buf_states(q, bufnr, state, 1, auto_ack); 237 } 238 239 /* wrap-around safe setting of slsb states, returns number of changed buffers */ 240 static inline int set_buf_states(struct qdio_q *q, int bufnr, 241 unsigned char state, int count) 242 { 243 int i; 244 245 if (is_qebsm(q)) 246 return qdio_do_sqbs(q, state, bufnr, count); 247 248 /* Ensure that all preceding changes to the SBALs are visible: */ 249 mb(); 250 251 for (i = 0; i < count; i++) { 252 WRITE_ONCE(q->slsb.val[bufnr], state); 253 bufnr = next_buf(bufnr); 254 } 255 256 /* Make our SLSB changes visible: */ 257 mb(); 258 259 return count; 260 } 261 262 static inline int set_buf_state(struct qdio_q *q, int bufnr, 263 unsigned char state) 264 { 265 return set_buf_states(q, bufnr, state, 1); 266 } 267 268 /* set slsb states to initial state */ 269 static void qdio_init_buf_states(struct qdio_irq *irq_ptr) 270 { 271 struct qdio_q *q; 272 int i; 273 274 for_each_input_queue(irq_ptr, q, i) 275 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT, 276 QDIO_MAX_BUFFERS_PER_Q); 277 for_each_output_queue(irq_ptr, q, i) 278 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT, 279 QDIO_MAX_BUFFERS_PER_Q); 280 } 281 282 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output, 283 unsigned int input) 284 { 285 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 286 unsigned int fc = QDIO_SIGA_SYNC; 287 int cc; 288 289 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr); 290 qperf_inc(q, siga_sync); 291 292 if (is_qebsm(q)) { 293 schid = q->irq_ptr->sch_token; 294 fc |= QDIO_SIGA_QEBSM_FLAG; 295 } 296 297 cc = do_siga_sync(schid, output, input, fc); 298 if (unlikely(cc)) 299 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc); 300 return (cc) ? -EIO : 0; 301 } 302 303 static inline int qdio_sync_input_queue(struct qdio_q *q) 304 { 305 return qdio_siga_sync(q, 0, q->mask); 306 } 307 308 static inline int qdio_sync_output_queue(struct qdio_q *q) 309 { 310 return qdio_siga_sync(q, q->mask, 0); 311 } 312 313 static inline int qdio_siga_sync_q(struct qdio_q *q) 314 { 315 if (q->is_input_q) 316 return qdio_sync_input_queue(q); 317 else 318 return qdio_sync_output_queue(q); 319 } 320 321 static int qdio_siga_output(struct qdio_q *q, unsigned int count, 322 unsigned int *busy_bit, dma64_t aob) 323 { 324 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 325 unsigned int fc = QDIO_SIGA_WRITE; 326 u64 start_time = 0; 327 int retries = 0, cc; 328 329 if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) { 330 if (count > 1) 331 fc = QDIO_SIGA_WRITEM; 332 else if (aob) 333 fc = QDIO_SIGA_WRITEQ; 334 } 335 336 if (is_qebsm(q)) { 337 schid = q->irq_ptr->sch_token; 338 fc |= QDIO_SIGA_QEBSM_FLAG; 339 } 340 again: 341 cc = do_siga_output(schid, q->mask, busy_bit, fc, aob); 342 343 /* hipersocket busy condition */ 344 if (unlikely(*busy_bit)) { 345 retries++; 346 347 if (!start_time) { 348 start_time = get_tod_clock_fast(); 349 goto again; 350 } 351 if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE) 352 goto again; 353 } 354 if (retries) { 355 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, 356 "%4x cc2 BB1:%1d", SCH_NO(q), q->nr); 357 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries); 358 } 359 return cc; 360 } 361 362 static inline int qdio_siga_input(struct qdio_q *q) 363 { 364 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 365 unsigned int fc = QDIO_SIGA_READ; 366 int cc; 367 368 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr); 369 qperf_inc(q, siga_read); 370 371 if (is_qebsm(q)) { 372 schid = q->irq_ptr->sch_token; 373 fc |= QDIO_SIGA_QEBSM_FLAG; 374 } 375 376 cc = do_siga_input(schid, q->mask, fc); 377 if (unlikely(cc)) 378 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc); 379 return (cc) ? -EIO : 0; 380 } 381 382 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, 383 unsigned char *state) 384 { 385 if (qdio_need_siga_sync(q->irq_ptr)) 386 qdio_siga_sync_q(q); 387 return get_buf_state(q, bufnr, state, 0); 388 } 389 390 static inline void qdio_stop_polling(struct qdio_q *q) 391 { 392 if (!q->u.in.batch_count) 393 return; 394 395 qperf_inc(q, stop_polling); 396 397 /* show the card that we are not polling anymore */ 398 set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT, 399 q->u.in.batch_count); 400 q->u.in.batch_count = 0; 401 } 402 403 static inline void account_sbals(struct qdio_q *q, unsigned int count) 404 { 405 q->q_stats.nr_sbal_total += count; 406 q->q_stats.nr_sbals[ilog2(count)]++; 407 } 408 409 static void process_buffer_error(struct qdio_q *q, unsigned int start, 410 int count) 411 { 412 /* special handling for no target buffer empty */ 413 if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q && 414 q->sbal[start]->element[15].sflags == 0x10) { 415 qperf_inc(q, target_full); 416 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start); 417 return; 418 } 419 420 DBF_ERROR("%4x BUF ERROR", SCH_NO(q)); 421 DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr); 422 DBF_ERROR("FTC:%3d C:%3d", start, count); 423 DBF_ERROR("F14:%2x F15:%2x", 424 q->sbal[start]->element[14].sflags, 425 q->sbal[start]->element[15].sflags); 426 } 427 428 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start, 429 int count, bool auto_ack) 430 { 431 /* ACK the newest SBAL: */ 432 if (!auto_ack) 433 set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK); 434 435 if (!q->u.in.batch_count) 436 q->u.in.batch_start = start; 437 q->u.in.batch_count += count; 438 } 439 440 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start, 441 unsigned int *error) 442 { 443 unsigned char state = 0; 444 int count; 445 446 q->timestamp = get_tod_clock_fast(); 447 448 count = atomic_read(&q->nr_buf_used); 449 if (!count) 450 return 0; 451 452 if (qdio_need_siga_sync(q->irq_ptr)) 453 qdio_sync_input_queue(q); 454 455 count = get_buf_states(q, start, &state, count, 1); 456 if (!count) 457 return 0; 458 459 switch (state) { 460 case SLSB_P_INPUT_PRIMED: 461 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, 462 count); 463 464 inbound_handle_work(q, start, count, is_qebsm(q)); 465 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 466 qperf_inc(q, inbound_queue_full); 467 if (q->irq_ptr->perf_stat_enabled) 468 account_sbals(q, count); 469 return count; 470 case SLSB_P_INPUT_ERROR: 471 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr, 472 count); 473 474 *error = QDIO_ERROR_SLSB_STATE; 475 process_buffer_error(q, start, count); 476 inbound_handle_work(q, start, count, false); 477 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 478 qperf_inc(q, inbound_queue_full); 479 if (q->irq_ptr->perf_stat_enabled) 480 account_sbals_error(q, count); 481 return count; 482 case SLSB_CU_INPUT_EMPTY: 483 if (q->irq_ptr->perf_stat_enabled) 484 q->q_stats.nr_sbal_nop++; 485 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x", 486 q->nr, start); 487 return 0; 488 case SLSB_P_INPUT_NOT_INIT: 489 case SLSB_P_INPUT_ACK: 490 /* We should never see this state, throw a WARN: */ 491 default: 492 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 493 "found state %#x at index %u on queue %u\n", 494 state, start, q->nr); 495 return 0; 496 } 497 } 498 499 int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr, 500 unsigned int *bufnr, unsigned int *error) 501 { 502 struct qdio_irq *irq = cdev->private->qdio_data; 503 unsigned int start; 504 struct qdio_q *q; 505 int count; 506 507 if (!irq) 508 return -ENODEV; 509 510 q = irq->input_qs[nr]; 511 start = q->first_to_check; 512 *error = 0; 513 514 count = get_inbound_buffer_frontier(q, start, error); 515 if (count == 0) 516 return 0; 517 518 *bufnr = start; 519 q->first_to_check = add_buf(start, count); 520 return count; 521 } 522 EXPORT_SYMBOL_GPL(qdio_inspect_input_queue); 523 524 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) 525 { 526 unsigned char state = 0; 527 528 if (!atomic_read(&q->nr_buf_used)) 529 return 1; 530 531 if (qdio_need_siga_sync(q->irq_ptr)) 532 qdio_sync_input_queue(q); 533 get_buf_state(q, start, &state, 0); 534 535 if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) 536 /* more work coming */ 537 return 0; 538 539 return 1; 540 } 541 542 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start, 543 unsigned int *error) 544 { 545 unsigned char state = 0; 546 int count; 547 548 q->timestamp = get_tod_clock_fast(); 549 550 count = atomic_read(&q->nr_buf_used); 551 if (!count) 552 return 0; 553 554 if (qdio_need_siga_sync(q->irq_ptr)) 555 qdio_sync_output_queue(q); 556 557 count = get_buf_states(q, start, &state, count, 0); 558 if (!count) 559 return 0; 560 561 switch (state) { 562 case SLSB_P_OUTPUT_PENDING: 563 *error = QDIO_ERROR_SLSB_PENDING; 564 fallthrough; 565 case SLSB_P_OUTPUT_EMPTY: 566 /* the adapter got it */ 567 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, 568 "out empty:%1d %02x", q->nr, count); 569 570 atomic_sub(count, &q->nr_buf_used); 571 if (q->irq_ptr->perf_stat_enabled) 572 account_sbals(q, count); 573 return count; 574 case SLSB_P_OUTPUT_ERROR: 575 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x", 576 q->nr, count); 577 578 *error = QDIO_ERROR_SLSB_STATE; 579 process_buffer_error(q, start, count); 580 atomic_sub(count, &q->nr_buf_used); 581 if (q->irq_ptr->perf_stat_enabled) 582 account_sbals_error(q, count); 583 return count; 584 case SLSB_CU_OUTPUT_PRIMED: 585 /* the adapter has not fetched the output yet */ 586 if (q->irq_ptr->perf_stat_enabled) 587 q->q_stats.nr_sbal_nop++; 588 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", 589 q->nr); 590 return 0; 591 case SLSB_P_OUTPUT_HALTED: 592 return 0; 593 case SLSB_P_OUTPUT_NOT_INIT: 594 /* We should never see this state, throw a WARN: */ 595 default: 596 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 597 "found state %#x at index %u on queue %u\n", 598 state, start, q->nr); 599 return 0; 600 } 601 } 602 603 int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr, 604 unsigned int *bufnr, unsigned int *error) 605 { 606 struct qdio_irq *irq = cdev->private->qdio_data; 607 unsigned int start; 608 struct qdio_q *q; 609 int count; 610 611 if (!irq) 612 return -ENODEV; 613 614 q = irq->output_qs[nr]; 615 start = q->first_to_check; 616 *error = 0; 617 618 count = get_outbound_buffer_frontier(q, start, error); 619 if (count == 0) 620 return 0; 621 622 *bufnr = start; 623 q->first_to_check = add_buf(start, count); 624 return count; 625 } 626 EXPORT_SYMBOL_GPL(qdio_inspect_output_queue); 627 628 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count, 629 dma64_t aob) 630 { 631 int retries = 0, cc; 632 unsigned int busy_bit; 633 634 if (!qdio_need_siga_out(q->irq_ptr)) 635 return 0; 636 637 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr); 638 retry: 639 qperf_inc(q, siga_write); 640 641 cc = qdio_siga_output(q, count, &busy_bit, aob); 642 switch (cc) { 643 case 0: 644 break; 645 case 2: 646 if (busy_bit) { 647 while (++retries < QDIO_BUSY_BIT_RETRIES) { 648 mdelay(QDIO_BUSY_BIT_RETRY_DELAY); 649 goto retry; 650 } 651 DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr); 652 cc = -EBUSY; 653 } else { 654 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr); 655 cc = -ENOBUFS; 656 } 657 break; 658 case 1: 659 case 3: 660 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc); 661 cc = -EIO; 662 break; 663 } 664 if (retries) { 665 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr); 666 DBF_ERROR("count:%u", retries); 667 } 668 return cc; 669 } 670 671 static inline void qdio_set_state(struct qdio_irq *irq_ptr, 672 enum qdio_irq_states state) 673 { 674 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state); 675 676 irq_ptr->state = state; 677 mb(); 678 } 679 680 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb) 681 { 682 if (irb->esw.esw0.erw.cons) { 683 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no); 684 DBF_ERROR_HEX(irb, 64); 685 DBF_ERROR_HEX(irb->ecw, 64); 686 } 687 } 688 689 /* PCI interrupt handler */ 690 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) 691 { 692 if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 693 return; 694 695 qdio_deliver_irq(irq_ptr); 696 irq_ptr->last_data_irq_time = get_lowcore()->int_clock; 697 } 698 699 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr, 700 unsigned long intparm, int cstat, 701 int dstat) 702 { 703 unsigned int first_to_check = 0; 704 705 DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no); 706 DBF_ERROR("intp :%lx", intparm); 707 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 708 709 /* zfcp wants this: */ 710 if (irq_ptr->nr_input_qs) 711 first_to_check = irq_ptr->input_qs[0]->first_to_check; 712 713 irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0, 714 first_to_check, 0, irq_ptr->int_parm); 715 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 716 /* 717 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen. 718 * Therefore we call the LGR detection function here. 719 */ 720 lgr_info_log(); 721 } 722 723 static int qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat, 724 int dstat, int dcc) 725 { 726 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq"); 727 728 if (cstat) 729 goto error; 730 if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END)) 731 goto error; 732 if (dcc == 1) 733 return -EAGAIN; 734 if (!(dstat & DEV_STAT_DEV_END)) 735 goto error; 736 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED); 737 return 0; 738 739 error: 740 DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no); 741 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 742 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 743 return -EIO; 744 } 745 746 /* qdio interrupt handler */ 747 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, 748 struct irb *irb) 749 { 750 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 751 struct subchannel_id schid; 752 int cstat, dstat, rc, dcc; 753 754 if (!intparm || !irq_ptr) { 755 ccw_device_get_schid(cdev, &schid); 756 DBF_ERROR("qint:%4x", schid.sch_no); 757 return; 758 } 759 760 if (irq_ptr->perf_stat_enabled) 761 irq_ptr->perf_stat.qdio_int++; 762 763 if (IS_ERR(irb)) { 764 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no); 765 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 766 wake_up(&cdev->private->wait_q); 767 return; 768 } 769 qdio_irq_check_sense(irq_ptr, irb); 770 cstat = irb->scsw.cmd.cstat; 771 dstat = irb->scsw.cmd.dstat; 772 dcc = scsw_cmd_is_valid_cc(&irb->scsw) ? irb->scsw.cmd.cc : 0; 773 rc = 0; 774 775 switch (irq_ptr->state) { 776 case QDIO_IRQ_STATE_INACTIVE: 777 rc = qdio_establish_handle_irq(irq_ptr, cstat, dstat, dcc); 778 break; 779 case QDIO_IRQ_STATE_CLEANUP: 780 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 781 break; 782 case QDIO_IRQ_STATE_ESTABLISHED: 783 case QDIO_IRQ_STATE_ACTIVE: 784 if (cstat & SCHN_STAT_PCI) { 785 qdio_int_handler_pci(irq_ptr); 786 return; 787 } 788 if (cstat || dstat) 789 qdio_handle_activate_check(irq_ptr, intparm, cstat, 790 dstat); 791 else if (dcc == 1) 792 rc = -EAGAIN; 793 break; 794 case QDIO_IRQ_STATE_STOPPED: 795 break; 796 default: 797 WARN_ON_ONCE(1); 798 } 799 800 if (rc == -EAGAIN) { 801 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qint retry"); 802 rc = ccw_device_start(cdev, irq_ptr->ccw, intparm, 0, 0); 803 if (!rc) 804 return; 805 DBF_ERROR("%4x RETRY ERR", irq_ptr->schid.sch_no); 806 DBF_ERROR("rc:%4x", rc); 807 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 808 } 809 810 wake_up(&cdev->private->wait_q); 811 } 812 813 /** 814 * qdio_get_ssqd_desc - get qdio subchannel description 815 * @cdev: ccw device to get description for 816 * @data: where to store the ssqd 817 * 818 * Returns 0 or an error code. The results of the chsc are stored in the 819 * specified structure. 820 */ 821 int qdio_get_ssqd_desc(struct ccw_device *cdev, 822 struct qdio_ssqd_desc *data) 823 { 824 struct subchannel_id schid; 825 826 if (!cdev || !cdev->private) 827 return -EINVAL; 828 829 ccw_device_get_schid(cdev, &schid); 830 DBF_EVENT("get ssqd:%4x", schid.sch_no); 831 return qdio_setup_get_ssqd(NULL, &schid, data); 832 } 833 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc); 834 835 static int qdio_cancel_ccw(struct qdio_irq *irq, int how) 836 { 837 struct ccw_device *cdev = irq->cdev; 838 long timeout; 839 int rc; 840 841 spin_lock_irq(get_ccwdev_lock(cdev)); 842 qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP); 843 if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) 844 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); 845 else 846 /* default behaviour is halt */ 847 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); 848 spin_unlock_irq(get_ccwdev_lock(cdev)); 849 if (rc) { 850 DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no); 851 DBF_ERROR("rc:%4d", rc); 852 return rc; 853 } 854 855 timeout = wait_event_interruptible_timeout(cdev->private->wait_q, 856 irq->state == QDIO_IRQ_STATE_INACTIVE || 857 irq->state == QDIO_IRQ_STATE_ERR, 858 10 * HZ); 859 if (timeout <= 0) 860 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; 861 862 return rc; 863 } 864 865 /** 866 * qdio_shutdown - shut down a qdio subchannel 867 * @cdev: associated ccw device 868 * @how: use halt or clear to shutdown 869 */ 870 int qdio_shutdown(struct ccw_device *cdev, int how) 871 { 872 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 873 struct subchannel_id schid; 874 int rc; 875 876 if (!irq_ptr) 877 return -ENODEV; 878 879 WARN_ON_ONCE(irqs_disabled()); 880 ccw_device_get_schid(cdev, &schid); 881 DBF_EVENT("qshutdown:%4x", schid.sch_no); 882 883 mutex_lock(&irq_ptr->setup_mutex); 884 /* 885 * Subchannel was already shot down. We cannot prevent being called 886 * twice since cio may trigger a shutdown asynchronously. 887 */ 888 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 889 mutex_unlock(&irq_ptr->setup_mutex); 890 return 0; 891 } 892 893 /* 894 * Indicate that the device is going down. 895 */ 896 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 897 898 qdio_shutdown_debug_entries(irq_ptr); 899 900 rc = qdio_cancel_ccw(irq_ptr, how); 901 qdio_shutdown_thinint(irq_ptr); 902 qdio_shutdown_irq(irq_ptr); 903 904 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 905 mutex_unlock(&irq_ptr->setup_mutex); 906 if (rc) 907 return rc; 908 return 0; 909 } 910 EXPORT_SYMBOL_GPL(qdio_shutdown); 911 912 /** 913 * qdio_free - free data structures for a qdio subchannel 914 * @cdev: associated ccw device 915 */ 916 int qdio_free(struct ccw_device *cdev) 917 { 918 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 919 struct subchannel_id schid; 920 921 if (!irq_ptr) 922 return -ENODEV; 923 924 ccw_device_get_schid(cdev, &schid); 925 DBF_EVENT("qfree:%4x", schid.sch_no); 926 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned"); 927 mutex_lock(&irq_ptr->setup_mutex); 928 929 irq_ptr->debug_area = NULL; 930 cdev->private->qdio_data = NULL; 931 mutex_unlock(&irq_ptr->setup_mutex); 932 933 qdio_free_queues(irq_ptr); 934 free_page((unsigned long) irq_ptr->qdr); 935 free_page(irq_ptr->chsc_page); 936 kfree(irq_ptr->ccw); 937 free_page((unsigned long) irq_ptr); 938 return 0; 939 } 940 EXPORT_SYMBOL_GPL(qdio_free); 941 942 /** 943 * qdio_allocate - allocate qdio queues and associated data 944 * @cdev: associated ccw device 945 * @no_input_qs: allocate this number of Input Queues 946 * @no_output_qs: allocate this number of Output Queues 947 */ 948 int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, 949 unsigned int no_output_qs) 950 { 951 struct subchannel_id schid; 952 struct qdio_irq *irq_ptr; 953 int rc = -ENOMEM; 954 955 ccw_device_get_schid(cdev, &schid); 956 DBF_EVENT("qallocate:%4x", schid.sch_no); 957 958 if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ || 959 no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) 960 return -EINVAL; 961 962 irq_ptr = (void *) get_zeroed_page(GFP_KERNEL); 963 if (!irq_ptr) 964 return -ENOMEM; 965 966 irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA); 967 if (!irq_ptr->ccw) 968 goto err_ccw; 969 970 /* kmemleak doesn't scan the page-allocated irq_ptr: */ 971 kmemleak_not_leak(irq_ptr->ccw); 972 973 irq_ptr->cdev = cdev; 974 mutex_init(&irq_ptr->setup_mutex); 975 if (qdio_allocate_dbf(irq_ptr)) 976 goto err_dbf; 977 978 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs, 979 no_output_qs); 980 981 /* 982 * Allocate a page for the chsc calls in qdio_establish. 983 * Must be pre-allocated since a zfcp recovery will call 984 * qdio_establish. In case of low memory and swap on a zfcp disk 985 * we may not be able to allocate memory otherwise. 986 */ 987 irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); 988 if (!irq_ptr->chsc_page) 989 goto err_chsc; 990 991 /* qdr is used in ccw1.cda which is u32 */ 992 irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 993 if (!irq_ptr->qdr) 994 goto err_qdr; 995 996 rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs); 997 if (rc) 998 goto err_queues; 999 1000 cdev->private->qdio_data = irq_ptr; 1001 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1002 return 0; 1003 1004 err_queues: 1005 free_page((unsigned long) irq_ptr->qdr); 1006 err_qdr: 1007 free_page(irq_ptr->chsc_page); 1008 err_chsc: 1009 err_dbf: 1010 kfree(irq_ptr->ccw); 1011 err_ccw: 1012 free_page((unsigned long) irq_ptr); 1013 return rc; 1014 } 1015 EXPORT_SYMBOL_GPL(qdio_allocate); 1016 1017 static void qdio_trace_init_data(struct qdio_irq *irq, 1018 struct qdio_initialize *data) 1019 { 1020 DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format); 1021 DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format); 1022 DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR); 1023 DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs, 1024 data->no_output_qs); 1025 DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR); 1026 DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR); 1027 DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR); 1028 DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR); 1029 DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *), 1030 DBF_ERR); 1031 } 1032 1033 /** 1034 * qdio_establish - establish queues on a qdio subchannel 1035 * @cdev: associated ccw device 1036 * @init_data: initialization data 1037 */ 1038 int qdio_establish(struct ccw_device *cdev, 1039 struct qdio_initialize *init_data) 1040 { 1041 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1042 struct subchannel_id schid; 1043 struct ciw *ciw; 1044 long timeout; 1045 int rc; 1046 1047 ccw_device_get_schid(cdev, &schid); 1048 DBF_EVENT("qestablish:%4x", schid.sch_no); 1049 1050 if (!irq_ptr) 1051 return -ENODEV; 1052 1053 if (init_data->no_input_qs > irq_ptr->max_input_qs || 1054 init_data->no_output_qs > irq_ptr->max_output_qs) 1055 return -EINVAL; 1056 1057 /* Needed as error_handler: */ 1058 if (!init_data->input_handler) 1059 return -EINVAL; 1060 1061 if (init_data->no_output_qs && !init_data->output_handler) 1062 return -EINVAL; 1063 1064 if (!init_data->input_sbal_addr_array || 1065 !init_data->output_sbal_addr_array) 1066 return -EINVAL; 1067 1068 if (!init_data->irq_poll) 1069 return -EINVAL; 1070 1071 ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE); 1072 if (!ciw) { 1073 DBF_ERROR("%4x NO EQ", schid.sch_no); 1074 return -EIO; 1075 } 1076 1077 mutex_lock(&irq_ptr->setup_mutex); 1078 qdio_trace_init_data(irq_ptr, init_data); 1079 qdio_setup_irq(irq_ptr, init_data); 1080 1081 rc = qdio_establish_thinint(irq_ptr); 1082 if (rc) 1083 goto err_thinint; 1084 1085 /* establish q */ 1086 irq_ptr->ccw->cmd_code = ciw->cmd; 1087 irq_ptr->ccw->flags = CCW_FLAG_SLI; 1088 irq_ptr->ccw->count = ciw->count; 1089 irq_ptr->ccw->cda = virt_to_dma32(irq_ptr->qdr); 1090 1091 spin_lock_irq(get_ccwdev_lock(cdev)); 1092 ccw_device_set_options_mask(cdev, 0); 1093 1094 rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); 1095 spin_unlock_irq(get_ccwdev_lock(cdev)); 1096 if (rc) { 1097 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no); 1098 DBF_ERROR("rc:%4x", rc); 1099 goto err_ccw_start; 1100 } 1101 1102 timeout = wait_event_interruptible_timeout(cdev->private->wait_q, 1103 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || 1104 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); 1105 if (timeout <= 0) { 1106 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; 1107 goto err_ccw_timeout; 1108 } 1109 1110 if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) { 1111 rc = -EIO; 1112 goto err_ccw_error; 1113 } 1114 1115 qdio_setup_ssqd_info(irq_ptr); 1116 1117 /* qebsm is now setup if available, initialize buffer states */ 1118 qdio_init_buf_states(irq_ptr); 1119 1120 mutex_unlock(&irq_ptr->setup_mutex); 1121 qdio_print_subchannel_info(irq_ptr); 1122 qdio_setup_debug_entries(irq_ptr); 1123 return 0; 1124 1125 err_ccw_timeout: 1126 qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR); 1127 err_ccw_error: 1128 err_ccw_start: 1129 qdio_shutdown_thinint(irq_ptr); 1130 err_thinint: 1131 qdio_shutdown_irq(irq_ptr); 1132 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1133 mutex_unlock(&irq_ptr->setup_mutex); 1134 return rc; 1135 } 1136 EXPORT_SYMBOL_GPL(qdio_establish); 1137 1138 /** 1139 * qdio_activate - activate queues on a qdio subchannel 1140 * @cdev: associated cdev 1141 */ 1142 int qdio_activate(struct ccw_device *cdev) 1143 { 1144 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1145 struct subchannel_id schid; 1146 struct ciw *ciw; 1147 int rc; 1148 1149 ccw_device_get_schid(cdev, &schid); 1150 DBF_EVENT("qactivate:%4x", schid.sch_no); 1151 1152 if (!irq_ptr) 1153 return -ENODEV; 1154 1155 ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE); 1156 if (!ciw) { 1157 DBF_ERROR("%4x NO AQ", schid.sch_no); 1158 return -EIO; 1159 } 1160 1161 mutex_lock(&irq_ptr->setup_mutex); 1162 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1163 rc = -EBUSY; 1164 goto out; 1165 } 1166 1167 irq_ptr->ccw->cmd_code = ciw->cmd; 1168 irq_ptr->ccw->flags = CCW_FLAG_SLI; 1169 irq_ptr->ccw->count = ciw->count; 1170 irq_ptr->ccw->cda = 0; 1171 1172 spin_lock_irq(get_ccwdev_lock(cdev)); 1173 ccw_device_set_options(cdev, CCWDEV_REPORT_ALL); 1174 1175 rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE, 1176 0, DOIO_DENY_PREFETCH); 1177 spin_unlock_irq(get_ccwdev_lock(cdev)); 1178 if (rc) { 1179 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no); 1180 DBF_ERROR("rc:%4x", rc); 1181 goto out; 1182 } 1183 1184 /* wait for subchannel to become active */ 1185 msleep(5); 1186 1187 switch (irq_ptr->state) { 1188 case QDIO_IRQ_STATE_STOPPED: 1189 case QDIO_IRQ_STATE_ERR: 1190 rc = -EIO; 1191 break; 1192 default: 1193 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE); 1194 rc = 0; 1195 } 1196 out: 1197 mutex_unlock(&irq_ptr->setup_mutex); 1198 return rc; 1199 } 1200 EXPORT_SYMBOL_GPL(qdio_activate); 1201 1202 /** 1203 * handle_inbound - reset processed input buffers 1204 * @q: queue containing the buffers 1205 * @bufnr: first buffer to process 1206 * @count: how many buffers are emptied 1207 */ 1208 static int handle_inbound(struct qdio_q *q, int bufnr, int count) 1209 { 1210 int overlap; 1211 1212 qperf_inc(q, inbound_call); 1213 1214 /* If any processed SBALs are returned to HW, adjust our tracking: */ 1215 overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr), 1216 q->u.in.batch_count); 1217 if (overlap > 0) { 1218 q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap); 1219 q->u.in.batch_count -= overlap; 1220 } 1221 1222 count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count); 1223 atomic_add(count, &q->nr_buf_used); 1224 1225 if (qdio_need_siga_in(q->irq_ptr)) 1226 return qdio_siga_input(q); 1227 1228 return 0; 1229 } 1230 1231 /** 1232 * qdio_add_bufs_to_input_queue - process buffers on an Input Queue 1233 * @cdev: associated ccw_device for the qdio subchannel 1234 * @q_nr: queue number 1235 * @bufnr: buffer number 1236 * @count: how many buffers to process 1237 */ 1238 int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr, 1239 unsigned int bufnr, unsigned int count) 1240 { 1241 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1242 1243 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1244 return -EINVAL; 1245 1246 if (!irq_ptr) 1247 return -ENODEV; 1248 1249 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count); 1250 1251 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1252 return -EIO; 1253 if (!count) 1254 return 0; 1255 1256 return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count); 1257 } 1258 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue); 1259 1260 /** 1261 * handle_outbound - process filled outbound buffers 1262 * @q: queue containing the buffers 1263 * @bufnr: first buffer to process 1264 * @count: how many buffers are filled 1265 * @aob: asynchronous operation block 1266 */ 1267 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count, 1268 struct qaob *aob) 1269 { 1270 unsigned char state = 0; 1271 int used, rc = 0; 1272 1273 qperf_inc(q, outbound_call); 1274 1275 count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count); 1276 used = atomic_add_return(count, &q->nr_buf_used); 1277 1278 if (used == QDIO_MAX_BUFFERS_PER_Q) 1279 qperf_inc(q, outbound_queue_full); 1280 1281 if (queue_type(q) == QDIO_IQDIO_QFMT) { 1282 dma64_t phys_aob = aob ? virt_to_dma64(aob) : 0; 1283 1284 WARN_ON_ONCE(!IS_ALIGNED(dma64_to_u64(phys_aob), 256)); 1285 rc = qdio_kick_outbound_q(q, count, phys_aob); 1286 } else if (qdio_need_siga_sync(q->irq_ptr)) { 1287 rc = qdio_sync_output_queue(q); 1288 } else if (count < QDIO_MAX_BUFFERS_PER_Q && 1289 get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 && 1290 state == SLSB_CU_OUTPUT_PRIMED) { 1291 /* The previous buffer is not processed yet, tack on. */ 1292 qperf_inc(q, fast_requeue); 1293 } else { 1294 rc = qdio_kick_outbound_q(q, count, 0); 1295 } 1296 1297 return rc; 1298 } 1299 1300 /** 1301 * qdio_add_bufs_to_output_queue - process buffers on an Output Queue 1302 * @cdev: associated ccw_device for the qdio subchannel 1303 * @q_nr: queue number 1304 * @bufnr: buffer number 1305 * @count: how many buffers to process 1306 * @aob: asynchronous operation block 1307 */ 1308 int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr, 1309 unsigned int bufnr, unsigned int count, 1310 struct qaob *aob) 1311 { 1312 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1313 1314 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1315 return -EINVAL; 1316 1317 if (!irq_ptr) 1318 return -ENODEV; 1319 1320 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count); 1321 1322 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1323 return -EIO; 1324 if (!count) 1325 return 0; 1326 1327 return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob); 1328 } 1329 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue); 1330 1331 /** 1332 * qdio_start_irq - enable interrupt processing for the device 1333 * @cdev: associated ccw_device for the qdio subchannel 1334 * 1335 * Return codes 1336 * 0 - success 1337 * 1 - irqs not started since new data is available 1338 */ 1339 int qdio_start_irq(struct ccw_device *cdev) 1340 { 1341 struct qdio_q *q; 1342 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1343 unsigned int i; 1344 1345 if (!irq_ptr) 1346 return -ENODEV; 1347 1348 for_each_input_queue(irq_ptr, q, i) 1349 qdio_stop_polling(q); 1350 1351 clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state); 1352 1353 /* 1354 * We need to check again to not lose initiative after 1355 * resetting the ACK state. 1356 */ 1357 if (test_nonshared_ind(irq_ptr)) 1358 goto rescan; 1359 1360 for_each_input_queue(irq_ptr, q, i) { 1361 if (!qdio_inbound_q_done(q, q->first_to_check)) 1362 goto rescan; 1363 } 1364 1365 return 0; 1366 1367 rescan: 1368 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1369 return 0; 1370 else 1371 return 1; 1372 1373 } 1374 EXPORT_SYMBOL(qdio_start_irq); 1375 1376 /** 1377 * qdio_stop_irq - disable interrupt processing for the device 1378 * @cdev: associated ccw_device for the qdio subchannel 1379 * 1380 * Return codes 1381 * 0 - interrupts were already disabled 1382 * 1 - interrupts successfully disabled 1383 */ 1384 int qdio_stop_irq(struct ccw_device *cdev) 1385 { 1386 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1387 1388 if (!irq_ptr) 1389 return -ENODEV; 1390 1391 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1392 return 0; 1393 else 1394 return 1; 1395 } 1396 EXPORT_SYMBOL(qdio_stop_irq); 1397 1398 static int __init init_QDIO(void) 1399 { 1400 int rc; 1401 1402 rc = qdio_debug_init(); 1403 if (rc) 1404 return rc; 1405 rc = qdio_setup_init(); 1406 if (rc) 1407 goto out_debug; 1408 rc = qdio_thinint_init(); 1409 if (rc) 1410 goto out_cache; 1411 return 0; 1412 1413 out_cache: 1414 qdio_setup_exit(); 1415 out_debug: 1416 qdio_debug_exit(); 1417 return rc; 1418 } 1419 1420 static void __exit exit_QDIO(void) 1421 { 1422 qdio_thinint_exit(); 1423 qdio_setup_exit(); 1424 qdio_debug_exit(); 1425 } 1426 1427 module_init(init_QDIO); 1428 module_exit(exit_QDIO); 1429