1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Linux for s390 qdio support, buffer handling, qdio API and module support. 4 * 5 * Copyright IBM Corp. 2000, 2008 6 * Author(s): Utz Bacher <utz.bacher@de.ibm.com> 7 * Jan Glauber <jang@linux.vnet.ibm.com> 8 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com> 9 */ 10 #include <linux/module.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/kmemleak.h> 14 #include <linux/delay.h> 15 #include <linux/gfp.h> 16 #include <linux/io.h> 17 #include <linux/atomic.h> 18 #include <asm/debug.h> 19 #include <asm/qdio.h> 20 #include <asm/ipl.h> 21 22 #include "cio.h" 23 #include "css.h" 24 #include "device.h" 25 #include "qdio.h" 26 #include "qdio_debug.h" 27 28 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\ 29 "Jan Glauber <jang@linux.vnet.ibm.com>"); 30 MODULE_DESCRIPTION("QDIO base support"); 31 MODULE_LICENSE("GPL"); 32 33 static inline int do_siga_sync(unsigned long schid, 34 unsigned long out_mask, unsigned long in_mask, 35 unsigned int fc) 36 { 37 int cc; 38 39 asm volatile( 40 " lgr 0,%[fc]\n" 41 " lgr 1,%[schid]\n" 42 " lgr 2,%[out]\n" 43 " lgr 3,%[in]\n" 44 " siga 0\n" 45 " ipm %[cc]\n" 46 " srl %[cc],28\n" 47 : [cc] "=&d" (cc) 48 : [fc] "d" (fc), [schid] "d" (schid), 49 [out] "d" (out_mask), [in] "d" (in_mask) 50 : "cc", "0", "1", "2", "3"); 51 return cc; 52 } 53 54 static inline int do_siga_input(unsigned long schid, unsigned long mask, 55 unsigned long fc) 56 { 57 int cc; 58 59 asm volatile( 60 " lgr 0,%[fc]\n" 61 " lgr 1,%[schid]\n" 62 " lgr 2,%[mask]\n" 63 " siga 0\n" 64 " ipm %[cc]\n" 65 " srl %[cc],28\n" 66 : [cc] "=&d" (cc) 67 : [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask) 68 : "cc", "0", "1", "2"); 69 return cc; 70 } 71 72 /** 73 * do_siga_output - perform SIGA-w/wt function 74 * @schid: subchannel id or in case of QEBSM the subchannel token 75 * @mask: which output queues to process 76 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer 77 * @fc: function code to perform 78 * @aob: asynchronous operation block 79 * 80 * Returns condition code. 81 * Note: For IQDC unicast queues only the highest priority queue is processed. 82 */ 83 static inline int do_siga_output(unsigned long schid, unsigned long mask, 84 unsigned int *bb, unsigned long fc, 85 dma64_t aob) 86 { 87 int cc; 88 89 asm volatile( 90 " lgr 0,%[fc]\n" 91 " lgr 1,%[schid]\n" 92 " lgr 2,%[mask]\n" 93 " lgr 3,%[aob]\n" 94 " siga 0\n" 95 " lgr %[fc],0\n" 96 " ipm %[cc]\n" 97 " srl %[cc],28\n" 98 : [cc] "=&d" (cc), [fc] "+&d" (fc) 99 : [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob) 100 : "cc", "0", "1", "2", "3"); 101 *bb = fc >> 31; 102 return cc; 103 } 104 105 /** 106 * qdio_do_eqbs - extract buffer states for QEBSM 107 * @q: queue to manipulate 108 * @state: state of the extracted buffers 109 * @start: buffer number to start at 110 * @count: count of buffers to examine 111 * @auto_ack: automatically acknowledge buffers 112 * 113 * Returns the number of successfully extracted equal buffer states. 114 * Stops processing if a state is different from the last buffers state. 115 */ 116 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state, 117 int start, int count, int auto_ack) 118 { 119 int tmp_count = count, tmp_start = start, nr = q->nr; 120 unsigned int ccq = 0; 121 122 qperf_inc(q, eqbs); 123 124 if (!q->is_input_q) 125 nr += q->irq_ptr->nr_input_qs; 126 again: 127 ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count, 128 auto_ack); 129 130 switch (ccq) { 131 case 0: 132 case 32: 133 /* all done, or next buffer state different */ 134 return count - tmp_count; 135 case 96: 136 /* not all buffers processed */ 137 qperf_inc(q, eqbs_partial); 138 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x", 139 tmp_count); 140 return count - tmp_count; 141 case 97: 142 /* no buffer processed */ 143 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq); 144 goto again; 145 default: 146 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 147 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q)); 148 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 149 q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr, 150 q->first_to_check, count, q->irq_ptr->int_parm); 151 return 0; 152 } 153 } 154 155 /** 156 * qdio_do_sqbs - set buffer states for QEBSM 157 * @q: queue to manipulate 158 * @state: new state of the buffers 159 * @start: first buffer number to change 160 * @count: how many buffers to change 161 * 162 * Returns the number of successfully changed buffers. 163 * Does retrying until the specified count of buffer states is set or an 164 * error occurs. 165 */ 166 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start, 167 int count) 168 { 169 unsigned int ccq = 0; 170 int tmp_count = count, tmp_start = start; 171 int nr = q->nr; 172 173 qperf_inc(q, sqbs); 174 175 if (!q->is_input_q) 176 nr += q->irq_ptr->nr_input_qs; 177 again: 178 ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count); 179 180 switch (ccq) { 181 case 0: 182 case 32: 183 /* all done, or active buffer adapter-owned */ 184 WARN_ON_ONCE(tmp_count); 185 return count - tmp_count; 186 case 96: 187 /* not all buffers processed */ 188 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq); 189 qperf_inc(q, sqbs_partial); 190 goto again; 191 default: 192 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq); 193 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q)); 194 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); 195 q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr, 196 q->first_to_check, count, q->irq_ptr->int_parm); 197 return 0; 198 } 199 } 200 201 /* 202 * Returns number of examined buffers and their common state in *state. 203 * Requested number of buffers-to-examine must be > 0. 204 */ 205 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, 206 unsigned char *state, unsigned int count, 207 int auto_ack) 208 { 209 unsigned char __state = 0; 210 int i = 1; 211 212 if (is_qebsm(q)) 213 return qdio_do_eqbs(q, state, bufnr, count, auto_ack); 214 215 /* get initial state: */ 216 __state = q->slsb.val[bufnr]; 217 218 /* Bail out early if there is no work on the queue: */ 219 if (__state & SLSB_OWNER_CU) 220 goto out; 221 222 for (; i < count; i++) { 223 bufnr = next_buf(bufnr); 224 225 /* stop if next state differs from initial state: */ 226 if (q->slsb.val[bufnr] != __state) 227 break; 228 } 229 230 out: 231 *state = __state; 232 return i; 233 } 234 235 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr, 236 unsigned char *state, int auto_ack) 237 { 238 return get_buf_states(q, bufnr, state, 1, auto_ack); 239 } 240 241 /* wrap-around safe setting of slsb states, returns number of changed buffers */ 242 static inline int set_buf_states(struct qdio_q *q, int bufnr, 243 unsigned char state, int count) 244 { 245 int i; 246 247 if (is_qebsm(q)) 248 return qdio_do_sqbs(q, state, bufnr, count); 249 250 /* Ensure that all preceding changes to the SBALs are visible: */ 251 mb(); 252 253 for (i = 0; i < count; i++) { 254 WRITE_ONCE(q->slsb.val[bufnr], state); 255 bufnr = next_buf(bufnr); 256 } 257 258 /* Make our SLSB changes visible: */ 259 mb(); 260 261 return count; 262 } 263 264 static inline int set_buf_state(struct qdio_q *q, int bufnr, 265 unsigned char state) 266 { 267 return set_buf_states(q, bufnr, state, 1); 268 } 269 270 /* set slsb states to initial state */ 271 static void qdio_init_buf_states(struct qdio_irq *irq_ptr) 272 { 273 struct qdio_q *q; 274 int i; 275 276 for_each_input_queue(irq_ptr, q, i) 277 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT, 278 QDIO_MAX_BUFFERS_PER_Q); 279 for_each_output_queue(irq_ptr, q, i) 280 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT, 281 QDIO_MAX_BUFFERS_PER_Q); 282 } 283 284 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output, 285 unsigned int input) 286 { 287 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 288 unsigned int fc = QDIO_SIGA_SYNC; 289 int cc; 290 291 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr); 292 qperf_inc(q, siga_sync); 293 294 if (is_qebsm(q)) { 295 schid = q->irq_ptr->sch_token; 296 fc |= QDIO_SIGA_QEBSM_FLAG; 297 } 298 299 cc = do_siga_sync(schid, output, input, fc); 300 if (unlikely(cc)) 301 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc); 302 return (cc) ? -EIO : 0; 303 } 304 305 static inline int qdio_sync_input_queue(struct qdio_q *q) 306 { 307 return qdio_siga_sync(q, 0, q->mask); 308 } 309 310 static inline int qdio_sync_output_queue(struct qdio_q *q) 311 { 312 return qdio_siga_sync(q, q->mask, 0); 313 } 314 315 static inline int qdio_siga_sync_q(struct qdio_q *q) 316 { 317 if (q->is_input_q) 318 return qdio_sync_input_queue(q); 319 else 320 return qdio_sync_output_queue(q); 321 } 322 323 static int qdio_siga_output(struct qdio_q *q, unsigned int count, 324 unsigned int *busy_bit, dma64_t aob) 325 { 326 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 327 unsigned int fc = QDIO_SIGA_WRITE; 328 u64 start_time = 0; 329 int retries = 0, cc; 330 331 if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q)) { 332 if (count > 1) 333 fc = QDIO_SIGA_WRITEM; 334 else if (aob) 335 fc = QDIO_SIGA_WRITEQ; 336 } 337 338 if (is_qebsm(q)) { 339 schid = q->irq_ptr->sch_token; 340 fc |= QDIO_SIGA_QEBSM_FLAG; 341 } 342 again: 343 cc = do_siga_output(schid, q->mask, busy_bit, fc, aob); 344 345 /* hipersocket busy condition */ 346 if (unlikely(*busy_bit)) { 347 retries++; 348 349 if (!start_time) { 350 start_time = get_tod_clock_fast(); 351 goto again; 352 } 353 if (get_tod_clock_fast() - start_time < QDIO_BUSY_BIT_PATIENCE) 354 goto again; 355 } 356 if (retries) { 357 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, 358 "%4x cc2 BB1:%1d", SCH_NO(q), q->nr); 359 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries); 360 } 361 return cc; 362 } 363 364 static inline int qdio_siga_input(struct qdio_q *q) 365 { 366 unsigned long schid = *((u32 *) &q->irq_ptr->schid); 367 unsigned int fc = QDIO_SIGA_READ; 368 int cc; 369 370 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr); 371 qperf_inc(q, siga_read); 372 373 if (is_qebsm(q)) { 374 schid = q->irq_ptr->sch_token; 375 fc |= QDIO_SIGA_QEBSM_FLAG; 376 } 377 378 cc = do_siga_input(schid, q->mask, fc); 379 if (unlikely(cc)) 380 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc); 381 return (cc) ? -EIO : 0; 382 } 383 384 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, 385 unsigned char *state) 386 { 387 if (qdio_need_siga_sync(q->irq_ptr)) 388 qdio_siga_sync_q(q); 389 return get_buf_state(q, bufnr, state, 0); 390 } 391 392 static inline void qdio_stop_polling(struct qdio_q *q) 393 { 394 if (!q->u.in.batch_count) 395 return; 396 397 qperf_inc(q, stop_polling); 398 399 /* show the card that we are not polling anymore */ 400 set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT, 401 q->u.in.batch_count); 402 q->u.in.batch_count = 0; 403 } 404 405 static inline void account_sbals(struct qdio_q *q, unsigned int count) 406 { 407 q->q_stats.nr_sbal_total += count; 408 q->q_stats.nr_sbals[ilog2(count)]++; 409 } 410 411 static void process_buffer_error(struct qdio_q *q, unsigned int start, 412 int count) 413 { 414 /* special handling for no target buffer empty */ 415 if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q && 416 q->sbal[start]->element[15].sflags == 0x10) { 417 qperf_inc(q, target_full); 418 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start); 419 return; 420 } 421 422 DBF_ERROR("%4x BUF ERROR", SCH_NO(q)); 423 DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr); 424 DBF_ERROR("FTC:%3d C:%3d", start, count); 425 DBF_ERROR("F14:%2x F15:%2x", 426 q->sbal[start]->element[14].sflags, 427 q->sbal[start]->element[15].sflags); 428 } 429 430 static inline void inbound_handle_work(struct qdio_q *q, unsigned int start, 431 int count, bool auto_ack) 432 { 433 /* ACK the newest SBAL: */ 434 if (!auto_ack) 435 set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK); 436 437 if (!q->u.in.batch_count) 438 q->u.in.batch_start = start; 439 q->u.in.batch_count += count; 440 } 441 442 static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start, 443 unsigned int *error) 444 { 445 unsigned char state = 0; 446 int count; 447 448 q->timestamp = get_tod_clock_fast(); 449 450 count = atomic_read(&q->nr_buf_used); 451 if (!count) 452 return 0; 453 454 if (qdio_need_siga_sync(q->irq_ptr)) 455 qdio_sync_input_queue(q); 456 457 count = get_buf_states(q, start, &state, count, 1); 458 if (!count) 459 return 0; 460 461 switch (state) { 462 case SLSB_P_INPUT_PRIMED: 463 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, 464 count); 465 466 inbound_handle_work(q, start, count, is_qebsm(q)); 467 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 468 qperf_inc(q, inbound_queue_full); 469 if (q->irq_ptr->perf_stat_enabled) 470 account_sbals(q, count); 471 return count; 472 case SLSB_P_INPUT_ERROR: 473 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr, 474 count); 475 476 *error = QDIO_ERROR_SLSB_STATE; 477 process_buffer_error(q, start, count); 478 inbound_handle_work(q, start, count, false); 479 if (atomic_sub_return(count, &q->nr_buf_used) == 0) 480 qperf_inc(q, inbound_queue_full); 481 if (q->irq_ptr->perf_stat_enabled) 482 account_sbals_error(q, count); 483 return count; 484 case SLSB_CU_INPUT_EMPTY: 485 if (q->irq_ptr->perf_stat_enabled) 486 q->q_stats.nr_sbal_nop++; 487 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x", 488 q->nr, start); 489 return 0; 490 case SLSB_P_INPUT_NOT_INIT: 491 case SLSB_P_INPUT_ACK: 492 /* We should never see this state, throw a WARN: */ 493 default: 494 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 495 "found state %#x at index %u on queue %u\n", 496 state, start, q->nr); 497 return 0; 498 } 499 } 500 501 int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr, 502 unsigned int *bufnr, unsigned int *error) 503 { 504 struct qdio_irq *irq = cdev->private->qdio_data; 505 unsigned int start; 506 struct qdio_q *q; 507 int count; 508 509 if (!irq) 510 return -ENODEV; 511 512 q = irq->input_qs[nr]; 513 start = q->first_to_check; 514 *error = 0; 515 516 count = get_inbound_buffer_frontier(q, start, error); 517 if (count == 0) 518 return 0; 519 520 *bufnr = start; 521 q->first_to_check = add_buf(start, count); 522 return count; 523 } 524 EXPORT_SYMBOL_GPL(qdio_inspect_input_queue); 525 526 static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) 527 { 528 unsigned char state = 0; 529 530 if (!atomic_read(&q->nr_buf_used)) 531 return 1; 532 533 if (qdio_need_siga_sync(q->irq_ptr)) 534 qdio_sync_input_queue(q); 535 get_buf_state(q, start, &state, 0); 536 537 if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) 538 /* more work coming */ 539 return 0; 540 541 return 1; 542 } 543 544 static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start, 545 unsigned int *error) 546 { 547 unsigned char state = 0; 548 int count; 549 550 q->timestamp = get_tod_clock_fast(); 551 552 count = atomic_read(&q->nr_buf_used); 553 if (!count) 554 return 0; 555 556 if (qdio_need_siga_sync(q->irq_ptr)) 557 qdio_sync_output_queue(q); 558 559 count = get_buf_states(q, start, &state, count, 0); 560 if (!count) 561 return 0; 562 563 switch (state) { 564 case SLSB_P_OUTPUT_PENDING: 565 *error = QDIO_ERROR_SLSB_PENDING; 566 fallthrough; 567 case SLSB_P_OUTPUT_EMPTY: 568 /* the adapter got it */ 569 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, 570 "out empty:%1d %02x", q->nr, count); 571 572 atomic_sub(count, &q->nr_buf_used); 573 if (q->irq_ptr->perf_stat_enabled) 574 account_sbals(q, count); 575 return count; 576 case SLSB_P_OUTPUT_ERROR: 577 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x", 578 q->nr, count); 579 580 *error = QDIO_ERROR_SLSB_STATE; 581 process_buffer_error(q, start, count); 582 atomic_sub(count, &q->nr_buf_used); 583 if (q->irq_ptr->perf_stat_enabled) 584 account_sbals_error(q, count); 585 return count; 586 case SLSB_CU_OUTPUT_PRIMED: 587 /* the adapter has not fetched the output yet */ 588 if (q->irq_ptr->perf_stat_enabled) 589 q->q_stats.nr_sbal_nop++; 590 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", 591 q->nr); 592 return 0; 593 case SLSB_P_OUTPUT_HALTED: 594 return 0; 595 case SLSB_P_OUTPUT_NOT_INIT: 596 /* We should never see this state, throw a WARN: */ 597 default: 598 dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, 599 "found state %#x at index %u on queue %u\n", 600 state, start, q->nr); 601 return 0; 602 } 603 } 604 605 int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr, 606 unsigned int *bufnr, unsigned int *error) 607 { 608 struct qdio_irq *irq = cdev->private->qdio_data; 609 unsigned int start; 610 struct qdio_q *q; 611 int count; 612 613 if (!irq) 614 return -ENODEV; 615 616 q = irq->output_qs[nr]; 617 start = q->first_to_check; 618 *error = 0; 619 620 count = get_outbound_buffer_frontier(q, start, error); 621 if (count == 0) 622 return 0; 623 624 *bufnr = start; 625 q->first_to_check = add_buf(start, count); 626 return count; 627 } 628 EXPORT_SYMBOL_GPL(qdio_inspect_output_queue); 629 630 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count, 631 dma64_t aob) 632 { 633 int retries = 0, cc; 634 unsigned int busy_bit; 635 636 if (!qdio_need_siga_out(q->irq_ptr)) 637 return 0; 638 639 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr); 640 retry: 641 qperf_inc(q, siga_write); 642 643 cc = qdio_siga_output(q, count, &busy_bit, aob); 644 switch (cc) { 645 case 0: 646 break; 647 case 2: 648 if (busy_bit) { 649 while (++retries < QDIO_BUSY_BIT_RETRIES) { 650 mdelay(QDIO_BUSY_BIT_RETRY_DELAY); 651 goto retry; 652 } 653 DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr); 654 cc = -EBUSY; 655 } else { 656 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr); 657 cc = -ENOBUFS; 658 } 659 break; 660 case 1: 661 case 3: 662 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc); 663 cc = -EIO; 664 break; 665 } 666 if (retries) { 667 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr); 668 DBF_ERROR("count:%u", retries); 669 } 670 return cc; 671 } 672 673 static inline void qdio_set_state(struct qdio_irq *irq_ptr, 674 enum qdio_irq_states state) 675 { 676 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state); 677 678 irq_ptr->state = state; 679 mb(); 680 } 681 682 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb) 683 { 684 if (irb->esw.esw0.erw.cons) { 685 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no); 686 DBF_ERROR_HEX(irb, 64); 687 DBF_ERROR_HEX(irb->ecw, 64); 688 } 689 } 690 691 /* PCI interrupt handler */ 692 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) 693 { 694 if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) 695 return; 696 697 qdio_deliver_irq(irq_ptr); 698 irq_ptr->last_data_irq_time = S390_lowcore.int_clock; 699 } 700 701 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr, 702 unsigned long intparm, int cstat, 703 int dstat) 704 { 705 unsigned int first_to_check = 0; 706 707 DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no); 708 DBF_ERROR("intp :%lx", intparm); 709 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 710 711 /* zfcp wants this: */ 712 if (irq_ptr->nr_input_qs) 713 first_to_check = irq_ptr->input_qs[0]->first_to_check; 714 715 irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0, 716 first_to_check, 0, irq_ptr->int_parm); 717 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 718 /* 719 * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen. 720 * Therefore we call the LGR detection function here. 721 */ 722 lgr_info_log(); 723 } 724 725 static int qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat, 726 int dstat, int dcc) 727 { 728 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq"); 729 730 if (cstat) 731 goto error; 732 if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END)) 733 goto error; 734 if (dcc == 1) 735 return -EAGAIN; 736 if (!(dstat & DEV_STAT_DEV_END)) 737 goto error; 738 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED); 739 return 0; 740 741 error: 742 DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no); 743 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); 744 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 745 return -EIO; 746 } 747 748 /* qdio interrupt handler */ 749 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, 750 struct irb *irb) 751 { 752 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 753 struct subchannel_id schid; 754 int cstat, dstat, rc, dcc; 755 756 if (!intparm || !irq_ptr) { 757 ccw_device_get_schid(cdev, &schid); 758 DBF_ERROR("qint:%4x", schid.sch_no); 759 return; 760 } 761 762 if (irq_ptr->perf_stat_enabled) 763 irq_ptr->perf_stat.qdio_int++; 764 765 if (IS_ERR(irb)) { 766 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no); 767 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 768 wake_up(&cdev->private->wait_q); 769 return; 770 } 771 qdio_irq_check_sense(irq_ptr, irb); 772 cstat = irb->scsw.cmd.cstat; 773 dstat = irb->scsw.cmd.dstat; 774 dcc = scsw_cmd_is_valid_cc(&irb->scsw) ? irb->scsw.cmd.cc : 0; 775 rc = 0; 776 777 switch (irq_ptr->state) { 778 case QDIO_IRQ_STATE_INACTIVE: 779 rc = qdio_establish_handle_irq(irq_ptr, cstat, dstat, dcc); 780 break; 781 case QDIO_IRQ_STATE_CLEANUP: 782 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 783 break; 784 case QDIO_IRQ_STATE_ESTABLISHED: 785 case QDIO_IRQ_STATE_ACTIVE: 786 if (cstat & SCHN_STAT_PCI) { 787 qdio_int_handler_pci(irq_ptr); 788 return; 789 } 790 if (cstat || dstat) 791 qdio_handle_activate_check(irq_ptr, intparm, cstat, 792 dstat); 793 else if (dcc == 1) 794 rc = -EAGAIN; 795 break; 796 case QDIO_IRQ_STATE_STOPPED: 797 break; 798 default: 799 WARN_ON_ONCE(1); 800 } 801 802 if (rc == -EAGAIN) { 803 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qint retry"); 804 rc = ccw_device_start(cdev, irq_ptr->ccw, intparm, 0, 0); 805 if (!rc) 806 return; 807 DBF_ERROR("%4x RETRY ERR", irq_ptr->schid.sch_no); 808 DBF_ERROR("rc:%4x", rc); 809 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR); 810 } 811 812 wake_up(&cdev->private->wait_q); 813 } 814 815 /** 816 * qdio_get_ssqd_desc - get qdio subchannel description 817 * @cdev: ccw device to get description for 818 * @data: where to store the ssqd 819 * 820 * Returns 0 or an error code. The results of the chsc are stored in the 821 * specified structure. 822 */ 823 int qdio_get_ssqd_desc(struct ccw_device *cdev, 824 struct qdio_ssqd_desc *data) 825 { 826 struct subchannel_id schid; 827 828 if (!cdev || !cdev->private) 829 return -EINVAL; 830 831 ccw_device_get_schid(cdev, &schid); 832 DBF_EVENT("get ssqd:%4x", schid.sch_no); 833 return qdio_setup_get_ssqd(NULL, &schid, data); 834 } 835 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc); 836 837 static int qdio_cancel_ccw(struct qdio_irq *irq, int how) 838 { 839 struct ccw_device *cdev = irq->cdev; 840 long timeout; 841 int rc; 842 843 spin_lock_irq(get_ccwdev_lock(cdev)); 844 qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP); 845 if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) 846 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); 847 else 848 /* default behaviour is halt */ 849 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); 850 spin_unlock_irq(get_ccwdev_lock(cdev)); 851 if (rc) { 852 DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no); 853 DBF_ERROR("rc:%4d", rc); 854 return rc; 855 } 856 857 timeout = wait_event_interruptible_timeout(cdev->private->wait_q, 858 irq->state == QDIO_IRQ_STATE_INACTIVE || 859 irq->state == QDIO_IRQ_STATE_ERR, 860 10 * HZ); 861 if (timeout <= 0) 862 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; 863 864 return rc; 865 } 866 867 /** 868 * qdio_shutdown - shut down a qdio subchannel 869 * @cdev: associated ccw device 870 * @how: use halt or clear to shutdown 871 */ 872 int qdio_shutdown(struct ccw_device *cdev, int how) 873 { 874 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 875 struct subchannel_id schid; 876 int rc; 877 878 if (!irq_ptr) 879 return -ENODEV; 880 881 WARN_ON_ONCE(irqs_disabled()); 882 ccw_device_get_schid(cdev, &schid); 883 DBF_EVENT("qshutdown:%4x", schid.sch_no); 884 885 mutex_lock(&irq_ptr->setup_mutex); 886 /* 887 * Subchannel was already shot down. We cannot prevent being called 888 * twice since cio may trigger a shutdown asynchronously. 889 */ 890 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 891 mutex_unlock(&irq_ptr->setup_mutex); 892 return 0; 893 } 894 895 /* 896 * Indicate that the device is going down. 897 */ 898 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); 899 900 qdio_shutdown_debug_entries(irq_ptr); 901 902 rc = qdio_cancel_ccw(irq_ptr, how); 903 qdio_shutdown_thinint(irq_ptr); 904 qdio_shutdown_irq(irq_ptr); 905 906 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 907 mutex_unlock(&irq_ptr->setup_mutex); 908 if (rc) 909 return rc; 910 return 0; 911 } 912 EXPORT_SYMBOL_GPL(qdio_shutdown); 913 914 /** 915 * qdio_free - free data structures for a qdio subchannel 916 * @cdev: associated ccw device 917 */ 918 int qdio_free(struct ccw_device *cdev) 919 { 920 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 921 struct subchannel_id schid; 922 923 if (!irq_ptr) 924 return -ENODEV; 925 926 ccw_device_get_schid(cdev, &schid); 927 DBF_EVENT("qfree:%4x", schid.sch_no); 928 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned"); 929 mutex_lock(&irq_ptr->setup_mutex); 930 931 irq_ptr->debug_area = NULL; 932 cdev->private->qdio_data = NULL; 933 mutex_unlock(&irq_ptr->setup_mutex); 934 935 qdio_free_queues(irq_ptr); 936 free_page((unsigned long) irq_ptr->qdr); 937 free_page(irq_ptr->chsc_page); 938 kfree(irq_ptr->ccw); 939 free_page((unsigned long) irq_ptr); 940 return 0; 941 } 942 EXPORT_SYMBOL_GPL(qdio_free); 943 944 /** 945 * qdio_allocate - allocate qdio queues and associated data 946 * @cdev: associated ccw device 947 * @no_input_qs: allocate this number of Input Queues 948 * @no_output_qs: allocate this number of Output Queues 949 */ 950 int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, 951 unsigned int no_output_qs) 952 { 953 struct subchannel_id schid; 954 struct qdio_irq *irq_ptr; 955 int rc = -ENOMEM; 956 957 ccw_device_get_schid(cdev, &schid); 958 DBF_EVENT("qallocate:%4x", schid.sch_no); 959 960 if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ || 961 no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) 962 return -EINVAL; 963 964 irq_ptr = (void *) get_zeroed_page(GFP_KERNEL); 965 if (!irq_ptr) 966 return -ENOMEM; 967 968 irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA); 969 if (!irq_ptr->ccw) 970 goto err_ccw; 971 972 /* kmemleak doesn't scan the page-allocated irq_ptr: */ 973 kmemleak_not_leak(irq_ptr->ccw); 974 975 irq_ptr->cdev = cdev; 976 mutex_init(&irq_ptr->setup_mutex); 977 if (qdio_allocate_dbf(irq_ptr)) 978 goto err_dbf; 979 980 DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs, 981 no_output_qs); 982 983 /* 984 * Allocate a page for the chsc calls in qdio_establish. 985 * Must be pre-allocated since a zfcp recovery will call 986 * qdio_establish. In case of low memory and swap on a zfcp disk 987 * we may not be able to allocate memory otherwise. 988 */ 989 irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); 990 if (!irq_ptr->chsc_page) 991 goto err_chsc; 992 993 /* qdr is used in ccw1.cda which is u32 */ 994 irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 995 if (!irq_ptr->qdr) 996 goto err_qdr; 997 998 rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs); 999 if (rc) 1000 goto err_queues; 1001 1002 cdev->private->qdio_data = irq_ptr; 1003 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1004 return 0; 1005 1006 err_queues: 1007 free_page((unsigned long) irq_ptr->qdr); 1008 err_qdr: 1009 free_page(irq_ptr->chsc_page); 1010 err_chsc: 1011 err_dbf: 1012 kfree(irq_ptr->ccw); 1013 err_ccw: 1014 free_page((unsigned long) irq_ptr); 1015 return rc; 1016 } 1017 EXPORT_SYMBOL_GPL(qdio_allocate); 1018 1019 static void qdio_trace_init_data(struct qdio_irq *irq, 1020 struct qdio_initialize *data) 1021 { 1022 DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format); 1023 DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format); 1024 DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR); 1025 DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs, 1026 data->no_output_qs); 1027 DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR); 1028 DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR); 1029 DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR); 1030 DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR); 1031 DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *), 1032 DBF_ERR); 1033 } 1034 1035 /** 1036 * qdio_establish - establish queues on a qdio subchannel 1037 * @cdev: associated ccw device 1038 * @init_data: initialization data 1039 */ 1040 int qdio_establish(struct ccw_device *cdev, 1041 struct qdio_initialize *init_data) 1042 { 1043 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1044 struct subchannel_id schid; 1045 struct ciw *ciw; 1046 long timeout; 1047 int rc; 1048 1049 ccw_device_get_schid(cdev, &schid); 1050 DBF_EVENT("qestablish:%4x", schid.sch_no); 1051 1052 if (!irq_ptr) 1053 return -ENODEV; 1054 1055 if (init_data->no_input_qs > irq_ptr->max_input_qs || 1056 init_data->no_output_qs > irq_ptr->max_output_qs) 1057 return -EINVAL; 1058 1059 /* Needed as error_handler: */ 1060 if (!init_data->input_handler) 1061 return -EINVAL; 1062 1063 if (init_data->no_output_qs && !init_data->output_handler) 1064 return -EINVAL; 1065 1066 if (!init_data->input_sbal_addr_array || 1067 !init_data->output_sbal_addr_array) 1068 return -EINVAL; 1069 1070 if (!init_data->irq_poll) 1071 return -EINVAL; 1072 1073 ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE); 1074 if (!ciw) { 1075 DBF_ERROR("%4x NO EQ", schid.sch_no); 1076 return -EIO; 1077 } 1078 1079 mutex_lock(&irq_ptr->setup_mutex); 1080 qdio_trace_init_data(irq_ptr, init_data); 1081 qdio_setup_irq(irq_ptr, init_data); 1082 1083 rc = qdio_establish_thinint(irq_ptr); 1084 if (rc) 1085 goto err_thinint; 1086 1087 /* establish q */ 1088 irq_ptr->ccw->cmd_code = ciw->cmd; 1089 irq_ptr->ccw->flags = CCW_FLAG_SLI; 1090 irq_ptr->ccw->count = ciw->count; 1091 irq_ptr->ccw->cda = virt_to_dma32(irq_ptr->qdr); 1092 1093 spin_lock_irq(get_ccwdev_lock(cdev)); 1094 ccw_device_set_options_mask(cdev, 0); 1095 1096 rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); 1097 spin_unlock_irq(get_ccwdev_lock(cdev)); 1098 if (rc) { 1099 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no); 1100 DBF_ERROR("rc:%4x", rc); 1101 goto err_ccw_start; 1102 } 1103 1104 timeout = wait_event_interruptible_timeout(cdev->private->wait_q, 1105 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || 1106 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); 1107 if (timeout <= 0) { 1108 rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; 1109 goto err_ccw_timeout; 1110 } 1111 1112 if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) { 1113 rc = -EIO; 1114 goto err_ccw_error; 1115 } 1116 1117 qdio_setup_ssqd_info(irq_ptr); 1118 1119 /* qebsm is now setup if available, initialize buffer states */ 1120 qdio_init_buf_states(irq_ptr); 1121 1122 mutex_unlock(&irq_ptr->setup_mutex); 1123 qdio_print_subchannel_info(irq_ptr); 1124 qdio_setup_debug_entries(irq_ptr); 1125 return 0; 1126 1127 err_ccw_timeout: 1128 qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR); 1129 err_ccw_error: 1130 err_ccw_start: 1131 qdio_shutdown_thinint(irq_ptr); 1132 err_thinint: 1133 qdio_shutdown_irq(irq_ptr); 1134 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); 1135 mutex_unlock(&irq_ptr->setup_mutex); 1136 return rc; 1137 } 1138 EXPORT_SYMBOL_GPL(qdio_establish); 1139 1140 /** 1141 * qdio_activate - activate queues on a qdio subchannel 1142 * @cdev: associated cdev 1143 */ 1144 int qdio_activate(struct ccw_device *cdev) 1145 { 1146 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1147 struct subchannel_id schid; 1148 struct ciw *ciw; 1149 int rc; 1150 1151 ccw_device_get_schid(cdev, &schid); 1152 DBF_EVENT("qactivate:%4x", schid.sch_no); 1153 1154 if (!irq_ptr) 1155 return -ENODEV; 1156 1157 ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE); 1158 if (!ciw) { 1159 DBF_ERROR("%4x NO AQ", schid.sch_no); 1160 return -EIO; 1161 } 1162 1163 mutex_lock(&irq_ptr->setup_mutex); 1164 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { 1165 rc = -EBUSY; 1166 goto out; 1167 } 1168 1169 irq_ptr->ccw->cmd_code = ciw->cmd; 1170 irq_ptr->ccw->flags = CCW_FLAG_SLI; 1171 irq_ptr->ccw->count = ciw->count; 1172 irq_ptr->ccw->cda = 0; 1173 1174 spin_lock_irq(get_ccwdev_lock(cdev)); 1175 ccw_device_set_options(cdev, CCWDEV_REPORT_ALL); 1176 1177 rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE, 1178 0, DOIO_DENY_PREFETCH); 1179 spin_unlock_irq(get_ccwdev_lock(cdev)); 1180 if (rc) { 1181 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no); 1182 DBF_ERROR("rc:%4x", rc); 1183 goto out; 1184 } 1185 1186 /* wait for subchannel to become active */ 1187 msleep(5); 1188 1189 switch (irq_ptr->state) { 1190 case QDIO_IRQ_STATE_STOPPED: 1191 case QDIO_IRQ_STATE_ERR: 1192 rc = -EIO; 1193 break; 1194 default: 1195 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE); 1196 rc = 0; 1197 } 1198 out: 1199 mutex_unlock(&irq_ptr->setup_mutex); 1200 return rc; 1201 } 1202 EXPORT_SYMBOL_GPL(qdio_activate); 1203 1204 /** 1205 * handle_inbound - reset processed input buffers 1206 * @q: queue containing the buffers 1207 * @bufnr: first buffer to process 1208 * @count: how many buffers are emptied 1209 */ 1210 static int handle_inbound(struct qdio_q *q, int bufnr, int count) 1211 { 1212 int overlap; 1213 1214 qperf_inc(q, inbound_call); 1215 1216 /* If any processed SBALs are returned to HW, adjust our tracking: */ 1217 overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr), 1218 q->u.in.batch_count); 1219 if (overlap > 0) { 1220 q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap); 1221 q->u.in.batch_count -= overlap; 1222 } 1223 1224 count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count); 1225 atomic_add(count, &q->nr_buf_used); 1226 1227 if (qdio_need_siga_in(q->irq_ptr)) 1228 return qdio_siga_input(q); 1229 1230 return 0; 1231 } 1232 1233 /** 1234 * qdio_add_bufs_to_input_queue - process buffers on an Input Queue 1235 * @cdev: associated ccw_device for the qdio subchannel 1236 * @q_nr: queue number 1237 * @bufnr: buffer number 1238 * @count: how many buffers to process 1239 */ 1240 int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr, 1241 unsigned int bufnr, unsigned int count) 1242 { 1243 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1244 1245 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1246 return -EINVAL; 1247 1248 if (!irq_ptr) 1249 return -ENODEV; 1250 1251 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count); 1252 1253 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1254 return -EIO; 1255 if (!count) 1256 return 0; 1257 1258 return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count); 1259 } 1260 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue); 1261 1262 /** 1263 * handle_outbound - process filled outbound buffers 1264 * @q: queue containing the buffers 1265 * @bufnr: first buffer to process 1266 * @count: how many buffers are filled 1267 * @aob: asynchronous operation block 1268 */ 1269 static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count, 1270 struct qaob *aob) 1271 { 1272 unsigned char state = 0; 1273 int used, rc = 0; 1274 1275 qperf_inc(q, outbound_call); 1276 1277 count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count); 1278 used = atomic_add_return(count, &q->nr_buf_used); 1279 1280 if (used == QDIO_MAX_BUFFERS_PER_Q) 1281 qperf_inc(q, outbound_queue_full); 1282 1283 if (queue_type(q) == QDIO_IQDIO_QFMT) { 1284 dma64_t phys_aob = aob ? virt_to_dma64(aob) : 0; 1285 1286 WARN_ON_ONCE(!IS_ALIGNED(dma64_to_u64(phys_aob), 256)); 1287 rc = qdio_kick_outbound_q(q, count, phys_aob); 1288 } else if (qdio_need_siga_sync(q->irq_ptr)) { 1289 rc = qdio_sync_output_queue(q); 1290 } else if (count < QDIO_MAX_BUFFERS_PER_Q && 1291 get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 && 1292 state == SLSB_CU_OUTPUT_PRIMED) { 1293 /* The previous buffer is not processed yet, tack on. */ 1294 qperf_inc(q, fast_requeue); 1295 } else { 1296 rc = qdio_kick_outbound_q(q, count, 0); 1297 } 1298 1299 return rc; 1300 } 1301 1302 /** 1303 * qdio_add_bufs_to_output_queue - process buffers on an Output Queue 1304 * @cdev: associated ccw_device for the qdio subchannel 1305 * @q_nr: queue number 1306 * @bufnr: buffer number 1307 * @count: how many buffers to process 1308 * @aob: asynchronous operation block 1309 */ 1310 int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr, 1311 unsigned int bufnr, unsigned int count, 1312 struct qaob *aob) 1313 { 1314 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1315 1316 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) 1317 return -EINVAL; 1318 1319 if (!irq_ptr) 1320 return -ENODEV; 1321 1322 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count); 1323 1324 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) 1325 return -EIO; 1326 if (!count) 1327 return 0; 1328 1329 return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob); 1330 } 1331 EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue); 1332 1333 /** 1334 * qdio_start_irq - enable interrupt processing for the device 1335 * @cdev: associated ccw_device for the qdio subchannel 1336 * 1337 * Return codes 1338 * 0 - success 1339 * 1 - irqs not started since new data is available 1340 */ 1341 int qdio_start_irq(struct ccw_device *cdev) 1342 { 1343 struct qdio_q *q; 1344 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1345 unsigned int i; 1346 1347 if (!irq_ptr) 1348 return -ENODEV; 1349 1350 for_each_input_queue(irq_ptr, q, i) 1351 qdio_stop_polling(q); 1352 1353 clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state); 1354 1355 /* 1356 * We need to check again to not lose initiative after 1357 * resetting the ACK state. 1358 */ 1359 if (test_nonshared_ind(irq_ptr)) 1360 goto rescan; 1361 1362 for_each_input_queue(irq_ptr, q, i) { 1363 if (!qdio_inbound_q_done(q, q->first_to_check)) 1364 goto rescan; 1365 } 1366 1367 return 0; 1368 1369 rescan: 1370 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1371 return 0; 1372 else 1373 return 1; 1374 1375 } 1376 EXPORT_SYMBOL(qdio_start_irq); 1377 1378 /** 1379 * qdio_stop_irq - disable interrupt processing for the device 1380 * @cdev: associated ccw_device for the qdio subchannel 1381 * 1382 * Return codes 1383 * 0 - interrupts were already disabled 1384 * 1 - interrupts successfully disabled 1385 */ 1386 int qdio_stop_irq(struct ccw_device *cdev) 1387 { 1388 struct qdio_irq *irq_ptr = cdev->private->qdio_data; 1389 1390 if (!irq_ptr) 1391 return -ENODEV; 1392 1393 if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) 1394 return 0; 1395 else 1396 return 1; 1397 } 1398 EXPORT_SYMBOL(qdio_stop_irq); 1399 1400 static int __init init_QDIO(void) 1401 { 1402 int rc; 1403 1404 rc = qdio_debug_init(); 1405 if (rc) 1406 return rc; 1407 rc = qdio_setup_init(); 1408 if (rc) 1409 goto out_debug; 1410 rc = qdio_thinint_init(); 1411 if (rc) 1412 goto out_cache; 1413 return 0; 1414 1415 out_cache: 1416 qdio_setup_exit(); 1417 out_debug: 1418 qdio_debug_exit(); 1419 return rc; 1420 } 1421 1422 static void __exit exit_QDIO(void) 1423 { 1424 qdio_thinint_exit(); 1425 qdio_setup_exit(); 1426 qdio_debug_exit(); 1427 } 1428 1429 module_init(init_QDIO); 1430 module_exit(exit_QDIO); 1431