Lines Matching +full:ats +full:- +full:supported
1 // SPDX-License-Identifier: GPL-2.0-only
8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
26 #include <linux/dma-mapping.h>
30 #include <linux/crc-itu-t.h>
41 * +---+---+---+---+---+---+---+---+
44 * +---+---+---+---+---+---+---+---+
46 * enqueued_ddcbs = A - N = 2 - 2 = 0
49 * +---+---+---+---+---+---+---+---+
52 * +---+---+---+---+---+---+---+---+
54 * enqueued_ddcbs = N - A = 4 - 2 = 2
57 * +---+---+---+---+---+---+---+---+
60 * +---+---+---+---+---+---+---+---+
62 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 2) = 6
65 * +---+---+---+---+---+---+---+---+
68 * +---+---+---+---+---+---+---+---+
71 * enqueued_ddcbs = N - A = 7 - 0 = 7
74 * +---+---+---+---+---+---+---+---+
77 * +---+---+---+---+---+---+---+---+
79 * enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
84 return queue->ddcb_next == queue->ddcb_act; in queue_empty()
89 if (queue->ddcb_next >= queue->ddcb_act) in queue_enqueued_ddcbs()
90 return queue->ddcb_next - queue->ddcb_act; in queue_enqueued_ddcbs()
92 return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next); in queue_enqueued_ddcbs()
97 int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1; in queue_free_ddcbs()
109 * pddcb->priv[6] = 0xcc; # cleared
112 * pddcb->priv[7] = 0xaa; # appended
115 * pddcb->priv[7] = 0xbb; # tapped
118 * pddcb->priv[6] = 0xff; # finished
123 pddcb->priv[7] = 0xbb; /* tapped */ in ddcb_mark_tapped()
128 pddcb->priv[7] = 0xaa; /* appended */ in ddcb_mark_appended()
133 pddcb->priv[6] = 0xcc; /* cleared */ in ddcb_mark_cleared()
138 pddcb->priv[6] = 0xff; /* finished */ in ddcb_mark_finished()
143 pddcb->priv_64 = cpu_to_be64(0); /* not tapped */ in ddcb_mark_unused()
147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
168 struct pci_dev *pci_dev = cd->pci_dev; in print_ddcb_info()
170 spin_lock_irqsave(&cd->print_lock, flags); in print_ddcb_info()
172 dev_info(&pci_dev->dev, in print_ddcb_info()
174 cd->card_idx, queue->ddcb_act, queue->ddcb_next); in print_ddcb_info()
176 pddcb = queue->ddcb_vaddr; in print_ddcb_info()
177 for (i = 0; i < queue->ddcb_max; i++) { in print_ddcb_info()
178 dev_err(&pci_dev->dev, in print_ddcb_info()
179 " %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n", in print_ddcb_info()
180 i == queue->ddcb_act ? '>' : ' ', in print_ddcb_info()
182 be16_to_cpu(pddcb->retc_16), in print_ddcb_info()
183 be16_to_cpu(pddcb->seqnum_16), in print_ddcb_info()
184 pddcb->hsi, in print_ddcb_info()
185 pddcb->shi, in print_ddcb_info()
186 be64_to_cpu(pddcb->priv_64), in print_ddcb_info()
187 pddcb->cmd); in print_ddcb_info()
190 spin_unlock_irqrestore(&cd->print_lock, flags); in print_ddcb_info()
201 return &req->cmd; in ddcb_requ_alloc()
213 return req->req_state; in ddcb_requ_get_state()
219 req->req_state = new_state; in ddcb_requ_set_state()
224 return req->cmd.ddata_addr != 0x0; in ddcb_requ_collect_debug_data()
228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
244 (cd->card_state != GENWQE_CARD_USED); in ddcb_requ_finished()
250 * enqueue_ddcb() - Enqueue a DDCB
282 prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1; in enqueue_ddcb()
283 prev_ddcb = &queue->ddcb_vaddr[prev_no]; in enqueue_ddcb()
292 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ in enqueue_ddcb()
301 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new); in enqueue_ddcb()
307 /* Queue must be re-started by updating QUEUE_OFFSET */ in enqueue_ddcb()
312 __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */ in enqueue_ddcb()
318 * copy_ddcb_results() - Copy output state from real DDCB to request
327 * - genwqe_purge_ddcb()
328 * - genwqe_check_ddcb_queue()
332 struct ddcb_queue *queue = req->queue; in copy_ddcb_results()
333 struct ddcb *pddcb = &queue->ddcb_vaddr[req->num]; in copy_ddcb_results()
335 memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH); in copy_ddcb_results()
338 req->cmd.vcrc = be16_to_cpu(pddcb->vcrc_16); in copy_ddcb_results()
339 req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64); in copy_ddcb_results()
340 req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64); in copy_ddcb_results()
342 req->cmd.attn = be16_to_cpu(pddcb->attn_16); in copy_ddcb_results()
343 req->cmd.progress = be32_to_cpu(pddcb->progress_32); in copy_ddcb_results()
344 req->cmd.retc = be16_to_cpu(pddcb->retc_16); in copy_ddcb_results()
348 queue->ddcb_max - 1 : ddcb_no - 1; in copy_ddcb_results()
349 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no]; in copy_ddcb_results()
351 memcpy(&req->debug_data.ddcb_finished, pddcb, in copy_ddcb_results()
352 sizeof(req->debug_data.ddcb_finished)); in copy_ddcb_results()
353 memcpy(&req->debug_data.ddcb_prev, prev_pddcb, in copy_ddcb_results()
354 sizeof(req->debug_data.ddcb_prev)); in copy_ddcb_results()
359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.
370 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_check_ddcb_queue()
372 spin_lock_irqsave(&queue->ddcb_lock, flags); in genwqe_check_ddcb_queue()
375 while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) { in genwqe_check_ddcb_queue()
381 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; in genwqe_check_ddcb_queue()
383 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == in genwqe_check_ddcb_queue()
390 req = queue->ddcb_req[queue->ddcb_act]; in genwqe_check_ddcb_queue()
405 retc_16 = be16_to_cpu(pddcb->retc_16); in genwqe_check_ddcb_queue()
406 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) { in genwqe_check_ddcb_queue()
408 u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr; in genwqe_check_ddcb_queue()
410 errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS); in genwqe_check_ddcb_queue()
411 status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); in genwqe_check_ddcb_queue()
413 dev_err(&pci_dev->dev, in genwqe_check_ddcb_queue()
415 __func__, be16_to_cpu(pddcb->seqnum_16), in genwqe_check_ddcb_queue()
416 pddcb->hsi, retc_16, errcnts, status, in genwqe_check_ddcb_queue()
417 queue->ddcb_daddr + ddcb_offs); in genwqe_check_ddcb_queue()
420 copy_ddcb_results(req, queue->ddcb_act); in genwqe_check_ddcb_queue()
421 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */ in genwqe_check_ddcb_queue()
423 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num); in genwqe_check_ddcb_queue()
429 vcrc = genwqe_crc16(pddcb->asv, in genwqe_check_ddcb_queue()
430 VCRC_LENGTH(req->cmd.asv_length), in genwqe_check_ddcb_queue()
432 vcrc_16 = be16_to_cpu(pddcb->vcrc_16); in genwqe_check_ddcb_queue()
436 GENWQE_DEVNAME, dev_name(&pci_dev->dev), in genwqe_check_ddcb_queue()
437 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length), in genwqe_check_ddcb_queue()
442 queue->ddcbs_completed++; in genwqe_check_ddcb_queue()
443 queue->ddcbs_in_flight--; in genwqe_check_ddcb_queue()
447 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); in genwqe_check_ddcb_queue()
448 wake_up_interruptible(&queue->busy_waitq); in genwqe_check_ddcb_queue()
451 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max; in genwqe_check_ddcb_queue()
456 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in genwqe_check_ddcb_queue()
469 * -ETIMEDOUT when timeout
470 * -ERESTARTSYS when ^C
471 * -EINVAL when unknown error condition
482 struct pci_dev *pci_dev = cd->pci_dev; in __genwqe_wait_ddcb()
485 return -EINVAL; in __genwqe_wait_ddcb()
487 queue = req->queue; in __genwqe_wait_ddcb()
489 return -EINVAL; in __genwqe_wait_ddcb()
491 ddcb_no = req->num; in __genwqe_wait_ddcb()
492 if (ddcb_no >= queue->ddcb_max) in __genwqe_wait_ddcb()
493 return -EINVAL; in __genwqe_wait_ddcb()
495 rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no], in __genwqe_wait_ddcb()
502 * 2. rc == -ERESTARTSYS signal received in __genwqe_wait_ddcb()
506 struct ddcb_queue *queue = req->queue; in __genwqe_wait_ddcb()
514 genwqe_check_ddcb_queue(cd, req->queue); in __genwqe_wait_ddcb()
518 dev_err(&pci_dev->dev, in __genwqe_wait_ddcb()
520 __func__, req->num, rc, ddcb_requ_get_state(req), in __genwqe_wait_ddcb()
522 dev_err(&pci_dev->dev, in __genwqe_wait_ddcb()
524 __genwqe_readq(cd, queue->IO_QUEUE_STATUS)); in __genwqe_wait_ddcb()
526 pddcb = &queue->ddcb_vaddr[req->num]; in __genwqe_wait_ddcb()
529 print_ddcb_info(cd, req->queue); in __genwqe_wait_ddcb()
530 return -ETIMEDOUT; in __genwqe_wait_ddcb()
532 } else if (rc == -ERESTARTSYS) { in __genwqe_wait_ddcb()
540 dev_err(&pci_dev->dev, in __genwqe_wait_ddcb()
542 __func__, req->num, rc, ddcb_requ_get_state(req)); in __genwqe_wait_ddcb()
543 return -EINVAL; in __genwqe_wait_ddcb()
547 if (cd->card_state != GENWQE_CARD_USED) { in __genwqe_wait_ddcb()
548 dev_err(&pci_dev->dev, in __genwqe_wait_ddcb()
550 __func__, req->num, rc); in __genwqe_wait_ddcb()
551 return -EIO; in __genwqe_wait_ddcb()
557 * get_next_ddcb() - Get next available DDCB
578 pddcb = &queue->ddcb_vaddr[queue->ddcb_next]; in get_next_ddcb()
582 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000) in get_next_ddcb()
585 *num = queue->ddcb_next; /* internal DDCB number */ in get_next_ddcb()
586 queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max; in get_next_ddcb()
600 pddcb->pre = DDCB_PRESET_PRE; /* 128 */ in get_next_ddcb()
601 pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++); in get_next_ddcb()
606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
625 struct ddcb_queue *queue = req->queue; in __genwqe_purge_ddcb()
626 struct pci_dev *pci_dev = cd->pci_dev; in __genwqe_purge_ddcb()
633 dev_err(&pci_dev->dev, in __genwqe_purge_ddcb()
635 return -EFAULT; in __genwqe_purge_ddcb()
638 pddcb = &queue->ddcb_vaddr[req->num]; in __genwqe_purge_ddcb()
642 spin_lock_irqsave(&queue->ddcb_lock, flags); in __genwqe_purge_ddcb()
649 old = pddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */ in __genwqe_purge_ddcb()
653 icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32, in __genwqe_purge_ddcb()
661 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; in __genwqe_purge_ddcb()
665 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in __genwqe_purge_ddcb()
674 copy_ddcb_results(req, req->num); /* for the failing case */ in __genwqe_purge_ddcb()
679 copy_ddcb_results(req, req->num); in __genwqe_purge_ddcb()
681 queue->ddcbs_in_flight--; in __genwqe_purge_ddcb()
682 queue->ddcb_req[req->num] = NULL; /* delete from array */ in __genwqe_purge_ddcb()
695 icrc_hsi_shi = pddcb->icrc_hsi_shi_32; in __genwqe_purge_ddcb()
697 (queue->ddcb_act == req->num)) { in __genwqe_purge_ddcb()
698 queue->ddcb_act = ((queue->ddcb_act + 1) % in __genwqe_purge_ddcb()
699 queue->ddcb_max); in __genwqe_purge_ddcb()
702 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in __genwqe_purge_ddcb()
710 queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS); in __genwqe_purge_ddcb()
712 dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num); in __genwqe_purge_ddcb()
715 dev_err(&pci_dev->dev, in __genwqe_purge_ddcb()
717 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT, in __genwqe_purge_ddcb()
720 print_ddcb_info(cd, req->queue); in __genwqe_purge_ddcb()
722 return -EFAULT; in __genwqe_purge_ddcb()
728 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_init_debug_data()
731 dev_err(&pci_dev->dev, in genwqe_init_debug_data()
734 return -EFAULT; in genwqe_init_debug_data()
737 len = sizeof(d->driver_version); in genwqe_init_debug_data()
738 snprintf(d->driver_version, len, "%s", DRV_VERSION); in genwqe_init_debug_data()
739 d->slu_unitcfg = cd->slu_unitcfg; in genwqe_init_debug_data()
740 d->app_unitcfg = cd->app_unitcfg; in genwqe_init_debug_data()
745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
748 * @f_flags: file mode: blocking, non-blocking
751 * -EIO if card is unusable/PCIe problems
752 * -EBUSY if enqueuing failed
760 struct pci_dev *pci_dev = cd->pci_dev; in __genwqe_enqueue_ddcb()
764 if (cd->card_state != GENWQE_CARD_USED) { in __genwqe_enqueue_ddcb()
767 GENWQE_DEVNAME, dev_name(&pci_dev->dev), in __genwqe_enqueue_ddcb()
768 __func__, req->num); in __genwqe_enqueue_ddcb()
769 return -EIO; in __genwqe_enqueue_ddcb()
772 queue = req->queue = &cd->queue; in __genwqe_enqueue_ddcb()
785 spin_lock_irqsave(&queue->ddcb_lock, flags); in __genwqe_enqueue_ddcb()
787 pddcb = get_next_ddcb(cd, queue, &req->num); /* get ptr and num */ in __genwqe_enqueue_ddcb()
791 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in __genwqe_enqueue_ddcb()
794 queue->return_on_busy++; in __genwqe_enqueue_ddcb()
795 return -EBUSY; in __genwqe_enqueue_ddcb()
798 queue->wait_on_busy++; in __genwqe_enqueue_ddcb()
799 rc = wait_event_interruptible(queue->busy_waitq, in __genwqe_enqueue_ddcb()
801 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n", in __genwqe_enqueue_ddcb()
803 if (rc == -ERESTARTSYS) in __genwqe_enqueue_ddcb()
809 if (queue->ddcb_req[req->num] != NULL) { in __genwqe_enqueue_ddcb()
810 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in __genwqe_enqueue_ddcb()
812 dev_err(&pci_dev->dev, in __genwqe_enqueue_ddcb()
814 __func__, req->num, req); in __genwqe_enqueue_ddcb()
815 return -EFAULT; in __genwqe_enqueue_ddcb()
818 queue->ddcb_req[req->num] = req; in __genwqe_enqueue_ddcb()
820 pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts); in __genwqe_enqueue_ddcb()
821 pddcb->cmd = req->cmd.cmd; in __genwqe_enqueue_ddcb()
822 pddcb->acfunc = req->cmd.acfunc; /* functional unit */ in __genwqe_enqueue_ddcb()
832 if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull) in __genwqe_enqueue_ddcb()
833 pddcb->xdir = 0x1; in __genwqe_enqueue_ddcb()
835 pddcb->xdir = 0x0; in __genwqe_enqueue_ddcb()
838 pddcb->psp = (((req->cmd.asiv_length / 8) << 4) | in __genwqe_enqueue_ddcb()
839 ((req->cmd.asv_length / 8))); in __genwqe_enqueue_ddcb()
840 pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts); in __genwqe_enqueue_ddcb()
845 * req->cmd.asiv_length. But simulation benefits from some in __genwqe_enqueue_ddcb()
846 * non-architectured bits behind the architectured content. in __genwqe_enqueue_ddcb()
849 * ATS field, which was introduced late. If the ATS field is in __genwqe_enqueue_ddcb()
850 * supported ASIV is 8 bytes shorter than it used to be. Since in __genwqe_enqueue_ddcb()
851 * the ATS field is copied too, the code should do exactly in __genwqe_enqueue_ddcb()
852 * what it did before, but I wanted to make copying of the ATS in __genwqe_enqueue_ddcb()
856 memcpy(&pddcb->__asiv[0], /* destination */ in __genwqe_enqueue_ddcb()
857 &req->cmd.__asiv[0], /* source */ in __genwqe_enqueue_ddcb()
858 DDCB_ASIV_LENGTH); /* req->cmd.asiv_length */ in __genwqe_enqueue_ddcb()
860 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats); in __genwqe_enqueue_ddcb()
861 memcpy(&pddcb->n.asiv[0], /* destination */ in __genwqe_enqueue_ddcb()
862 &req->cmd.asiv[0], /* source */ in __genwqe_enqueue_ddcb()
863 DDCB_ASIV_LENGTH_ATS); /* req->cmd.asiv_length */ in __genwqe_enqueue_ddcb()
866 pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */ in __genwqe_enqueue_ddcb()
873 ICRC_LENGTH(req->cmd.asiv_length), 0xffff); in __genwqe_enqueue_ddcb()
874 pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16); in __genwqe_enqueue_ddcb()
878 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32; in __genwqe_enqueue_ddcb()
880 dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num); in __genwqe_enqueue_ddcb()
887 genwqe_init_debug_data(cd, &req->debug_data); in __genwqe_enqueue_ddcb()
888 memcpy(&req->debug_data.ddcb_before, pddcb, in __genwqe_enqueue_ddcb()
889 sizeof(req->debug_data.ddcb_before)); in __genwqe_enqueue_ddcb()
892 enqueue_ddcb(cd, queue, pddcb, req->num); in __genwqe_enqueue_ddcb()
893 queue->ddcbs_in_flight++; in __genwqe_enqueue_ddcb()
895 if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight) in __genwqe_enqueue_ddcb()
896 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight; in __genwqe_enqueue_ddcb()
899 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in __genwqe_enqueue_ddcb()
900 wake_up_interruptible(&cd->queue_waitq); in __genwqe_enqueue_ddcb()
906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
909 * @f_flags: file mode: blocking, non-blocking
916 struct pci_dev *pci_dev = cd->pci_dev; in __genwqe_execute_raw_ddcb()
919 if (cmd->asiv_length > DDCB_ASIV_LENGTH) { in __genwqe_execute_raw_ddcb()
920 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n", in __genwqe_execute_raw_ddcb()
921 __func__, cmd->asiv_length); in __genwqe_execute_raw_ddcb()
922 return -EINVAL; in __genwqe_execute_raw_ddcb()
924 if (cmd->asv_length > DDCB_ASV_LENGTH) { in __genwqe_execute_raw_ddcb()
925 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n", in __genwqe_execute_raw_ddcb()
926 __func__, cmd->asiv_length); in __genwqe_execute_raw_ddcb()
927 return -EINVAL; in __genwqe_execute_raw_ddcb()
939 (unsigned long)cmd->ddata_addr, in __genwqe_execute_raw_ddcb()
940 &req->debug_data, in __genwqe_execute_raw_ddcb()
942 return -EFAULT; in __genwqe_execute_raw_ddcb()
950 if (cmd->retc != DDCB_RETC_COMPLETE) { in __genwqe_execute_raw_ddcb()
953 rc = -EBADMSG; /* not processed/error retc */ in __genwqe_execute_raw_ddcb()
963 (unsigned long)cmd->ddata_addr, in __genwqe_execute_raw_ddcb()
964 &req->debug_data, in __genwqe_execute_raw_ddcb()
966 return -EFAULT; in __genwqe_execute_raw_ddcb()
972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
975 * We use this as condition for our wait-queue code.
981 struct ddcb_queue *queue = &cd->queue; in genwqe_next_ddcb_ready()
983 spin_lock_irqsave(&queue->ddcb_lock, flags); in genwqe_next_ddcb_ready()
986 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in genwqe_next_ddcb_ready()
990 pddcb = &queue->ddcb_vaddr[queue->ddcb_act]; in genwqe_next_ddcb_ready()
991 if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */ in genwqe_next_ddcb_ready()
992 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in genwqe_next_ddcb_ready()
996 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in genwqe_next_ddcb_ready()
1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1012 struct ddcb_queue *queue = &cd->queue; in genwqe_ddcbs_in_flight()
1014 spin_lock_irqsave(&queue->ddcb_lock, flags); in genwqe_ddcbs_in_flight()
1015 ddcbs_in_flight += queue->ddcbs_in_flight; in genwqe_ddcbs_in_flight()
1016 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in genwqe_ddcbs_in_flight()
1027 struct pci_dev *pci_dev = cd->pci_dev; in setup_ddcb_queue()
1030 return -EINVAL; in setup_ddcb_queue()
1034 queue->ddcbs_in_flight = 0; /* statistics */ in setup_ddcb_queue()
1035 queue->ddcbs_max_in_flight = 0; in setup_ddcb_queue()
1036 queue->ddcbs_completed = 0; in setup_ddcb_queue()
1037 queue->return_on_busy = 0; in setup_ddcb_queue()
1038 queue->wait_on_busy = 0; in setup_ddcb_queue()
1040 queue->ddcb_seq = 0x100; /* start sequence number */ in setup_ddcb_queue()
1041 queue->ddcb_max = GENWQE_DDCB_MAX; in setup_ddcb_queue()
1042 queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size, in setup_ddcb_queue()
1043 &queue->ddcb_daddr); in setup_ddcb_queue()
1044 if (queue->ddcb_vaddr == NULL) { in setup_ddcb_queue()
1045 dev_err(&pci_dev->dev, in setup_ddcb_queue()
1047 return -ENOMEM; in setup_ddcb_queue()
1049 queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *), in setup_ddcb_queue()
1051 if (!queue->ddcb_req) { in setup_ddcb_queue()
1052 rc = -ENOMEM; in setup_ddcb_queue()
1056 queue->ddcb_waitqs = kcalloc(queue->ddcb_max, in setup_ddcb_queue()
1059 if (!queue->ddcb_waitqs) { in setup_ddcb_queue()
1060 rc = -ENOMEM; in setup_ddcb_queue()
1064 for (i = 0; i < queue->ddcb_max; i++) { in setup_ddcb_queue()
1065 pddcb = &queue->ddcb_vaddr[i]; /* DDCBs */ in setup_ddcb_queue()
1066 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32; in setup_ddcb_queue()
1067 pddcb->retc_16 = cpu_to_be16(0xfff); in setup_ddcb_queue()
1069 queue->ddcb_req[i] = NULL; /* requests */ in setup_ddcb_queue()
1070 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */ in setup_ddcb_queue()
1073 queue->ddcb_act = 0; in setup_ddcb_queue()
1074 queue->ddcb_next = 0; /* queue is empty */ in setup_ddcb_queue()
1076 spin_lock_init(&queue->ddcb_lock); in setup_ddcb_queue()
1077 init_waitqueue_head(&queue->busy_waitq); in setup_ddcb_queue()
1079 val64 = ((u64)(queue->ddcb_max - 1) << 8); /* lastptr */ in setup_ddcb_queue()
1080 __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG, 0x07); /* iCRC/vCRC */ in setup_ddcb_queue()
1081 __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr); in setup_ddcb_queue()
1082 __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq); in setup_ddcb_queue()
1083 __genwqe_writeq(cd, queue->IO_QUEUE_WRAP, val64); in setup_ddcb_queue()
1087 kfree(queue->ddcb_req); in setup_ddcb_queue()
1088 queue->ddcb_req = NULL; in setup_ddcb_queue()
1090 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, in setup_ddcb_queue()
1091 queue->ddcb_daddr); in setup_ddcb_queue()
1092 queue->ddcb_vaddr = NULL; in setup_ddcb_queue()
1093 queue->ddcb_daddr = 0ull; in setup_ddcb_queue()
1100 return queue->ddcb_vaddr != NULL; in ddcb_queue_initialized()
1107 queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE); in free_ddcb_queue()
1109 kfree(queue->ddcb_req); in free_ddcb_queue()
1110 queue->ddcb_req = NULL; in free_ddcb_queue()
1112 if (queue->ddcb_vaddr) { in free_ddcb_queue()
1113 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr, in free_ddcb_queue()
1114 queue->ddcb_daddr); in free_ddcb_queue()
1115 queue->ddcb_vaddr = NULL; in free_ddcb_queue()
1116 queue->ddcb_daddr = 0ull; in free_ddcb_queue()
1124 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_pf_isr()
1130 cd->irqs_processed++; in genwqe_pf_isr()
1131 wake_up_interruptible(&cd->queue_waitq); in genwqe_pf_isr()
1135 * safer, but slower for the good-case ... See above. in genwqe_pf_isr()
1141 if (cd->use_platform_recovery) { in genwqe_pf_isr()
1144 * detected by the platform until we do a non-raw in genwqe_pf_isr()
1147 readq(cd->mmio + IO_SLC_CFGREG_GFIR); in genwqe_pf_isr()
1154 wake_up_interruptible(&cd->health_waitq); in genwqe_pf_isr()
1160 dev_err_ratelimited(&pci_dev->dev, in genwqe_pf_isr()
1173 cd->irqs_processed++; in genwqe_vf_isr()
1174 wake_up_interruptible(&cd->queue_waitq); in genwqe_vf_isr()
1180 * genwqe_card_thread() - Work thread for the DDCB queue
1195 genwqe_check_ddcb_queue(cd, &cd->queue); in genwqe_card_thread()
1199 cd->queue_waitq, in genwqe_card_thread()
1204 cd->queue_waitq, in genwqe_card_thread()
1221 * genwqe_setup_service_layer() - Setup DDCB queue
1232 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_setup_service_layer()
1237 dev_err(&pci_dev->dev, in genwqe_setup_service_layer()
1244 queue = &cd->queue; in genwqe_setup_service_layer()
1245 queue->IO_QUEUE_CONFIG = IO_SLC_QUEUE_CONFIG; in genwqe_setup_service_layer()
1246 queue->IO_QUEUE_STATUS = IO_SLC_QUEUE_STATUS; in genwqe_setup_service_layer()
1247 queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT; in genwqe_setup_service_layer()
1248 queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN; in genwqe_setup_service_layer()
1249 queue->IO_QUEUE_OFFSET = IO_SLC_QUEUE_OFFSET; in genwqe_setup_service_layer()
1250 queue->IO_QUEUE_WRAP = IO_SLC_QUEUE_WRAP; in genwqe_setup_service_layer()
1251 queue->IO_QUEUE_WTIME = IO_SLC_QUEUE_WTIME; in genwqe_setup_service_layer()
1252 queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS; in genwqe_setup_service_layer()
1253 queue->IO_QUEUE_LRW = IO_SLC_QUEUE_LRW; in genwqe_setup_service_layer()
1257 rc = -ENODEV; in genwqe_setup_service_layer()
1261 init_waitqueue_head(&cd->queue_waitq); in genwqe_setup_service_layer()
1262 cd->card_thread = kthread_run(genwqe_card_thread, cd, in genwqe_setup_service_layer()
1264 cd->card_idx); in genwqe_setup_service_layer()
1265 if (IS_ERR(cd->card_thread)) { in genwqe_setup_service_layer()
1266 rc = PTR_ERR(cd->card_thread); in genwqe_setup_service_layer()
1267 cd->card_thread = NULL; in genwqe_setup_service_layer()
1276 * We must have all wait-queues initialized when we enable the in genwqe_setup_service_layer()
1280 init_waitqueue_head(&cd->health_waitq); in genwqe_setup_service_layer()
1283 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED, in genwqe_setup_service_layer()
1286 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED, in genwqe_setup_service_layer()
1290 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq); in genwqe_setup_service_layer()
1294 cd->card_state = GENWQE_CARD_USED; in genwqe_setup_service_layer()
1300 kthread_stop(cd->card_thread); in genwqe_setup_service_layer()
1301 cd->card_thread = NULL; in genwqe_setup_service_layer()
1309 * queue_wake_up_all() - Handles fatal error case
1320 struct ddcb_queue *queue = &cd->queue; in queue_wake_up_all()
1322 spin_lock_irqsave(&queue->ddcb_lock, flags); in queue_wake_up_all()
1324 for (i = 0; i < queue->ddcb_max; i++) in queue_wake_up_all()
1325 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]); in queue_wake_up_all()
1327 wake_up_interruptible(&queue->busy_waitq); in queue_wake_up_all()
1328 spin_unlock_irqrestore(&queue->ddcb_lock, flags); in queue_wake_up_all()
1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1337 * Relies on the pre-condition that there are no users of the card
1338 * device anymore e.g. with open file-descriptors.
1346 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_finish_queue()
1347 struct ddcb_queue *queue = &cd->queue; in genwqe_finish_queue()
1353 if (cd->card_state == GENWQE_CARD_USED) in genwqe_finish_queue()
1354 cd->card_state = GENWQE_CARD_UNUSED; in genwqe_finish_queue()
1367 dev_dbg(&pci_dev->dev, in genwqe_finish_queue()
1381 dev_err(&pci_dev->dev, " [%s] err: queue is not empty!!\n", in genwqe_finish_queue()
1383 rc = -EIO; in genwqe_finish_queue()
1389 * genwqe_release_service_layer() - Shutdown DDCB queue
1396 struct pci_dev *pci_dev = cd->pci_dev; in genwqe_release_service_layer()
1398 if (!ddcb_queue_initialized(&cd->queue)) in genwqe_release_service_layer()
1401 free_irq(pci_dev->irq, cd); in genwqe_release_service_layer()
1404 if (cd->card_thread != NULL) { in genwqe_release_service_layer()
1405 kthread_stop(cd->card_thread); in genwqe_release_service_layer()
1406 cd->card_thread = NULL; in genwqe_release_service_layer()
1409 free_ddcb_queue(cd, &cd->queue); in genwqe_release_service_layer()