689 static inline void ata_set_tf_cdl(struct ata_queued_cmd *qc, int cdl)  in ata_set_tf_cdl()  argument
691 	struct ata_taskfile *tf = &qc->tf;  in ata_set_tf_cdl()
703 	qc->flags |= ATA_QCFLAG_HAS_CDL | ATA_QCFLAG_RESULT_TF;  in ata_set_tf_cdl()
726 int ata_build_rw_tf(struct ata_queued_cmd *qc, u64 block, u32 n_block,  in ata_build_rw_tf()  argument
729 	struct ata_taskfile *tf = &qc->tf;  in ata_build_rw_tf()
730 	struct ata_device *dev = qc->dev;  in ata_build_rw_tf()
748 		tf->nsect = qc->hw_tag << 3;  in ata_build_rw_tf()
768 			ata_set_tf_cdl(qc, cdl);  in ata_build_rw_tf()
774 			ata_set_tf_cdl(qc, cdl);  in ata_build_rw_tf()
778 		    !(qc->flags & ATA_QCFLAG_HAS_CDL) &&  in ata_build_rw_tf()
1493 static void ata_qc_complete_internal(struct ata_queued_cmd *qc)  in ata_qc_complete_internal()  argument
1495 	struct completion *waiting = qc->private_data;  in ata_qc_complete_internal()
1530 	struct ata_queued_cmd *qc;  in ata_exec_internal()  local
1554 	qc = __ata_qc_from_tag(ap, ATA_TAG_INTERNAL);  in ata_exec_internal()
1556 	qc->tag = ATA_TAG_INTERNAL;  in ata_exec_internal()
1557 	qc->hw_tag = 0;  in ata_exec_internal()
1558 	qc->scsicmd = NULL;  in ata_exec_internal()
1559 	qc->ap = ap;  in ata_exec_internal()
1560 	qc->dev = dev;  in ata_exec_internal()
1561 	ata_qc_reinit(qc);  in ata_exec_internal()
1573 	qc->tf = *tf;  in ata_exec_internal()
1575 		memcpy(qc->cdb, cdb, ATAPI_CDB_LEN);  in ata_exec_internal()
1580 		qc->tf.feature |= ATAPI_DMADIR;  in ata_exec_internal()
1582 	qc->flags |= ATA_QCFLAG_RESULT_TF;  in ata_exec_internal()
1583 	qc->dma_dir = dma_dir;  in ata_exec_internal()
1586 		ata_sg_init(qc, &sgl, 1);  in ata_exec_internal()
1587 		qc->nbytes = buflen;  in ata_exec_internal()
1590 	qc->private_data = &wait;  in ata_exec_internal()
1591 	qc->complete_fn = ata_qc_complete_internal;  in ata_exec_internal()
1593 	ata_qc_issue(qc);  in ata_exec_internal()
1621 		if (qc->flags & ATA_QCFLAG_ACTIVE) {  in ata_exec_internal()
1622 			qc->err_mask |= AC_ERR_TIMEOUT;  in ata_exec_internal()
1631 		ap->ops->post_internal_cmd(qc);  in ata_exec_internal()
1634 	if (qc->flags & ATA_QCFLAG_EH) {  in ata_exec_internal()
1635 		if (qc->result_tf.status & (ATA_ERR | ATA_DF))  in ata_exec_internal()
1636 			qc->err_mask |= AC_ERR_DEV;  in ata_exec_internal()
1638 		if (!qc->err_mask)  in ata_exec_internal()
1639 			qc->err_mask |= AC_ERR_OTHER;  in ata_exec_internal()
1641 		if (qc->err_mask & ~AC_ERR_OTHER)  in ata_exec_internal()
1642 			qc->err_mask &= ~AC_ERR_OTHER;  in ata_exec_internal()
1643 	} else if (qc->tf.command == ATA_CMD_REQ_SENSE_DATA) {  in ata_exec_internal()
1644 		qc->result_tf.status |= ATA_SENSE;  in ata_exec_internal()
1650 	*tf = qc->result_tf;  in ata_exec_internal()
1651 	err_mask = qc->err_mask;  in ata_exec_internal()
1653 	ata_qc_free(qc);  in ata_exec_internal()
4663 int atapi_check_dma(struct ata_queued_cmd *qc)  in atapi_check_dma()  argument
4665 	struct ata_port *ap = qc->ap;  in atapi_check_dma()
4670 	if (!(qc->dev->quirks & ATA_QUIRK_ATAPI_MOD16_DMA) &&  in atapi_check_dma()
4671 	    unlikely(qc->nbytes & 15))  in atapi_check_dma()
4675 		return ap->ops->check_atapi_dma(qc);  in atapi_check_dma()
4695 int ata_std_qc_defer(struct ata_queued_cmd *qc)  in ata_std_qc_defer()  argument
4697 	struct ata_link *link = qc->dev->link;  in ata_std_qc_defer()
4699 	if (ata_is_ncq(qc->tf.protocol)) {  in ata_std_qc_defer()
4724 void ata_sg_init(struct ata_queued_cmd *qc, struct scatterlist *sg,  in ata_sg_init()  argument
4727 	qc->sg = sg;  in ata_sg_init()
4728 	qc->n_elem = n_elem;  in ata_sg_init()
4729 	qc->cursg = qc->sg;  in ata_sg_init()
4743 static void ata_sg_clean(struct ata_queued_cmd *qc)  in ata_sg_clean()  argument
4745 	struct ata_port *ap = qc->ap;  in ata_sg_clean()
4746 	struct scatterlist *sg = qc->sg;  in ata_sg_clean()
4747 	int dir = qc->dma_dir;  in ata_sg_clean()
4751 	if (qc->n_elem)  in ata_sg_clean()
4752 		dma_unmap_sg(ap->dev, sg, qc->orig_n_elem, dir);  in ata_sg_clean()
4754 	qc->flags &= ~ATA_QCFLAG_DMAMAP;  in ata_sg_clean()
4755 	qc->sg = NULL;  in ata_sg_clean()
4771 static int ata_sg_setup(struct ata_queued_cmd *qc)  in ata_sg_setup()  argument
4773 	struct ata_port *ap = qc->ap;  in ata_sg_setup()
4776 	n_elem = dma_map_sg(ap->dev, qc->sg, qc->n_elem, qc->dma_dir);  in ata_sg_setup()
4780 	qc->orig_n_elem = qc->n_elem;  in ata_sg_setup()
4781 	qc->n_elem = n_elem;  in ata_sg_setup()
4782 	qc->flags |= ATA_QCFLAG_DMAMAP;  in ata_sg_setup()
4789 static inline void ata_sg_clean(struct ata_queued_cmd *qc) {}  in ata_sg_clean()  argument
4790 static inline int ata_sg_setup(struct ata_queued_cmd *qc) { return -1; }  in ata_sg_setup()  argument
4826 void ata_qc_free(struct ata_queued_cmd *qc)  in ata_qc_free()  argument
4828 	qc->flags = 0;  in ata_qc_free()
4829 	if (ata_tag_valid(qc->tag))  in ata_qc_free()
4830 		qc->tag = ATA_TAG_POISON;  in ata_qc_free()
4833 void __ata_qc_complete(struct ata_queued_cmd *qc)  in __ata_qc_complete()  argument
4838 	if (WARN_ON_ONCE(!(qc->flags & ATA_QCFLAG_ACTIVE)))  in __ata_qc_complete()
4841 	ap = qc->ap;  in __ata_qc_complete()
4842 	link = qc->dev->link;  in __ata_qc_complete()
4844 	if (likely(qc->flags & ATA_QCFLAG_DMAMAP))  in __ata_qc_complete()
4845 		ata_sg_clean(qc);  in __ata_qc_complete()
4848 	if (ata_is_ncq(qc->tf.protocol)) {  in __ata_qc_complete()
4849 		link->sactive &= ~(1 << qc->hw_tag);  in __ata_qc_complete()
4858 	if (unlikely(qc->flags & ATA_QCFLAG_CLEAR_EXCL &&  in __ata_qc_complete()
4867 	qc->flags &= ~ATA_QCFLAG_ACTIVE;  in __ata_qc_complete()
4868 	ap->qc_active &= ~(1ULL << qc->tag);  in __ata_qc_complete()
4871 	qc->complete_fn(qc);  in __ata_qc_complete()
4874 static void fill_result_tf(struct ata_queued_cmd *qc)  in fill_result_tf()  argument
4876 	struct ata_port *ap = qc->ap;  in fill_result_tf()
4882 	if (qc->flags & ATA_QCFLAG_RTF_FILLED)  in fill_result_tf()
4885 	qc->result_tf.flags = qc->tf.flags;  in fill_result_tf()
4886 	ap->ops->qc_fill_rtf(qc);  in fill_result_tf()
4887 	qc->flags |= ATA_QCFLAG_RTF_FILLED;  in fill_result_tf()
4890 static void ata_verify_xfer(struct ata_queued_cmd *qc)  in ata_verify_xfer()  argument
4892 	struct ata_device *dev = qc->dev;  in ata_verify_xfer()
4894 	if (!ata_is_data(qc->tf.protocol))  in ata_verify_xfer()
4897 	if ((dev->mwdma_mask || dev->udma_mask) && ata_is_pio(qc->tf.protocol))  in ata_verify_xfer()
4918 void ata_qc_complete(struct ata_queued_cmd *qc)  in ata_qc_complete()  argument
4920 	struct ata_port *ap = qc->ap;  in ata_qc_complete()
4921 	struct ata_device *dev = qc->dev;  in ata_qc_complete()
4925 	ledtrig_disk_activity(!!(qc->tf.flags & ATA_TFLAG_WRITE));  in ata_qc_complete()
4935 	if (unlikely(qc->err_mask))  in ata_qc_complete()
4936 		qc->flags |= ATA_QCFLAG_EH;  in ata_qc_complete()
4942 	if (unlikely(ata_tag_internal(qc->tag))) {  in ata_qc_complete()
4943 		fill_result_tf(qc);  in ata_qc_complete()
4944 		trace_ata_qc_complete_internal(qc);  in ata_qc_complete()
4945 		__ata_qc_complete(qc);  in ata_qc_complete()
4950 	if (unlikely(qc->flags & ATA_QCFLAG_EH)) {  in ata_qc_complete()
4951 		fill_result_tf(qc);  in ata_qc_complete()
4952 		trace_ata_qc_complete_failed(qc);  in ata_qc_complete()
4953 		ata_qc_schedule_eh(qc);  in ata_qc_complete()
4960 	if (qc->flags & ATA_QCFLAG_RESULT_TF)  in ata_qc_complete()
4961 		fill_result_tf(qc);  in ata_qc_complete()
4963 	trace_ata_qc_complete_done(qc);  in ata_qc_complete()
4971 	if (qc->flags & ATA_QCFLAG_HAS_CDL &&  in ata_qc_complete()
4972 	    qc->result_tf.status & ATA_SENSE) {  in ata_qc_complete()
4977 		qc->scsicmd->flags |= SCMD_FORCE_EH_SUCCESS;  in ata_qc_complete()
4978 		qc->flags |= ATA_QCFLAG_EH_SUCCESS_CMD;  in ata_qc_complete()
4986 		ata_qc_schedule_eh(qc);  in ata_qc_complete()
4991 	switch (qc->tf.command) {  in ata_qc_complete()
4993 		if (qc->tf.feature != SETFEATURES_WC_ON &&  in ata_qc_complete()
4994 		    qc->tf.feature != SETFEATURES_WC_OFF &&  in ata_qc_complete()
4995 		    qc->tf.feature != SETFEATURES_RA_ON &&  in ata_qc_complete()
4996 		    qc->tf.feature != SETFEATURES_RA_OFF)  in ata_qc_complete()
5012 		ata_verify_xfer(qc);  in ata_qc_complete()
5014 	__ata_qc_complete(qc);  in ata_qc_complete()
5054 void ata_qc_issue(struct ata_queued_cmd *qc)  in ata_qc_issue()  argument
5056 	struct ata_port *ap = qc->ap;  in ata_qc_issue()
5057 	struct ata_link *link = qc->dev->link;  in ata_qc_issue()
5058 	u8 prot = qc->tf.protocol;  in ata_qc_issue()
5064 		WARN_ON_ONCE(link->sactive & (1 << qc->hw_tag));  in ata_qc_issue()
5068 		link->sactive |= 1 << qc->hw_tag;  in ata_qc_issue()
5073 		link->active_tag = qc->tag;  in ata_qc_issue()
5076 	qc->flags |= ATA_QCFLAG_ACTIVE;  in ata_qc_issue()
5077 	ap->qc_active |= 1ULL << qc->tag;  in ata_qc_issue()
5083 	if (ata_is_data(prot) && (!qc->sg || !qc->n_elem || !qc->nbytes))  in ata_qc_issue()
5088 		if (ata_sg_setup(qc))  in ata_qc_issue()
5092 	if (unlikely(qc->dev->flags & ATA_DFLAG_SLEEPING)) {  in ata_qc_issue()
5100 		trace_ata_qc_prep(qc);  in ata_qc_issue()
5101 		qc->err_mask |= ap->ops->qc_prep(qc);  in ata_qc_issue()
5102 		if (unlikely(qc->err_mask))  in ata_qc_issue()
5106 	trace_ata_qc_issue(qc);  in ata_qc_issue()
5107 	qc->err_mask |= ap->ops->qc_issue(qc);  in ata_qc_issue()
5108 	if (unlikely(qc->err_mask))  in ata_qc_issue()
5113 	qc->err_mask |= AC_ERR_SYSTEM;  in ata_qc_issue()
5115 	ata_qc_complete(qc);  in ata_qc_issue()
6750 static unsigned int ata_dummy_qc_issue(struct ata_queued_cmd *qc)  in ata_dummy_qc_issue()  argument