1 // SPDX-License-Identifier: GPL-2.0-or-later
3  *  libahci.c - Common AHCI SATA low-level routines
6  *    		    Please ALWAYS copy linux-ide@vger.kernel.org
9  *  Copyright 2004-2005 Red Hat, Inc.
12  * as Documentation/driver-api/libata.rst
27 #include <linux/dma-mapping.h>
237  *	ahci_rpm_get_port - Make sure the port is powered on
238  *	@ap: Port to power on
246 	return pm_runtime_get_sync(ap->dev);  in ahci_rpm_get_port()
250  *	ahci_rpm_put_port - Undoes ahci_rpm_get_port()
251  *	@ap: Port to power down
258 	pm_runtime_put(ap->dev);  in ahci_rpm_put_port()
266 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_show_host_caps()
268 	return sprintf(buf, "%x\n", hpriv->cap);  in ahci_show_host_caps()
276 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_show_host_cap2()
278 	return sprintf(buf, "%x\n", hpriv->cap2);  in ahci_show_host_cap2()
286 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_show_host_version()
288 	return sprintf(buf, "%x\n", hpriv->version);  in ahci_show_host_version()
311 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_read_em_buffer()
312 	void __iomem *mmio = hpriv->mmio;  in ahci_read_em_buffer()
313 	void __iomem *em_mmio = mmio + hpriv->em_loc;  in ahci_read_em_buffer()
320 	spin_lock_irqsave(ap->lock, flags);  in ahci_read_em_buffer()
323 	if (!(ap->flags & ATA_FLAG_EM) || em_ctl & EM_CTL_XMT ||  in ahci_read_em_buffer()
324 	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {  in ahci_read_em_buffer()
325 		spin_unlock_irqrestore(ap->lock, flags);  in ahci_read_em_buffer()
327 		return -EINVAL;  in ahci_read_em_buffer()
331 		spin_unlock_irqrestore(ap->lock, flags);  in ahci_read_em_buffer()
333 		return -EAGAIN;  in ahci_read_em_buffer()
337 		em_mmio += hpriv->em_buf_sz;  in ahci_read_em_buffer()
339 	count = hpriv->em_buf_sz;  in ahci_read_em_buffer()
347 				      hpriv->em_buf_sz, PAGE_SIZE);  in ahci_read_em_buffer()
359 	spin_unlock_irqrestore(ap->lock, flags);  in ahci_read_em_buffer()
371 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_store_em_buffer()
372 	void __iomem *mmio = hpriv->mmio;  in ahci_store_em_buffer()
373 	void __iomem *em_mmio = mmio + hpriv->em_loc;  in ahci_store_em_buffer()
380 	if (!(ap->flags & ATA_FLAG_EM) ||  in ahci_store_em_buffer()
381 	    !(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) ||  in ahci_store_em_buffer()
382 	    size % 4 || size > hpriv->em_buf_sz)  in ahci_store_em_buffer()
383 		return -EINVAL;  in ahci_store_em_buffer()
386 	spin_lock_irqsave(ap->lock, flags);  in ahci_store_em_buffer()
390 		spin_unlock_irqrestore(ap->lock, flags);  in ahci_store_em_buffer()
392 		return -EBUSY;  in ahci_store_em_buffer()
403 	spin_unlock_irqrestore(ap->lock, flags);  in ahci_store_em_buffer()
414 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_show_em_supported()
415 	void __iomem *mmio = hpriv->mmio;  in ahci_show_em_supported()
424 		       em_ctl & EM_CTL_SAFTE ? "saf-te " : "",  in ahci_show_em_supported()
425 		       em_ctl & EM_CTL_SES ? "ses-2 " : "",  in ahci_show_em_supported()
430  *	ahci_save_initial_config - Save and fixup initial config values
441  *	If it is not set already this function sets hpriv->start_engine to
449 	void __iomem *mmio = hpriv->mmio;  in ahci_save_initial_config()
452 	u32 cap, cap2, vers;  in ahci_save_initial_config()  local
455 	/* make sure AHCI mode is enabled before accessing CAP */  in ahci_save_initial_config()
459 	 * Values prefixed with saved_ are written back to the HBA and ports  in ahci_save_initial_config()
464 	 * Override HW-init HBA capability fields with the platform-specific  in ahci_save_initial_config()
465 	 * values. The rest of the HBA capabilities are defined as Read-only  in ahci_save_initial_config()
468 	cap = readl(mmio + HOST_CAP);  in ahci_save_initial_config()
469 	if (hpriv->saved_cap)  in ahci_save_initial_config()
470 		cap = (cap & ~(HOST_CAP_SSS | HOST_CAP_MPS)) | hpriv->saved_cap;  in ahci_save_initial_config()
471 	hpriv->saved_cap = cap;  in ahci_save_initial_config()
477 		hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);  in ahci_save_initial_config()
479 		hpriv->saved_cap2 = cap2 = 0;  in ahci_save_initial_config()
482 	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {  in ahci_save_initial_config()
484 		cap &= ~HOST_CAP_64;  in ahci_save_initial_config()
487 	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {  in ahci_save_initial_config()
489 		cap &= ~HOST_CAP_NCQ;  in ahci_save_initial_config()
492 	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {  in ahci_save_initial_config()
494 		cap |= HOST_CAP_NCQ;  in ahci_save_initial_config()
497 	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {  in ahci_save_initial_config()
499 		cap &= ~HOST_CAP_PMP;  in ahci_save_initial_config()
502 	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {  in ahci_save_initial_config()
505 		cap &= ~HOST_CAP_SNTF;  in ahci_save_initial_config()
508 	if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {  in ahci_save_initial_config()
515 	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {  in ahci_save_initial_config()
517 		cap |= HOST_CAP_FBS;  in ahci_save_initial_config()
520 	if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {  in ahci_save_initial_config()
522 		cap &= ~HOST_CAP_FBS;  in ahci_save_initial_config()
525 	if (!(cap & HOST_CAP_ALPM) && (hpriv->flags & AHCI_HFLAG_YES_ALPM)) {  in ahci_save_initial_config()
527 		cap |= HOST_CAP_ALPM;  in ahci_save_initial_config()
530 	if ((cap & HOST_CAP_SXS) && (hpriv->flags & AHCI_HFLAG_NO_SXS)) {  in ahci_save_initial_config()
532 		cap &= ~HOST_CAP_SXS;  in ahci_save_initial_config()
535 	/* Override the HBA ports mapping if the platform needs it */  in ahci_save_initial_config()
537 	if (hpriv->saved_port_map && port_map != hpriv->saved_port_map) {  in ahci_save_initial_config()
538 		dev_info(dev, "forcing port_map 0x%lx -> 0x%x\n",  in ahci_save_initial_config()
539 			 port_map, hpriv->saved_port_map);  in ahci_save_initial_config()
540 		port_map = hpriv->saved_port_map;  in ahci_save_initial_config()
542 		hpriv->saved_port_map = port_map;  in ahci_save_initial_config()
546 	if (hpriv->mask_port_map) {  in ahci_save_initial_config()
547 		dev_warn(dev, "masking port_map 0x%lx -> 0x%lx\n",  in ahci_save_initial_config()
549 			port_map & hpriv->mask_port_map);  in ahci_save_initial_config()
550 		port_map &= hpriv->mask_port_map;  in ahci_save_initial_config()
553 	/* cross check port_map and cap.n_ports */  in ahci_save_initial_config()
564 		if (map_ports > ahci_nr_ports(cap)) {  in ahci_save_initial_config()
566 				 "implemented port map (0x%lx) contains more ports than nr_ports (%u), using nr_ports\n",  in ahci_save_initial_config()
567 				 port_map, ahci_nr_ports(cap));  in ahci_save_initial_config()
572 	/* fabricate port_map from cap.nr_ports for < AHCI 1.3 */  in ahci_save_initial_config()
574 		port_map = (1 << ahci_nr_ports(cap)) - 1;  in ahci_save_initial_config()
578 		hpriv->saved_port_map = port_map;  in ahci_save_initial_config()
587 		if (hpriv->saved_port_cap[i])  in ahci_save_initial_config()
591 		hpriv->saved_port_cap[i] =  in ahci_save_initial_config()
596 	hpriv->cap = cap;  in ahci_save_initial_config()
597 	hpriv->cap2 = cap2;  in ahci_save_initial_config()
598 	hpriv->version = vers;  in ahci_save_initial_config()
599 	hpriv->port_map = port_map;  in ahci_save_initial_config()
601 	if (!hpriv->start_engine)  in ahci_save_initial_config()
602 		hpriv->start_engine = ahci_start_engine;  in ahci_save_initial_config()
604 	if (!hpriv->stop_engine)  in ahci_save_initial_config()
605 		hpriv->stop_engine = ahci_stop_engine;  in ahci_save_initial_config()
607 	if (!hpriv->irq_handler)  in ahci_save_initial_config()
608 		hpriv->irq_handler = ahci_single_level_irq_intr;  in ahci_save_initial_config()
613  *	ahci_restore_initial_config - Restore initial config
623 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_restore_initial_config()
624 	unsigned long port_map = hpriv->port_map;  in ahci_restore_initial_config()
625 	void __iomem *mmio = hpriv->mmio;  in ahci_restore_initial_config()
629 	writel(hpriv->saved_cap, mmio + HOST_CAP);  in ahci_restore_initial_config()
630 	if (hpriv->saved_cap2)  in ahci_restore_initial_config()
631 		writel(hpriv->saved_cap2, mmio + HOST_CAP2);  in ahci_restore_initial_config()
632 	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);  in ahci_restore_initial_config()
637 		writel(hpriv->saved_port_cap[i], port_mmio + PORT_CMD);  in ahci_restore_initial_config()
650 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_scr_offset()
653 	    (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))  in ahci_scr_offset()
660 	void __iomem *port_mmio = ahci_port_base(link->ap);  in ahci_scr_read()
661 	int offset = ahci_scr_offset(link->ap, sc_reg);  in ahci_scr_read()
667 	return -EINVAL;  in ahci_scr_read()
672 	void __iomem *port_mmio = ahci_port_base(link->ap);  in ahci_scr_write()
673 	int offset = ahci_scr_offset(link->ap, sc_reg);  in ahci_scr_write()
679 	return -EINVAL;  in ahci_scr_write()
698 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_stop_engine()
702 	 * On some controllers, stopping a port's DMA engine while the port  in ahci_stop_engine()
705 	 * port back in active state before stopping its DMA engine.  in ahci_stop_engine()
707 	if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&  in ahci_stop_engine()
708 	    (ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&  in ahci_stop_engine()
709 	    ahci_set_lpm(&ap->link, ATA_LPM_MAX_POWER, ATA_LPM_WAKE_ONLY)) {  in ahci_stop_engine()
710 		dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");  in ahci_stop_engine()
711 		return -EIO;  in ahci_stop_engine()
716 	/* check if the HBA is idle */  in ahci_stop_engine()
723 	 * unplugging). Otherwise a 500ms delay for each port is added.  in ahci_stop_engine()
726 		dev_err(ap->host->dev, "AHCI controller unavailable!\n");  in ahci_stop_engine()
727 		return -ENODEV;  in ahci_stop_engine()
730 	/* setting HBA to idle */  in ahci_stop_engine()
738 		return -EIO;  in ahci_stop_engine()
747 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_start_fis_rx()
748 	struct ahci_port_priv *pp = ap->private_data;  in ahci_start_fis_rx()
752 	if (hpriv->cap & HOST_CAP_64)  in ahci_start_fis_rx()
753 		writel((pp->cmd_slot_dma >> 16) >> 16,  in ahci_start_fis_rx()
755 	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);  in ahci_start_fis_rx()
757 	if (hpriv->cap & HOST_CAP_64)  in ahci_start_fis_rx()
758 		writel((pp->rx_fis_dma >> 16) >> 16,  in ahci_start_fis_rx()
760 	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);  in ahci_start_fis_rx()
786 		return -EBUSY;  in ahci_stop_fis_rx()
793 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_power_up()
800 	if (hpriv->cap & HOST_CAP_SSS) {  in ahci_power_up()
812 	struct ata_port *ap = link->ap;  in ahci_set_lpm()
813 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_set_lpm()
814 	struct ahci_port_priv *pp = ap->private_data;  in ahci_set_lpm()
826 		pp->intr_mask &= ~PORT_IRQ_PHYRDY;  in ahci_set_lpm()
827 		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);  in ahci_set_lpm()
832 	if (hpriv->cap & HOST_CAP_ALPM) {  in ahci_set_lpm()
861 	if ((hpriv->cap2 & HOST_CAP2_SDS) &&  in ahci_set_lpm()
862 	    (hpriv->cap2 & HOST_CAP2_SADM) &&  in ahci_set_lpm()
863 	    (link->device->flags & ATA_DFLAG_DEVSLP)) {  in ahci_set_lpm()
875 		pp->intr_mask |= PORT_IRQ_PHYRDY;  in ahci_set_lpm()
876 		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);  in ahci_set_lpm()
885 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_power_down()
889 	if (!(hpriv->cap & HOST_CAP_SSS))  in ahci_power_down()
906 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_start_port()
907 	struct ahci_port_priv *pp = ap->private_data;  in ahci_start_port()
917 	if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))  in ahci_start_port()
918 		hpriv->start_engine(ap);  in ahci_start_port()
921 	if (ap->flags & ATA_FLAG_EM) {  in ahci_start_port()
923 			emp = &pp->em_priv[link->pmp];  in ahci_start_port()
927 				rc = ap->ops->transmit_led_message(ap,  in ahci_start_port()
928 							       emp->led_state,  in ahci_start_port()
938 				if (rc == -EBUSY)  in ahci_start_port()
946 	if (ap->flags & ATA_FLAG_SW_ACTIVITY)  in ahci_start_port()
955 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_deinit_port()
958 	rc = hpriv->stop_engine(ap);  in ahci_deinit_port()
976 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_reset_controller()
977 	void __iomem *mmio = hpriv->mmio;  in ahci_reset_controller()
981 	 * We must be in AHCI mode, before using anything AHCI-specific, such  in ahci_reset_controller()
988 		dev_info(host->dev, "Skipping global host reset\n");  in ahci_reset_controller()
1006 		dev_err(host->dev, "Controller reset failed (0x%x)\n",  in ahci_reset_controller()
1008 		return -EIO;  in ahci_reset_controller()
1015 	if (!(hpriv->flags & AHCI_HFLAG_NO_WRITE_TO_RO))  in ahci_reset_controller()
1024 	struct ata_port *ap = link->ap;  in ahci_sw_activity()
1025 	struct ahci_port_priv *pp = ap->private_data;  in ahci_sw_activity()
1026 	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];  in ahci_sw_activity()
1028 	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))  in ahci_sw_activity()
1031 	emp->activity++;  in ahci_sw_activity()
1032 	if (!timer_pending(&emp->timer))  in ahci_sw_activity()
1033 		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));  in ahci_sw_activity()
1039 	struct ata_link *link = emp->link;  in ahci_sw_activity_blink()
1040 	struct ata_port *ap = link->ap;  in ahci_sw_activity_blink()
1042 	unsigned long led_message = emp->led_state;  in ahci_sw_activity_blink()
1047 	led_message |= ap->port_no | (link->pmp << 8);  in ahci_sw_activity_blink()
1053 	spin_lock_irqsave(ap->lock, flags);  in ahci_sw_activity_blink()
1054 	if (emp->saved_activity != emp->activity) {  in ahci_sw_activity_blink()
1055 		emp->saved_activity = emp->activity;  in ahci_sw_activity_blink()
1069 		mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));  in ahci_sw_activity_blink()
1073 		if (emp->blink_policy == BLINK_OFF)  in ahci_sw_activity_blink()
1076 	spin_unlock_irqrestore(ap->lock, flags);  in ahci_sw_activity_blink()
1077 	ap->ops->transmit_led_message(ap, led_message, 4);  in ahci_sw_activity_blink()
1082 	struct ata_port *ap = link->ap;  in ahci_init_sw_activity()
1083 	struct ahci_port_priv *pp = ap->private_data;  in ahci_init_sw_activity()
1084 	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];  in ahci_init_sw_activity()
1087 	emp->saved_activity = emp->activity = 0;  in ahci_init_sw_activity()
1088 	emp->link = link;  in ahci_init_sw_activity()
1089 	timer_setup(&emp->timer, ahci_sw_activity_blink, 0);  in ahci_init_sw_activity()
1092 	if (emp->blink_policy)  in ahci_init_sw_activity()
1093 		link->flags |= ATA_LFLAG_SW_ACTIVITY;  in ahci_init_sw_activity()
1098 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_reset_em()
1099 	void __iomem *mmio = hpriv->mmio;  in ahci_reset_em()
1104 		return -EINVAL;  in ahci_reset_em()
1114 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_transmit_led_message()
1115 	struct ahci_port_priv *pp = ap->private_data;  in ahci_transmit_led_message()
1116 	void __iomem *mmio = hpriv->mmio;  in ahci_transmit_led_message()
1126 		emp = &pp->em_priv[pmp];  in ahci_transmit_led_message()
1128 		return -EINVAL;  in ahci_transmit_led_message()
1131 	spin_lock_irqsave(ap->lock, flags);  in ahci_transmit_led_message()
1139 		spin_unlock_irqrestore(ap->lock, flags);  in ahci_transmit_led_message()
1141 		return -EBUSY;  in ahci_transmit_led_message()
1144 	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {  in ahci_transmit_led_message()
1146 		 * create message header - this is all zero except for  in ahci_transmit_led_message()
1151 		/* ignore 0:4 of byte zero, fill in port info yourself */  in ahci_transmit_led_message()
1152 		message[1] = ((state & ~EM_MSG_LED_HBA_PORT) | ap->port_no);  in ahci_transmit_led_message()
1155 		writel(message[0], mmio + hpriv->em_loc);  in ahci_transmit_led_message()
1156 		writel(message[1], mmio + hpriv->em_loc+4);  in ahci_transmit_led_message()
1164 	/* save off new led state for port/slot */  in ahci_transmit_led_message()
1165 	emp->led_state = state;  in ahci_transmit_led_message()
1167 	spin_unlock_irqrestore(ap->lock, flags);  in ahci_transmit_led_message()
1175 	struct ahci_port_priv *pp = ap->private_data;  in ahci_led_show()
1181 		emp = &pp->em_priv[link->pmp];  in ahci_led_show()
1182 		rc += sprintf(buf, "%lx\n", emp->led_state);  in ahci_led_show()
1192 	struct ahci_port_priv *pp = ap->private_data;  in ahci_led_store()
1196 		return -EINVAL;  in ahci_led_store()
1202 		emp = &pp->em_priv[pmp];  in ahci_led_store()
1204 		return -EINVAL;  in ahci_led_store()
1211 	if (emp->blink_policy)  in ahci_led_store()
1214 	return ap->ops->transmit_led_message(ap, state, size);  in ahci_led_store()
1219 	struct ata_link *link = dev->link;  in ahci_activity_store()
1220 	struct ata_port *ap = link->ap;  in ahci_activity_store()
1221 	struct ahci_port_priv *pp = ap->private_data;  in ahci_activity_store()
1222 	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];  in ahci_activity_store()
1223 	u32 port_led_state = emp->led_state;  in ahci_activity_store()
1228 		link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);  in ahci_activity_store()
1232 		port_led_state |= (ap->port_no | (link->pmp << 8));  in ahci_activity_store()
1233 		ap->ops->transmit_led_message(ap, port_led_state, 4);  in ahci_activity_store()
1235 		link->flags |= ATA_LFLAG_SW_ACTIVITY;  in ahci_activity_store()
1239 			port_led_state |= (ap->port_no | (link->pmp << 8));  in ahci_activity_store()
1241 			ap->ops->transmit_led_message(ap, port_led_state, 4);  in ahci_activity_store()
1244 	emp->blink_policy = val;  in ahci_activity_store()
1250 	struct ata_link *link = dev->link;  in ahci_activity_show()
1251 	struct ata_port *ap = link->ap;  in ahci_activity_show()
1252 	struct ahci_port_priv *pp = ap->private_data;  in ahci_activity_show()
1253 	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];  in ahci_activity_show()
1258 	return sprintf(buf, "%d\n", emp->blink_policy);  in ahci_activity_show()
1263 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_port_clear_pending_irq()
1269 	dev_dbg(ap->host->dev, "PORT_SCR_ERR 0x%x\n", tmp);  in ahci_port_clear_pending_irq()
1272 	/* clear port IRQ */  in ahci_port_clear_pending_irq()
1274 	dev_dbg(ap->host->dev, "PORT_IRQ_STAT 0x%x\n", tmp);  in ahci_port_clear_pending_irq()
1278 	writel(1 << ap->port_no, hpriv->mmio + HOST_IRQ_STAT);  in ahci_port_clear_pending_irq()
1288 	/* make sure port is not active */  in ahci_port_init()
1298 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_init_controller()
1299 	void __iomem *mmio = hpriv->mmio;  in ahci_init_controller()
1304 	for (i = 0; i < host->n_ports; i++) {  in ahci_init_controller()
1305 		struct ata_port *ap = host->ports[i];  in ahci_init_controller()
1311 		ahci_port_init(host->dev, ap, i, mmio, port_mmio);  in ahci_init_controller()
1315 	dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);  in ahci_init_controller()
1318 	dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);  in ahci_init_controller()
1324 	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;  in ahci_dev_config()
1326 	if ((dev->class == ATA_DEV_ATAPI) &&  in ahci_dev_config()
1327 	    (hpriv->flags & AHCI_HFLAG_ATAPI_DMA_QUIRK))  in ahci_dev_config()
1328 		dev->quirks |= ATA_QUIRK_ATAPI_MOD16_DMA;  in ahci_dev_config()
1330 	if (hpriv->flags & AHCI_HFLAG_SECT255) {  in ahci_dev_config()
1331 		dev->max_sectors = 255;  in ahci_dev_config()
1358 	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;  in ahci_fill_cmd_slot()
1360 	pp->cmd_slot[tag].opts = cpu_to_le32(opts);  in ahci_fill_cmd_slot()
1361 	pp->cmd_slot[tag].status = 0;  in ahci_fill_cmd_slot()
1362 	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);  in ahci_fill_cmd_slot()
1363 	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);  in ahci_fill_cmd_slot()
1370 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_kick_engine()
1376 	rc = hpriv->stop_engine(ap);  in ahci_kick_engine()
1381 	 * always do CLO if PMP is attached (AHCI-1.3 9.2)  in ahci_kick_engine()
1389 	if (!(hpriv->cap & HOST_CAP_CLO)) {  in ahci_kick_engine()
1390 		rc = -EOPNOTSUPP;  in ahci_kick_engine()
1403 		rc = -EIO;  in ahci_kick_engine()
1407 	hpriv->start_engine(ap);  in ahci_kick_engine()
1417 	struct ahci_port_priv *pp = ap->private_data;  in ahci_exec_polled_cmd()
1419 	u8 *fis = pp->cmd_tbl;  in ahci_exec_polled_cmd()
1426 	/* set port value for softreset of Port Multiplier */  in ahci_exec_polled_cmd()
1427 	if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {  in ahci_exec_polled_cmd()
1432 		pp->fbs_last_dev = pmp;  in ahci_exec_polled_cmd()
1443 			return -EBUSY;  in ahci_exec_polled_cmd()
1455 	struct ata_port *ap = link->ap;  in ahci_do_softreset()
1456 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_do_softreset()
1457 	struct ahci_port_priv *pp = ap->private_data;  in ahci_do_softreset()
1465 	/* prepare for SRST (AHCI-1.1 10.4.1) */  in ahci_do_softreset()
1467 	if (rc && rc != -EOPNOTSUPP)  in ahci_do_softreset()
1471 	 * According to AHCI-1.2 9.3.9: if FBS is enable, software shall  in ahci_do_softreset()
1473 	 * that is attached to port multiplier.  in ahci_do_softreset()
1475 	if (!ata_is_host_link(link) && pp->fbs_enabled) {  in ahci_do_softreset()
1480 	ata_tf_init(link->device, &tf);  in ahci_do_softreset()
1486 		msecs = jiffies_to_msecs(deadline - now);  in ahci_do_softreset()
1491 		rc = -EIO;  in ahci_do_softreset()
1505 	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {  in ahci_do_softreset()
1514 		/* link occupied, -ENODEV too is an error */  in ahci_do_softreset()
1520 	/* re-enable FBS if disabled before */  in ahci_do_softreset()
1533 	void __iomem *port_mmio = ahci_port_base(link->ap);  in ahci_check_ready()
1551 	void __iomem *port_mmio = ahci_port_base(link->ap);  in ahci_bad_pmp_check_ready()
1560 		return -EIO;  in ahci_bad_pmp_check_ready()
1568 	struct ata_port *ap = link->ap;  in ahci_pmp_retry_softreset()
1579 	 * SATA HDD/ODD is connected to SATA port, do soft reset  in ahci_pmp_retry_softreset()
1580 	 * again to port 0.  in ahci_pmp_retry_softreset()
1582 	if (rc == -EIO) {  in ahci_pmp_retry_softreset()
1599 	const unsigned int *timing = sata_ehc_deb_timing(&link->eh_context);  in ahci_do_hardreset()
1600 	struct ata_port *ap = link->ap;  in ahci_do_hardreset()
1601 	struct ahci_port_priv *pp = ap->private_data;  in ahci_do_hardreset()
1602 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_do_hardreset()
1603 	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;  in ahci_do_hardreset()
1607 	hpriv->stop_engine(ap);  in ahci_do_hardreset()
1610 	ata_tf_init(link->device, &tf);  in ahci_do_hardreset()
1619 	hpriv->start_engine(ap);  in ahci_do_hardreset()
1638 	struct ata_port *ap = link->ap;  in ahci_postreset()
1644 	/* Make sure port's ATAPI bit is set appropriately */  in ahci_postreset()
1665 	for_each_sg(qc->sg, sg, qc->n_elem, si) {  in ahci_fill_sg()
1671 		ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);  in ahci_fill_sg()
1679 	struct ata_port *ap = qc->ap;  in ahci_pmp_qc_defer()
1680 	struct ahci_port_priv *pp = ap->private_data;  in ahci_pmp_qc_defer()
1682 	if (!sata_pmp_attached(ap) || pp->fbs_enabled)  in ahci_pmp_qc_defer()
1690 	struct ata_port *ap = qc->ap;  in ahci_qc_prep()
1691 	struct ahci_port_priv *pp = ap->private_data;  in ahci_qc_prep()
1692 	int is_atapi = ata_is_atapi(qc->tf.protocol);  in ahci_qc_prep()
1700 	 * a SATA Register - Host to Device command FIS.  in ahci_qc_prep()
1702 	cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;  in ahci_qc_prep()
1704 	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);  in ahci_qc_prep()
1707 		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);  in ahci_qc_prep()
1711 	if (qc->flags & ATA_QCFLAG_DMAMAP)  in ahci_qc_prep()
1717 	opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);  in ahci_qc_prep()
1718 	if (qc->tf.flags & ATA_TFLAG_WRITE)  in ahci_qc_prep()
1723 	ahci_fill_cmd_slot(pp, qc->hw_tag, opts);  in ahci_qc_prep()
1730 	struct ahci_port_priv *pp = ap->private_data;  in ahci_fbs_dec_intr()
1735 	BUG_ON(!pp->fbs_enabled);  in ahci_fbs_dec_intr()
1742 	while ((fbs & PORT_FBS_DEC) && retries--) {  in ahci_fbs_dec_intr()
1748 		dev_err(ap->host->dev, "failed to clear device error\n");  in ahci_fbs_dec_intr()
1753 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_error_intr()
1754 	struct ahci_port_priv *pp = ap->private_data;  in ahci_error_intr()
1755 	struct ata_eh_info *host_ehi = &ap->link.eh_info;  in ahci_error_intr()
1763 	if (pp->fbs_enabled) {  in ahci_error_intr()
1768 		if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {  in ahci_error_intr()
1769 			link = &ap->pmp_link[pmp];  in ahci_error_intr()
1779 		link = &ap->link;  in ahci_error_intr()
1781 	active_qc = ata_qc_from_tag(ap, link->active_tag);  in ahci_error_intr()
1782 	active_ehi = &link->eh_info;  in ahci_error_intr()
1789 	ahci_scr_read(&ap->link, SCR_ERROR, &serror);  in ahci_error_intr()
1790 	ahci_scr_write(&ap->link, SCR_ERROR, serror);  in ahci_error_intr()
1791 	host_ehi->serror |= serror;  in ahci_error_intr()
1794 	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)  in ahci_error_intr()
1803 			active_qc->err_mask |= AC_ERR_DEV;  in ahci_error_intr()
1805 			active_ehi->err_mask |= AC_ERR_DEV;  in ahci_error_intr()
1807 		if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)  in ahci_error_intr()
1808 			host_ehi->serror &= ~SERR_INTERNAL;  in ahci_error_intr()
1812 		u32 *unk = pp->rx_fis + RX_FIS_UNK;  in ahci_error_intr()
1814 		active_ehi->err_mask |= AC_ERR_HSM;  in ahci_error_intr()
1815 		active_ehi->action |= ATA_EH_RESET;  in ahci_error_intr()
1822 		active_ehi->err_mask |= AC_ERR_HSM;  in ahci_error_intr()
1823 		active_ehi->action |= ATA_EH_RESET;  in ahci_error_intr()
1828 		host_ehi->err_mask |= AC_ERR_HOST_BUS;  in ahci_error_intr()
1829 		host_ehi->action |= ATA_EH_RESET;  in ahci_error_intr()
1835 			active_ehi->err_mask |= AC_ERR_DEV;  in ahci_error_intr()
1837 			host_ehi->err_mask |= AC_ERR_ATA_BUS;  in ahci_error_intr()
1838 			host_ehi->action |= ATA_EH_RESET;  in ahci_error_intr()
1864 	struct ata_eh_info *ehi = &ap->link.eh_info;  in ahci_qc_complete()
1865 	struct ahci_port_priv *pp = ap->private_data;  in ahci_qc_complete()
1870 	 * pp->active_link is not reliable once FBS is enabled, both  in ahci_qc_complete()
1872 	 * NCQ and non-NCQ commands may be in flight at the same time.  in ahci_qc_complete()
1874 	if (pp->fbs_enabled) {  in ahci_qc_complete()
1875 		if (ap->qc_active) {  in ahci_qc_complete()
1880 		/* pp->active_link is valid iff any command is in flight */  in ahci_qc_complete()
1881 		if (ap->qc_active && pp->active_link->sactive)  in ahci_qc_complete()
1888 	if (unlikely(rc < 0 && !(ap->pflags & ATA_PFLAG_RESETTING))) {  in ahci_qc_complete()
1889 		ehi->err_mask |= AC_ERR_HSM;  in ahci_qc_complete()
1890 		ehi->action |= ATA_EH_RESET;  in ahci_qc_complete()
1898 	struct ahci_port_priv *pp = ap->private_data;  in ahci_handle_port_interrupt()
1899 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_handle_port_interrupt()
1902 	if (unlikely(ap->pflags & ATA_PFLAG_RESETTING))  in ahci_handle_port_interrupt()
1905 	if (sata_lpm_ignore_phy_events(&ap->link)) {  in ahci_handle_port_interrupt()
1907 		ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);  in ahci_handle_port_interrupt()
1931 		if (hpriv->cap & HOST_CAP_SNTF)  in ahci_handle_port_interrupt()
1942 			if (pp->fbs_enabled)  in ahci_handle_port_interrupt()
1945 				const __le32 *f = pp->rx_fis + RX_FIS_SDB;  in ahci_handle_port_interrupt()
1977 	spin_lock(ap->lock);  in ahci_multi_irqs_intr_hard()
1979 	spin_unlock(ap->lock);  in ahci_multi_irqs_intr_hard()
1988 	for (i = 0; i < host->n_ports; i++) {  in ahci_handle_port_intr()
1994 		ap = host->ports[i];  in ahci_handle_port_intr()
1999 				dev_warn(host->dev,  in ahci_handle_port_intr()
2000 					 "interrupt on disabled port %u\n", i);  in ahci_handle_port_intr()
2018 	hpriv = host->private_data;  in ahci_single_level_irq_intr()
2019 	mmio = hpriv->mmio;  in ahci_single_level_irq_intr()
2026 	irq_masked = irq_stat & hpriv->port_map;  in ahci_single_level_irq_intr()
2028 	spin_lock(&host->lock);  in ahci_single_level_irq_intr()
2033 	 * it should be cleared after all the port events are cleared;  in ahci_single_level_irq_intr()
2039 	 * pending event on a dummy port might cause screaming IRQ.  in ahci_single_level_irq_intr()
2043 	spin_unlock(&host->lock);  in ahci_single_level_irq_intr()
2050 	struct ata_port *ap = qc->ap;  in ahci_qc_issue()
2052 	struct ahci_port_priv *pp = ap->private_data;  in ahci_qc_issue()
2058 	pp->active_link = qc->dev->link;  in ahci_qc_issue()
2060 	if (ata_is_ncq(qc->tf.protocol))  in ahci_qc_issue()
2061 		writel(1 << qc->hw_tag, port_mmio + PORT_SCR_ACT);  in ahci_qc_issue()
2063 	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {  in ahci_qc_issue()
2066 		fbs |= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;  in ahci_qc_issue()
2068 		pp->fbs_last_dev = qc->dev->link->pmp;  in ahci_qc_issue()
2071 	writel(1 << qc->hw_tag, port_mmio + PORT_CMD_ISSUE);  in ahci_qc_issue()
2073 	ahci_sw_activity(qc->dev->link);  in ahci_qc_issue()
2081 	struct ahci_port_priv *pp = qc->ap->private_data;  in ahci_qc_fill_rtf()
2082 	u8 *rx_fis = pp->rx_fis;  in ahci_qc_fill_rtf()
2084 	if (pp->fbs_enabled)  in ahci_qc_fill_rtf()
2085 		rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;  in ahci_qc_fill_rtf()
2088 	 * After a successful execution of an ATA PIO data-in command,  in ahci_qc_fill_rtf()
2093 	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&  in ahci_qc_fill_rtf()
2094 	    !(qc->flags & ATA_QCFLAG_EH)) {  in ahci_qc_fill_rtf()
2095 		ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);  in ahci_qc_fill_rtf()
2096 		qc->result_tf.status = (rx_fis + RX_FIS_PIO_SETUP)[15];  in ahci_qc_fill_rtf()
2108 	if (ata_is_ncq(qc->tf.protocol)) {  in ahci_qc_fill_rtf()
2117 		qc->result_tf.status = fis[2];  in ahci_qc_fill_rtf()
2118 		qc->result_tf.error = fis[3];  in ahci_qc_fill_rtf()
2122 	ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);  in ahci_qc_fill_rtf()
2127 	struct ahci_port_priv *pp = ap->private_data;  in ahci_qc_ncq_fill_rtf()
2131 	if (!ap->qc_active)  in ahci_qc_ncq_fill_rtf()
2138 	if (!pp->fbs_enabled) {  in ahci_qc_ncq_fill_rtf()
2142 		if (!pp->active_link->sactive)  in ahci_qc_ncq_fill_rtf()
2145 		fis = pp->rx_fis + RX_FIS_SDB;  in ahci_qc_ncq_fill_rtf()
2154 			if (qc && ata_is_ncq(qc->tf.protocol)) {  in ahci_qc_ncq_fill_rtf()
2155 				qc->result_tf.status = status;  in ahci_qc_ncq_fill_rtf()
2156 				qc->result_tf.error = error;  in ahci_qc_ncq_fill_rtf()
2157 				qc->result_tf.flags = qc->tf.flags;  in ahci_qc_ncq_fill_rtf()
2158 				qc->flags |= ATA_QCFLAG_RTF_FILLED;  in ahci_qc_ncq_fill_rtf()
2176 		if (qc && ata_is_ncq(qc->tf.protocol)) {  in ahci_qc_ncq_fill_rtf()
2177 			fis = pp->rx_fis;  in ahci_qc_ncq_fill_rtf()
2178 			fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;  in ahci_qc_ncq_fill_rtf()
2180 			qc->result_tf.status = fis[2];  in ahci_qc_ncq_fill_rtf()
2181 			qc->result_tf.error = fis[3];  in ahci_qc_ncq_fill_rtf()
2182 			qc->result_tf.flags = qc->tf.flags;  in ahci_qc_ncq_fill_rtf()
2183 			qc->flags |= ATA_QCFLAG_RTF_FILLED;  in ahci_qc_ncq_fill_rtf()
2199 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_thaw()
2200 	void __iomem *mmio = hpriv->mmio;  in ahci_thaw()
2203 	struct ahci_port_priv *pp = ap->private_data;  in ahci_thaw()
2208 	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);  in ahci_thaw()
2211 	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);  in ahci_thaw()
2216 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_error_handler()
2220 		hpriv->stop_engine(ap);  in ahci_error_handler()
2221 		hpriv->start_engine(ap);  in ahci_error_handler()
2226 	if (!ata_dev_enabled(ap->link.device))  in ahci_error_handler()
2227 		hpriv->stop_engine(ap);  in ahci_error_handler()
2233 	struct ata_port *ap = qc->ap;  in ahci_post_internal_cmd()
2236 	if (qc->flags & ATA_QCFLAG_EH)  in ahci_post_internal_cmd()
2242 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_set_aggressive_devslp()
2244 	struct ata_device *dev = ap->link.device;  in ahci_set_aggressive_devslp()
2251 		dev_info(ap->host->dev, "port does not support device sleep\n");  in ahci_set_aggressive_devslp()
2281 	rc = hpriv->stop_engine(ap);  in ahci_set_aggressive_devslp()
2288 	if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &  in ahci_set_aggressive_devslp()
2290 		mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &  in ahci_set_aggressive_devslp()
2294 		deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];  in ahci_set_aggressive_devslp()
2310 	hpriv->start_engine(ap);  in ahci_set_aggressive_devslp()
2322 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_enable_fbs()
2323 	struct ahci_port_priv *pp = ap->private_data;  in ahci_enable_fbs()
2328 	if (!pp->fbs_supported)  in ahci_enable_fbs()
2333 		pp->fbs_enabled = true;  in ahci_enable_fbs()
2334 		pp->fbs_last_dev = -1; /* initialization */  in ahci_enable_fbs()
2338 	rc = hpriv->stop_engine(ap);  in ahci_enable_fbs()
2345 		dev_info(ap->host->dev, "FBS is enabled\n");  in ahci_enable_fbs()
2346 		pp->fbs_enabled = true;  in ahci_enable_fbs()
2347 		pp->fbs_last_dev = -1; /* initialization */  in ahci_enable_fbs()
2349 		dev_err(ap->host->dev, "Failed to enable FBS\n");  in ahci_enable_fbs()
2351 	hpriv->start_engine(ap);  in ahci_enable_fbs()
2356 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_disable_fbs()
2357 	struct ahci_port_priv *pp = ap->private_data;  in ahci_disable_fbs()
2362 	if (!pp->fbs_supported)  in ahci_disable_fbs()
2367 		pp->fbs_enabled = false;  in ahci_disable_fbs()
2371 	rc = hpriv->stop_engine(ap);  in ahci_disable_fbs()
2378 		dev_err(ap->host->dev, "Failed to disable FBS\n");  in ahci_disable_fbs()
2380 		dev_info(ap->host->dev, "FBS is disabled\n");  in ahci_disable_fbs()
2381 		pp->fbs_enabled = false;  in ahci_disable_fbs()
2384 	hpriv->start_engine(ap);  in ahci_disable_fbs()
2390 	struct ahci_port_priv *pp = ap->private_data;  in ahci_pmp_attach()
2399 	pp->intr_mask |= PORT_IRQ_BAD_PMP;  in ahci_pmp_attach()
2402 	 * We must not change the port interrupt mask register if the  in ahci_pmp_attach()
2403 	 * port is marked frozen, the value in pp->intr_mask will be  in ahci_pmp_attach()
2404 	 * restored later when the port is thawed.  in ahci_pmp_attach()
2406 	 * Note that during initialization, the port is marked as  in ahci_pmp_attach()
2410 		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);  in ahci_pmp_attach()
2416 	struct ahci_port_priv *pp = ap->private_data;  in ahci_pmp_detach()
2425 	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;  in ahci_pmp_detach()
2429 		writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);  in ahci_pmp_detach()
2474 	if (acpi_storage_d3(ap->host->dev))  in ahci_port_suspend()
2484 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_port_start()
2485 	struct device *dev = ap->host->dev;  in ahci_port_start()
2493 		return -ENOMEM;  in ahci_port_start()
2495 	if (ap->host->n_ports > 1) {  in ahci_port_start()
2496 		pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);  in ahci_port_start()
2497 		if (!pp->irq_desc) {  in ahci_port_start()
2499 			return -ENOMEM;  in ahci_port_start()
2501 		snprintf(pp->irq_desc, 8,  in ahci_port_start()
2502 			 "%s%d", dev_driver_string(dev), ap->port_no);  in ahci_port_start()
2506 	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {  in ahci_port_start()
2510 			pp->fbs_supported = true;  in ahci_port_start()
2511 		else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {  in ahci_port_start()
2512 			dev_info(dev, "port %d can do FBS, forcing FBSCP\n",  in ahci_port_start()
2513 				 ap->port_no);  in ahci_port_start()
2514 			pp->fbs_supported = true;  in ahci_port_start()
2516 			dev_warn(dev, "port %d is not capable of FBS\n",  in ahci_port_start()
2517 				 ap->port_no);  in ahci_port_start()
2520 	if (pp->fbs_supported) {  in ahci_port_start()
2530 		return -ENOMEM;  in ahci_port_start()
2533 	 * First item in chunk of DMA memory: 32-slot command table,  in ahci_port_start()
2536 	pp->cmd_slot = mem;  in ahci_port_start()
2537 	pp->cmd_slot_dma = mem_dma;  in ahci_port_start()
2543 	 * Second item: Received-FIS area  in ahci_port_start()
2545 	pp->rx_fis = mem;  in ahci_port_start()
2546 	pp->rx_fis_dma = mem_dma;  in ahci_port_start()
2553 	 * and its scatter-gather table  in ahci_port_start()
2555 	pp->cmd_tbl = mem;  in ahci_port_start()
2556 	pp->cmd_tbl_dma = mem_dma;  in ahci_port_start()
2562 	pp->intr_mask = DEF_PORT_IRQ;  in ahci_port_start()
2565 	 * Switch to per-port locking in case each port has its own MSI vector.  in ahci_port_start()
2567 	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {  in ahci_port_start()
2568 		spin_lock_init(&pp->lock);  in ahci_port_start()
2569 		ap->lock = &pp->lock;  in ahci_port_start()
2572 	ap->private_data = pp;  in ahci_port_start()
2581 	struct ahci_host_priv *hpriv = ap->host->private_data;  in ahci_port_stop()
2582 	void __iomem *host_mmio = hpriv->mmio;  in ahci_port_stop()
2585 	/* de-initialize port */  in ahci_port_stop()
2592 	 * re-enabling INTx.  in ahci_port_stop()
2594 	writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT);  in ahci_port_stop()
2601 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_print_info()
2602 	u32 vers, cap, cap2, impl, speed;  in ahci_print_info()  local
2605 	vers = hpriv->version;  in ahci_print_info()
2606 	cap = hpriv->cap;  in ahci_print_info()
2607 	cap2 = hpriv->cap2;  in ahci_print_info()
2608 	impl = hpriv->port_map;  in ahci_print_info()
2610 	speed = (cap >> 20) & 0xf;  in ahci_print_info()
2620 	dev_info(host->dev,  in ahci_print_info()
2630 		((cap >> 8) & 0x1f) + 1,  in ahci_print_info()
2634 	dev_info(host->dev,  in ahci_print_info()
2635 		"%u/%u ports implemented (port mask 0x%x)\n"  in ahci_print_info()
2639 		(cap & 0x1f) + 1,  in ahci_print_info()
2642 	dev_info(host->dev,  in ahci_print_info()
2650 		cap & HOST_CAP_64 ? "64bit " : "",  in ahci_print_info()
2651 		cap & HOST_CAP_NCQ ? "ncq " : "",  in ahci_print_info()
2652 		cap & HOST_CAP_SNTF ? "sntf " : "",  in ahci_print_info()
2653 		cap & HOST_CAP_MPS ? "ilck " : "",  in ahci_print_info()
2654 		cap & HOST_CAP_SSS ? "stag " : "",  in ahci_print_info()
2655 		cap & HOST_CAP_ALPM ? "pm " : "",  in ahci_print_info()
2656 		cap & HOST_CAP_LED ? "led " : "",  in ahci_print_info()
2657 		cap & HOST_CAP_CLO ? "clo " : "",  in ahci_print_info()
2658 		cap & HOST_CAP_ONLY ? "only " : "",  in ahci_print_info()
2659 		cap & HOST_CAP_PMP ? "pmp " : "",  in ahci_print_info()
2660 		cap & HOST_CAP_FBS ? "fbs " : "",  in ahci_print_info()
2661 		cap & HOST_CAP_PIO_MULTI ? "pio " : "",  in ahci_print_info()
2662 		cap & HOST_CAP_SSC ? "slum " : "",  in ahci_print_info()
2663 		cap & HOST_CAP_PART ? "part " : "",  in ahci_print_info()
2664 		cap & HOST_CAP_CCC ? "ccc " : "",  in ahci_print_info()
2665 		cap & HOST_CAP_EMS ? "ems " : "",  in ahci_print_info()
2666 		cap & HOST_CAP_SXS ? "sxs " : "",  in ahci_print_info()
2681 	void __iomem *mmio = hpriv->mmio;  in ahci_set_em_messages()
2685 	if (!ahci_em_messages || !(hpriv->cap & HOST_CAP_EMS))  in ahci_set_em_messages()
2692 		hpriv->em_loc = ((em_loc >> 16) * 4);  in ahci_set_em_messages()
2693 		hpriv->em_buf_sz = ((em_loc & 0xff) * 4);  in ahci_set_em_messages()
2694 		hpriv->em_msg_type = messages;  in ahci_set_em_messages()
2695 		pi->flags |= ATA_FLAG_EM;  in ahci_set_em_messages()
2697 			pi->flags |= ATA_FLAG_SW_ACTIVITY;  in ahci_set_em_messages()
2705 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_host_activate_multi_irqs()
2712 	 * Requests IRQs according to AHCI-1.1 when multiple MSIs were  in ahci_host_activate_multi_irqs()
2713 	 * allocated. That is one MSI per port, starting from @irq.  in ahci_host_activate_multi_irqs()
2715 	for (i = 0; i < host->n_ports; i++) {  in ahci_host_activate_multi_irqs()
2716 		struct ahci_port_priv *pp = host->ports[i]->private_data;  in ahci_host_activate_multi_irqs()
2717 		int irq = hpriv->get_irq_vector(host, i);  in ahci_host_activate_multi_irqs()
2725 		rc = devm_request_irq(host->dev, irq, ahci_multi_irqs_intr_hard,  in ahci_host_activate_multi_irqs()
2726 				0, pp->irq_desc, host->ports[i]);  in ahci_host_activate_multi_irqs()
2730 		ata_port_desc_misc(host->ports[i], irq);  in ahci_host_activate_multi_irqs()
2737  *	ahci_host_activate - start AHCI host, request IRQs and register it
2745  *	0 on success, -errno otherwise.
2749 	struct ahci_host_priv *hpriv = host->private_data;  in ahci_host_activate()
2750 	int irq = hpriv->irq;  in ahci_host_activate()
2753 	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {  in ahci_host_activate()
2754 		if (hpriv->irq_handler &&  in ahci_host_activate()
2755 		    hpriv->irq_handler != ahci_single_level_irq_intr)  in ahci_host_activate()
2756 			dev_warn(host->dev,  in ahci_host_activate()
2758 		if (!hpriv->get_irq_vector) {  in ahci_host_activate()
2759 			dev_err(host->dev,  in ahci_host_activate()
2760 				"AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");  in ahci_host_activate()
2761 			return -EIO;  in ahci_host_activate()
2766 		rc = ata_host_activate(host, irq, hpriv->irq_handler,  in ahci_host_activate()
2776 MODULE_DESCRIPTION("Common AHCI SATA low-level routines");