xref: /linux/drivers/ata/sata_nv.c (revision 498ade1a133dffd0f3ee90952737045d56e6689a)
1 /*
2  *  sata_nv.c - NVIDIA nForce SATA
3  *
4  *  Copyright 2004 NVIDIA Corp.  All rights reserved.
5  *  Copyright 2004 Andrew Chew
6  *
7  *
8  *  This program is free software; you can redistribute it and/or modify
9  *  it under the terms of the GNU General Public License as published by
10  *  the Free Software Foundation; either version 2, or (at your option)
11  *  any later version.
12  *
13  *  This program is distributed in the hope that it will be useful,
14  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  *  GNU General Public License for more details.
17  *
18  *  You should have received a copy of the GNU General Public License
19  *  along with this program; see the file COPYING.  If not, write to
20  *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
21  *
22  *
23  *  libata documentation is available via 'make {ps|pdf}docs',
24  *  as Documentation/DocBook/libata.*
25  *
26  *  No hardware documentation available outside of NVIDIA.
27  *  This driver programs the NVIDIA SATA controller in a similar
28  *  fashion as with other PCI IDE BMDMA controllers, with a few
29  *  NV-specific details such as register offsets, SATA phy location,
30  *  hotplug info, etc.
31  *
32  *  CK804/MCP04 controllers support an alternate programming interface
33  *  similar to the ADMA specification (with some modifications).
34  *  This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35  *  sent through the legacy interface.
36  *
37  */
38 
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
50 
51 #define DRV_NAME			"sata_nv"
52 #define DRV_VERSION			"3.5"
53 
54 #define NV_ADMA_DMA_BOUNDARY		0xffffffffUL
55 
56 enum {
57 	NV_MMIO_BAR			= 5,
58 
59 	NV_PORTS			= 2,
60 	NV_PIO_MASK			= 0x1f,
61 	NV_MWDMA_MASK			= 0x07,
62 	NV_UDMA_MASK			= 0x7f,
63 	NV_PORT0_SCR_REG_OFFSET		= 0x00,
64 	NV_PORT1_SCR_REG_OFFSET		= 0x40,
65 
66 	/* INT_STATUS/ENABLE */
67 	NV_INT_STATUS			= 0x10,
68 	NV_INT_ENABLE			= 0x11,
69 	NV_INT_STATUS_CK804		= 0x440,
70 	NV_INT_ENABLE_CK804		= 0x441,
71 
72 	/* INT_STATUS/ENABLE bits */
73 	NV_INT_DEV			= 0x01,
74 	NV_INT_PM			= 0x02,
75 	NV_INT_ADDED			= 0x04,
76 	NV_INT_REMOVED			= 0x08,
77 
78 	NV_INT_PORT_SHIFT		= 4,	/* each port occupies 4 bits */
79 
80 	NV_INT_ALL			= 0x0f,
81 	NV_INT_MASK			= NV_INT_DEV |
82 					  NV_INT_ADDED | NV_INT_REMOVED,
83 
84 	/* INT_CONFIG */
85 	NV_INT_CONFIG			= 0x12,
86 	NV_INT_CONFIG_METHD		= 0x01, // 0 = INT, 1 = SMI
87 
88 	// For PCI config register 20
89 	NV_MCP_SATA_CFG_20		= 0x50,
90 	NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 	NV_MCP_SATA_CFG_20_PORT0_EN	= (1 << 17),
92 	NV_MCP_SATA_CFG_20_PORT1_EN	= (1 << 16),
93 	NV_MCP_SATA_CFG_20_PORT0_PWB_EN	= (1 << 14),
94 	NV_MCP_SATA_CFG_20_PORT1_PWB_EN	= (1 << 12),
95 
96 	NV_ADMA_MAX_CPBS		= 32,
97 	NV_ADMA_CPB_SZ			= 128,
98 	NV_ADMA_APRD_SZ			= 16,
99 	NV_ADMA_SGTBL_LEN		= (1024 - NV_ADMA_CPB_SZ) /
100 					   NV_ADMA_APRD_SZ,
101 	NV_ADMA_SGTBL_TOTAL_LEN		= NV_ADMA_SGTBL_LEN + 5,
102 	NV_ADMA_SGTBL_SZ                = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 	NV_ADMA_PORT_PRIV_DMA_SZ        = NV_ADMA_MAX_CPBS *
104 					   (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105 
106 	/* BAR5 offset to ADMA general registers */
107 	NV_ADMA_GEN			= 0x400,
108 	NV_ADMA_GEN_CTL			= 0x00,
109 	NV_ADMA_NOTIFIER_CLEAR		= 0x30,
110 
111 	/* BAR5 offset to ADMA ports */
112 	NV_ADMA_PORT			= 0x480,
113 
114 	/* size of ADMA port register space  */
115 	NV_ADMA_PORT_SIZE		= 0x100,
116 
117 	/* ADMA port registers */
118 	NV_ADMA_CTL			= 0x40,
119 	NV_ADMA_CPB_COUNT		= 0x42,
120 	NV_ADMA_NEXT_CPB_IDX		= 0x43,
121 	NV_ADMA_STAT			= 0x44,
122 	NV_ADMA_CPB_BASE_LOW		= 0x48,
123 	NV_ADMA_CPB_BASE_HIGH		= 0x4C,
124 	NV_ADMA_APPEND			= 0x50,
125 	NV_ADMA_NOTIFIER		= 0x68,
126 	NV_ADMA_NOTIFIER_ERROR		= 0x6C,
127 
128 	/* NV_ADMA_CTL register bits */
129 	NV_ADMA_CTL_HOTPLUG_IEN		= (1 << 0),
130 	NV_ADMA_CTL_CHANNEL_RESET	= (1 << 5),
131 	NV_ADMA_CTL_GO			= (1 << 7),
132 	NV_ADMA_CTL_AIEN		= (1 << 8),
133 	NV_ADMA_CTL_READ_NON_COHERENT	= (1 << 11),
134 	NV_ADMA_CTL_WRITE_NON_COHERENT	= (1 << 12),
135 
136 	/* CPB response flag bits */
137 	NV_CPB_RESP_DONE		= (1 << 0),
138 	NV_CPB_RESP_ATA_ERR		= (1 << 3),
139 	NV_CPB_RESP_CMD_ERR		= (1 << 4),
140 	NV_CPB_RESP_CPB_ERR		= (1 << 7),
141 
142 	/* CPB control flag bits */
143 	NV_CPB_CTL_CPB_VALID		= (1 << 0),
144 	NV_CPB_CTL_QUEUE		= (1 << 1),
145 	NV_CPB_CTL_APRD_VALID		= (1 << 2),
146 	NV_CPB_CTL_IEN			= (1 << 3),
147 	NV_CPB_CTL_FPDMA		= (1 << 4),
148 
149 	/* APRD flags */
150 	NV_APRD_WRITE			= (1 << 1),
151 	NV_APRD_END			= (1 << 2),
152 	NV_APRD_CONT			= (1 << 3),
153 
154 	/* NV_ADMA_STAT flags */
155 	NV_ADMA_STAT_TIMEOUT		= (1 << 0),
156 	NV_ADMA_STAT_HOTUNPLUG		= (1 << 1),
157 	NV_ADMA_STAT_HOTPLUG		= (1 << 2),
158 	NV_ADMA_STAT_CPBERR		= (1 << 4),
159 	NV_ADMA_STAT_SERROR		= (1 << 5),
160 	NV_ADMA_STAT_CMD_COMPLETE	= (1 << 6),
161 	NV_ADMA_STAT_IDLE		= (1 << 8),
162 	NV_ADMA_STAT_LEGACY		= (1 << 9),
163 	NV_ADMA_STAT_STOPPED		= (1 << 10),
164 	NV_ADMA_STAT_DONE		= (1 << 12),
165 	NV_ADMA_STAT_ERR		= NV_ADMA_STAT_CPBERR |
166 					  NV_ADMA_STAT_TIMEOUT,
167 
168 	/* port flags */
169 	NV_ADMA_PORT_REGISTER_MODE	= (1 << 0),
170 	NV_ADMA_ATAPI_SETUP_COMPLETE	= (1 << 1),
171 
172 	/* MCP55 reg offset */
173 	NV_CTL_MCP55			= 0x400,
174 	NV_INT_STATUS_MCP55		= 0x440,
175 	NV_INT_ENABLE_MCP55		= 0x444,
176 	NV_NCQ_REG_MCP55		= 0x448,
177 
178 	/* MCP55 */
179 	NV_INT_ALL_MCP55		= 0xffff,
180 	NV_INT_PORT_SHIFT_MCP55		= 16,	/* each port occupies 16 bits */
181 	NV_INT_MASK_MCP55		= NV_INT_ALL_MCP55 & 0xfffd,
182 
183 	/* SWNCQ ENABLE BITS*/
184 	NV_CTL_PRI_SWNCQ		= 0x02,
185 	NV_CTL_SEC_SWNCQ		= 0x04,
186 
187 	/* SW NCQ status bits*/
188 	NV_SWNCQ_IRQ_DEV		= (1 << 0),
189 	NV_SWNCQ_IRQ_PM			= (1 << 1),
190 	NV_SWNCQ_IRQ_ADDED		= (1 << 2),
191 	NV_SWNCQ_IRQ_REMOVED		= (1 << 3),
192 
193 	NV_SWNCQ_IRQ_BACKOUT		= (1 << 4),
194 	NV_SWNCQ_IRQ_SDBFIS		= (1 << 5),
195 	NV_SWNCQ_IRQ_DHREGFIS		= (1 << 6),
196 	NV_SWNCQ_IRQ_DMASETUP		= (1 << 7),
197 
198 	NV_SWNCQ_IRQ_HOTPLUG		= NV_SWNCQ_IRQ_ADDED |
199 					  NV_SWNCQ_IRQ_REMOVED,
200 
201 };
202 
203 /* ADMA Physical Region Descriptor - one SG segment */
204 struct nv_adma_prd {
205 	__le64			addr;
206 	__le32			len;
207 	u8			flags;
208 	u8			packet_len;
209 	__le16			reserved;
210 };
211 
212 enum nv_adma_regbits {
213 	CMDEND	= (1 << 15),		/* end of command list */
214 	WNB	= (1 << 14),		/* wait-not-BSY */
215 	IGN	= (1 << 13),		/* ignore this entry */
216 	CS1n	= (1 << (4 + 8)),	/* std. PATA signals follow... */
217 	DA2	= (1 << (2 + 8)),
218 	DA1	= (1 << (1 + 8)),
219 	DA0	= (1 << (0 + 8)),
220 };
221 
222 /* ADMA Command Parameter Block
223    The first 5 SG segments are stored inside the Command Parameter Block itself.
224    If there are more than 5 segments the remainder are stored in a separate
225    memory area indicated by next_aprd. */
226 struct nv_adma_cpb {
227 	u8			resp_flags;    /* 0 */
228 	u8			reserved1;     /* 1 */
229 	u8			ctl_flags;     /* 2 */
230 	/* len is length of taskfile in 64 bit words */
231 	u8			len;		/* 3  */
232 	u8			tag;           /* 4 */
233 	u8			next_cpb_idx;  /* 5 */
234 	__le16			reserved2;     /* 6-7 */
235 	__le16			tf[12];        /* 8-31 */
236 	struct nv_adma_prd	aprd[5];       /* 32-111 */
237 	__le64			next_aprd;     /* 112-119 */
238 	__le64			reserved3;     /* 120-127 */
239 };
240 
241 
242 struct nv_adma_port_priv {
243 	struct nv_adma_cpb	*cpb;
244 	dma_addr_t		cpb_dma;
245 	struct nv_adma_prd	*aprd;
246 	dma_addr_t		aprd_dma;
247 	void __iomem		*ctl_block;
248 	void __iomem		*gen_block;
249 	void __iomem		*notifier_clear_block;
250 	u8			flags;
251 	int			last_issue_ncq;
252 };
253 
254 struct nv_host_priv {
255 	unsigned long		type;
256 };
257 
258 struct defer_queue {
259 	u32		defer_bits;
260 	unsigned int	head;
261 	unsigned int	tail;
262 	unsigned int	tag[ATA_MAX_QUEUE];
263 };
264 
265 enum ncq_saw_flag_list {
266 	ncq_saw_d2h	= (1U << 0),
267 	ncq_saw_dmas	= (1U << 1),
268 	ncq_saw_sdb	= (1U << 2),
269 	ncq_saw_backout	= (1U << 3),
270 };
271 
272 struct nv_swncq_port_priv {
273 	struct ata_prd	*prd;	 /* our SG list */
274 	dma_addr_t	prd_dma; /* and its DMA mapping */
275 	void __iomem	*sactive_block;
276 	void __iomem	*irq_block;
277 	void __iomem	*tag_block;
278 	u32		qc_active;
279 
280 	unsigned int	last_issue_tag;
281 
282 	/* fifo circular queue to store deferral command */
283 	struct defer_queue defer_queue;
284 
285 	/* for NCQ interrupt analysis */
286 	u32		dhfis_bits;
287 	u32		dmafis_bits;
288 	u32		sdbfis_bits;
289 
290 	unsigned int	ncq_flags;
291 };
292 
293 
294 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
295 
296 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
297 #ifdef CONFIG_PM
298 static int nv_pci_device_resume(struct pci_dev *pdev);
299 #endif
300 static void nv_ck804_host_stop(struct ata_host *host);
301 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
302 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
303 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
304 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
305 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
306 
307 static void nv_nf2_freeze(struct ata_port *ap);
308 static void nv_nf2_thaw(struct ata_port *ap);
309 static void nv_ck804_freeze(struct ata_port *ap);
310 static void nv_ck804_thaw(struct ata_port *ap);
311 static void nv_error_handler(struct ata_port *ap);
312 static int nv_adma_slave_config(struct scsi_device *sdev);
313 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
314 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
315 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
316 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
317 static void nv_adma_irq_clear(struct ata_port *ap);
318 static int nv_adma_port_start(struct ata_port *ap);
319 static void nv_adma_port_stop(struct ata_port *ap);
320 #ifdef CONFIG_PM
321 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
322 static int nv_adma_port_resume(struct ata_port *ap);
323 #endif
324 static void nv_adma_freeze(struct ata_port *ap);
325 static void nv_adma_thaw(struct ata_port *ap);
326 static void nv_adma_error_handler(struct ata_port *ap);
327 static void nv_adma_host_stop(struct ata_host *host);
328 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
329 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
330 
331 static void nv_mcp55_thaw(struct ata_port *ap);
332 static void nv_mcp55_freeze(struct ata_port *ap);
333 static void nv_swncq_error_handler(struct ata_port *ap);
334 static int nv_swncq_slave_config(struct scsi_device *sdev);
335 static int nv_swncq_port_start(struct ata_port *ap);
336 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
337 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
338 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
339 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
340 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
341 #ifdef CONFIG_PM
342 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
343 static int nv_swncq_port_resume(struct ata_port *ap);
344 #endif
345 
346 enum nv_host_type
347 {
348 	GENERIC,
349 	NFORCE2,
350 	NFORCE3 = NFORCE2,	/* NF2 == NF3 as far as sata_nv is concerned */
351 	CK804,
352 	ADMA,
353 	SWNCQ,
354 };
355 
356 static const struct pci_device_id nv_pci_tbl[] = {
357 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
358 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
359 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
360 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
361 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
362 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
363 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
364 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
365 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
366 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
367 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
368 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
369 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
370 	{ PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
371 
372 	{ } /* terminate list */
373 };
374 
375 static struct pci_driver nv_pci_driver = {
376 	.name			= DRV_NAME,
377 	.id_table		= nv_pci_tbl,
378 	.probe			= nv_init_one,
379 #ifdef CONFIG_PM
380 	.suspend		= ata_pci_device_suspend,
381 	.resume			= nv_pci_device_resume,
382 #endif
383 	.remove			= ata_pci_remove_one,
384 };
385 
386 static struct scsi_host_template nv_sht = {
387 	.module			= THIS_MODULE,
388 	.name			= DRV_NAME,
389 	.ioctl			= ata_scsi_ioctl,
390 	.queuecommand		= ata_scsi_queuecmd,
391 	.can_queue		= ATA_DEF_QUEUE,
392 	.this_id		= ATA_SHT_THIS_ID,
393 	.sg_tablesize		= LIBATA_MAX_PRD,
394 	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
395 	.emulated		= ATA_SHT_EMULATED,
396 	.use_clustering		= ATA_SHT_USE_CLUSTERING,
397 	.proc_name		= DRV_NAME,
398 	.dma_boundary		= ATA_DMA_BOUNDARY,
399 	.slave_configure	= ata_scsi_slave_config,
400 	.slave_destroy		= ata_scsi_slave_destroy,
401 	.bios_param		= ata_std_bios_param,
402 };
403 
404 static struct scsi_host_template nv_adma_sht = {
405 	.module			= THIS_MODULE,
406 	.name			= DRV_NAME,
407 	.ioctl			= ata_scsi_ioctl,
408 	.queuecommand		= ata_scsi_queuecmd,
409 	.change_queue_depth	= ata_scsi_change_queue_depth,
410 	.can_queue		= NV_ADMA_MAX_CPBS,
411 	.this_id		= ATA_SHT_THIS_ID,
412 	.sg_tablesize		= NV_ADMA_SGTBL_TOTAL_LEN,
413 	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
414 	.emulated		= ATA_SHT_EMULATED,
415 	.use_clustering		= ATA_SHT_USE_CLUSTERING,
416 	.proc_name		= DRV_NAME,
417 	.dma_boundary		= NV_ADMA_DMA_BOUNDARY,
418 	.slave_configure	= nv_adma_slave_config,
419 	.slave_destroy		= ata_scsi_slave_destroy,
420 	.bios_param		= ata_std_bios_param,
421 };
422 
423 static struct scsi_host_template nv_swncq_sht = {
424 	.module			= THIS_MODULE,
425 	.name			= DRV_NAME,
426 	.ioctl			= ata_scsi_ioctl,
427 	.queuecommand		= ata_scsi_queuecmd,
428 	.change_queue_depth	= ata_scsi_change_queue_depth,
429 	.can_queue		= ATA_MAX_QUEUE,
430 	.this_id		= ATA_SHT_THIS_ID,
431 	.sg_tablesize		= LIBATA_MAX_PRD,
432 	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
433 	.emulated		= ATA_SHT_EMULATED,
434 	.use_clustering		= ATA_SHT_USE_CLUSTERING,
435 	.proc_name		= DRV_NAME,
436 	.dma_boundary		= ATA_DMA_BOUNDARY,
437 	.slave_configure	= nv_swncq_slave_config,
438 	.slave_destroy		= ata_scsi_slave_destroy,
439 	.bios_param		= ata_std_bios_param,
440 };
441 
442 static const struct ata_port_operations nv_generic_ops = {
443 	.tf_load		= ata_tf_load,
444 	.tf_read		= ata_tf_read,
445 	.exec_command		= ata_exec_command,
446 	.check_status		= ata_check_status,
447 	.dev_select		= ata_std_dev_select,
448 	.bmdma_setup		= ata_bmdma_setup,
449 	.bmdma_start		= ata_bmdma_start,
450 	.bmdma_stop		= ata_bmdma_stop,
451 	.bmdma_status		= ata_bmdma_status,
452 	.qc_prep		= ata_qc_prep,
453 	.qc_issue		= ata_qc_issue_prot,
454 	.freeze			= ata_bmdma_freeze,
455 	.thaw			= ata_bmdma_thaw,
456 	.error_handler		= nv_error_handler,
457 	.post_internal_cmd	= ata_bmdma_post_internal_cmd,
458 	.data_xfer		= ata_data_xfer,
459 	.irq_clear		= ata_bmdma_irq_clear,
460 	.irq_on			= ata_irq_on,
461 	.scr_read		= nv_scr_read,
462 	.scr_write		= nv_scr_write,
463 	.port_start		= ata_port_start,
464 };
465 
466 static const struct ata_port_operations nv_nf2_ops = {
467 	.tf_load		= ata_tf_load,
468 	.tf_read		= ata_tf_read,
469 	.exec_command		= ata_exec_command,
470 	.check_status		= ata_check_status,
471 	.dev_select		= ata_std_dev_select,
472 	.bmdma_setup		= ata_bmdma_setup,
473 	.bmdma_start		= ata_bmdma_start,
474 	.bmdma_stop		= ata_bmdma_stop,
475 	.bmdma_status		= ata_bmdma_status,
476 	.qc_prep		= ata_qc_prep,
477 	.qc_issue		= ata_qc_issue_prot,
478 	.freeze			= nv_nf2_freeze,
479 	.thaw			= nv_nf2_thaw,
480 	.error_handler		= nv_error_handler,
481 	.post_internal_cmd	= ata_bmdma_post_internal_cmd,
482 	.data_xfer		= ata_data_xfer,
483 	.irq_clear		= ata_bmdma_irq_clear,
484 	.irq_on			= ata_irq_on,
485 	.scr_read		= nv_scr_read,
486 	.scr_write		= nv_scr_write,
487 	.port_start		= ata_port_start,
488 };
489 
490 static const struct ata_port_operations nv_ck804_ops = {
491 	.tf_load		= ata_tf_load,
492 	.tf_read		= ata_tf_read,
493 	.exec_command		= ata_exec_command,
494 	.check_status		= ata_check_status,
495 	.dev_select		= ata_std_dev_select,
496 	.bmdma_setup		= ata_bmdma_setup,
497 	.bmdma_start		= ata_bmdma_start,
498 	.bmdma_stop		= ata_bmdma_stop,
499 	.bmdma_status		= ata_bmdma_status,
500 	.qc_prep		= ata_qc_prep,
501 	.qc_issue		= ata_qc_issue_prot,
502 	.freeze			= nv_ck804_freeze,
503 	.thaw			= nv_ck804_thaw,
504 	.error_handler		= nv_error_handler,
505 	.post_internal_cmd	= ata_bmdma_post_internal_cmd,
506 	.data_xfer		= ata_data_xfer,
507 	.irq_clear		= ata_bmdma_irq_clear,
508 	.irq_on			= ata_irq_on,
509 	.scr_read		= nv_scr_read,
510 	.scr_write		= nv_scr_write,
511 	.port_start		= ata_port_start,
512 	.host_stop		= nv_ck804_host_stop,
513 };
514 
515 static const struct ata_port_operations nv_adma_ops = {
516 	.tf_load		= ata_tf_load,
517 	.tf_read		= nv_adma_tf_read,
518 	.check_atapi_dma	= nv_adma_check_atapi_dma,
519 	.exec_command		= ata_exec_command,
520 	.check_status		= ata_check_status,
521 	.dev_select		= ata_std_dev_select,
522 	.bmdma_setup		= ata_bmdma_setup,
523 	.bmdma_start		= ata_bmdma_start,
524 	.bmdma_stop		= ata_bmdma_stop,
525 	.bmdma_status		= ata_bmdma_status,
526 	.qc_defer		= ata_std_qc_defer,
527 	.qc_prep		= nv_adma_qc_prep,
528 	.qc_issue		= nv_adma_qc_issue,
529 	.freeze			= nv_adma_freeze,
530 	.thaw			= nv_adma_thaw,
531 	.error_handler		= nv_adma_error_handler,
532 	.post_internal_cmd	= nv_adma_post_internal_cmd,
533 	.data_xfer		= ata_data_xfer,
534 	.irq_clear		= nv_adma_irq_clear,
535 	.irq_on			= ata_irq_on,
536 	.scr_read		= nv_scr_read,
537 	.scr_write		= nv_scr_write,
538 	.port_start		= nv_adma_port_start,
539 	.port_stop		= nv_adma_port_stop,
540 #ifdef CONFIG_PM
541 	.port_suspend		= nv_adma_port_suspend,
542 	.port_resume		= nv_adma_port_resume,
543 #endif
544 	.host_stop		= nv_adma_host_stop,
545 };
546 
547 static const struct ata_port_operations nv_swncq_ops = {
548 	.tf_load		= ata_tf_load,
549 	.tf_read		= ata_tf_read,
550 	.exec_command		= ata_exec_command,
551 	.check_status		= ata_check_status,
552 	.dev_select		= ata_std_dev_select,
553 	.bmdma_setup		= ata_bmdma_setup,
554 	.bmdma_start		= ata_bmdma_start,
555 	.bmdma_stop		= ata_bmdma_stop,
556 	.bmdma_status		= ata_bmdma_status,
557 	.qc_defer		= ata_std_qc_defer,
558 	.qc_prep		= nv_swncq_qc_prep,
559 	.qc_issue		= nv_swncq_qc_issue,
560 	.freeze			= nv_mcp55_freeze,
561 	.thaw			= nv_mcp55_thaw,
562 	.error_handler		= nv_swncq_error_handler,
563 	.post_internal_cmd	= ata_bmdma_post_internal_cmd,
564 	.data_xfer		= ata_data_xfer,
565 	.irq_clear		= ata_bmdma_irq_clear,
566 	.irq_on			= ata_irq_on,
567 	.scr_read		= nv_scr_read,
568 	.scr_write		= nv_scr_write,
569 #ifdef CONFIG_PM
570 	.port_suspend		= nv_swncq_port_suspend,
571 	.port_resume		= nv_swncq_port_resume,
572 #endif
573 	.port_start		= nv_swncq_port_start,
574 };
575 
576 static const struct ata_port_info nv_port_info[] = {
577 	/* generic */
578 	{
579 		.sht		= &nv_sht,
580 		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
581 		.link_flags	= ATA_LFLAG_HRST_TO_RESUME,
582 		.pio_mask	= NV_PIO_MASK,
583 		.mwdma_mask	= NV_MWDMA_MASK,
584 		.udma_mask	= NV_UDMA_MASK,
585 		.port_ops	= &nv_generic_ops,
586 		.irq_handler	= nv_generic_interrupt,
587 	},
588 	/* nforce2/3 */
589 	{
590 		.sht		= &nv_sht,
591 		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
592 		.link_flags	= ATA_LFLAG_HRST_TO_RESUME,
593 		.pio_mask	= NV_PIO_MASK,
594 		.mwdma_mask	= NV_MWDMA_MASK,
595 		.udma_mask	= NV_UDMA_MASK,
596 		.port_ops	= &nv_nf2_ops,
597 		.irq_handler	= nv_nf2_interrupt,
598 	},
599 	/* ck804 */
600 	{
601 		.sht		= &nv_sht,
602 		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
603 		.link_flags	= ATA_LFLAG_HRST_TO_RESUME,
604 		.pio_mask	= NV_PIO_MASK,
605 		.mwdma_mask	= NV_MWDMA_MASK,
606 		.udma_mask	= NV_UDMA_MASK,
607 		.port_ops	= &nv_ck804_ops,
608 		.irq_handler	= nv_ck804_interrupt,
609 	},
610 	/* ADMA */
611 	{
612 		.sht		= &nv_adma_sht,
613 		.flags		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
614 				  ATA_FLAG_MMIO | ATA_FLAG_NCQ,
615 		.link_flags	= ATA_LFLAG_HRST_TO_RESUME,
616 		.pio_mask	= NV_PIO_MASK,
617 		.mwdma_mask	= NV_MWDMA_MASK,
618 		.udma_mask	= NV_UDMA_MASK,
619 		.port_ops	= &nv_adma_ops,
620 		.irq_handler	= nv_adma_interrupt,
621 	},
622 	/* SWNCQ */
623 	{
624 		.sht		= &nv_swncq_sht,
625 		.flags	        = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
626 				  ATA_FLAG_NCQ,
627 		.link_flags	= ATA_LFLAG_HRST_TO_RESUME,
628 		.pio_mask	= NV_PIO_MASK,
629 		.mwdma_mask	= NV_MWDMA_MASK,
630 		.udma_mask	= NV_UDMA_MASK,
631 		.port_ops	= &nv_swncq_ops,
632 		.irq_handler	= nv_swncq_interrupt,
633 	},
634 };
635 
636 MODULE_AUTHOR("NVIDIA");
637 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
638 MODULE_LICENSE("GPL");
639 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
640 MODULE_VERSION(DRV_VERSION);
641 
642 static int adma_enabled = 1;
643 static int swncq_enabled;
644 
645 static void nv_adma_register_mode(struct ata_port *ap)
646 {
647 	struct nv_adma_port_priv *pp = ap->private_data;
648 	void __iomem *mmio = pp->ctl_block;
649 	u16 tmp, status;
650 	int count = 0;
651 
652 	if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
653 		return;
654 
655 	status = readw(mmio + NV_ADMA_STAT);
656 	while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
657 		ndelay(50);
658 		status = readw(mmio + NV_ADMA_STAT);
659 		count++;
660 	}
661 	if (count == 20)
662 		ata_port_printk(ap, KERN_WARNING,
663 			"timeout waiting for ADMA IDLE, stat=0x%hx\n",
664 			status);
665 
666 	tmp = readw(mmio + NV_ADMA_CTL);
667 	writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
668 
669 	count = 0;
670 	status = readw(mmio + NV_ADMA_STAT);
671 	while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
672 		ndelay(50);
673 		status = readw(mmio + NV_ADMA_STAT);
674 		count++;
675 	}
676 	if (count == 20)
677 		ata_port_printk(ap, KERN_WARNING,
678 			 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
679 			 status);
680 
681 	pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
682 }
683 
684 static void nv_adma_mode(struct ata_port *ap)
685 {
686 	struct nv_adma_port_priv *pp = ap->private_data;
687 	void __iomem *mmio = pp->ctl_block;
688 	u16 tmp, status;
689 	int count = 0;
690 
691 	if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
692 		return;
693 
694 	WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
695 
696 	tmp = readw(mmio + NV_ADMA_CTL);
697 	writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
698 
699 	status = readw(mmio + NV_ADMA_STAT);
700 	while (((status & NV_ADMA_STAT_LEGACY) ||
701 	      !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
702 		ndelay(50);
703 		status = readw(mmio + NV_ADMA_STAT);
704 		count++;
705 	}
706 	if (count == 20)
707 		ata_port_printk(ap, KERN_WARNING,
708 			"timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
709 			status);
710 
711 	pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
712 }
713 
714 static int nv_adma_slave_config(struct scsi_device *sdev)
715 {
716 	struct ata_port *ap = ata_shost_to_port(sdev->host);
717 	struct nv_adma_port_priv *pp = ap->private_data;
718 	struct pci_dev *pdev = to_pci_dev(ap->host->dev);
719 	u64 bounce_limit;
720 	unsigned long segment_boundary;
721 	unsigned short sg_tablesize;
722 	int rc;
723 	int adma_enable;
724 	u32 current_reg, new_reg, config_mask;
725 
726 	rc = ata_scsi_slave_config(sdev);
727 
728 	if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
729 		/* Not a proper libata device, ignore */
730 		return rc;
731 
732 	if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
733 		/*
734 		 * NVIDIA reports that ADMA mode does not support ATAPI commands.
735 		 * Therefore ATAPI commands are sent through the legacy interface.
736 		 * However, the legacy interface only supports 32-bit DMA.
737 		 * Restrict DMA parameters as required by the legacy interface
738 		 * when an ATAPI device is connected.
739 		 */
740 		bounce_limit = ATA_DMA_MASK;
741 		segment_boundary = ATA_DMA_BOUNDARY;
742 		/* Subtract 1 since an extra entry may be needed for padding, see
743 		   libata-scsi.c */
744 		sg_tablesize = LIBATA_MAX_PRD - 1;
745 
746 		/* Since the legacy DMA engine is in use, we need to disable ADMA
747 		   on the port. */
748 		adma_enable = 0;
749 		nv_adma_register_mode(ap);
750 	} else {
751 		bounce_limit = *ap->dev->dma_mask;
752 		segment_boundary = NV_ADMA_DMA_BOUNDARY;
753 		sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
754 		adma_enable = 1;
755 	}
756 
757 	pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
758 
759 	if (ap->port_no == 1)
760 		config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
761 			      NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
762 	else
763 		config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
764 			      NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
765 
766 	if (adma_enable) {
767 		new_reg = current_reg | config_mask;
768 		pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
769 	} else {
770 		new_reg = current_reg & ~config_mask;
771 		pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
772 	}
773 
774 	if (current_reg != new_reg)
775 		pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
776 
777 	blk_queue_bounce_limit(sdev->request_queue, bounce_limit);
778 	blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
779 	blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
780 	ata_port_printk(ap, KERN_INFO,
781 		"bounce limit 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
782 		(unsigned long long)bounce_limit, segment_boundary, sg_tablesize);
783 	return rc;
784 }
785 
786 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
787 {
788 	struct nv_adma_port_priv *pp = qc->ap->private_data;
789 	return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
790 }
791 
792 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
793 {
794 	/* Other than when internal or pass-through commands are executed,
795 	   the only time this function will be called in ADMA mode will be
796 	   if a command fails. In the failure case we don't care about going
797 	   into register mode with ADMA commands pending, as the commands will
798 	   all shortly be aborted anyway. We assume that NCQ commands are not
799 	   issued via passthrough, which is the only way that switching into
800 	   ADMA mode could abort outstanding commands. */
801 	nv_adma_register_mode(ap);
802 
803 	ata_tf_read(ap, tf);
804 }
805 
806 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
807 {
808 	unsigned int idx = 0;
809 
810 	if (tf->flags & ATA_TFLAG_ISADDR) {
811 		if (tf->flags & ATA_TFLAG_LBA48) {
812 			cpb[idx++] = cpu_to_le16((ATA_REG_ERR   << 8) | tf->hob_feature | WNB);
813 			cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
814 			cpb[idx++] = cpu_to_le16((ATA_REG_LBAL  << 8) | tf->hob_lbal);
815 			cpb[idx++] = cpu_to_le16((ATA_REG_LBAM  << 8) | tf->hob_lbam);
816 			cpb[idx++] = cpu_to_le16((ATA_REG_LBAH  << 8) | tf->hob_lbah);
817 			cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature);
818 		} else
819 			cpb[idx++] = cpu_to_le16((ATA_REG_ERR    << 8) | tf->feature | WNB);
820 
821 		cpb[idx++] = cpu_to_le16((ATA_REG_NSECT  << 8) | tf->nsect);
822 		cpb[idx++] = cpu_to_le16((ATA_REG_LBAL   << 8) | tf->lbal);
823 		cpb[idx++] = cpu_to_le16((ATA_REG_LBAM   << 8) | tf->lbam);
824 		cpb[idx++] = cpu_to_le16((ATA_REG_LBAH   << 8) | tf->lbah);
825 	}
826 
827 	if (tf->flags & ATA_TFLAG_DEVICE)
828 		cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
829 
830 	cpb[idx++] = cpu_to_le16((ATA_REG_CMD    << 8) | tf->command | CMDEND);
831 
832 	while (idx < 12)
833 		cpb[idx++] = cpu_to_le16(IGN);
834 
835 	return idx;
836 }
837 
838 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
839 {
840 	struct nv_adma_port_priv *pp = ap->private_data;
841 	u8 flags = pp->cpb[cpb_num].resp_flags;
842 
843 	VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
844 
845 	if (unlikely((force_err ||
846 		     flags & (NV_CPB_RESP_ATA_ERR |
847 			      NV_CPB_RESP_CMD_ERR |
848 			      NV_CPB_RESP_CPB_ERR)))) {
849 		struct ata_eh_info *ehi = &ap->link.eh_info;
850 		int freeze = 0;
851 
852 		ata_ehi_clear_desc(ehi);
853 		__ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
854 		if (flags & NV_CPB_RESP_ATA_ERR) {
855 			ata_ehi_push_desc(ehi, "ATA error");
856 			ehi->err_mask |= AC_ERR_DEV;
857 		} else if (flags & NV_CPB_RESP_CMD_ERR) {
858 			ata_ehi_push_desc(ehi, "CMD error");
859 			ehi->err_mask |= AC_ERR_DEV;
860 		} else if (flags & NV_CPB_RESP_CPB_ERR) {
861 			ata_ehi_push_desc(ehi, "CPB error");
862 			ehi->err_mask |= AC_ERR_SYSTEM;
863 			freeze = 1;
864 		} else {
865 			/* notifier error, but no error in CPB flags? */
866 			ata_ehi_push_desc(ehi, "unknown");
867 			ehi->err_mask |= AC_ERR_OTHER;
868 			freeze = 1;
869 		}
870 		/* Kill all commands. EH will determine what actually failed. */
871 		if (freeze)
872 			ata_port_freeze(ap);
873 		else
874 			ata_port_abort(ap);
875 		return 1;
876 	}
877 
878 	if (likely(flags & NV_CPB_RESP_DONE)) {
879 		struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
880 		VPRINTK("CPB flags done, flags=0x%x\n", flags);
881 		if (likely(qc)) {
882 			DPRINTK("Completing qc from tag %d\n", cpb_num);
883 			ata_qc_complete(qc);
884 		} else {
885 			struct ata_eh_info *ehi = &ap->link.eh_info;
886 			/* Notifier bits set without a command may indicate the drive
887 			   is misbehaving. Raise host state machine violation on this
888 			   condition. */
889 			ata_port_printk(ap, KERN_ERR,
890 					"notifier for tag %d with no cmd?\n",
891 					cpb_num);
892 			ehi->err_mask |= AC_ERR_HSM;
893 			ehi->action |= ATA_EH_SOFTRESET;
894 			ata_port_freeze(ap);
895 			return 1;
896 		}
897 	}
898 	return 0;
899 }
900 
901 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
902 {
903 	struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
904 
905 	/* freeze if hotplugged */
906 	if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
907 		ata_port_freeze(ap);
908 		return 1;
909 	}
910 
911 	/* bail out if not our interrupt */
912 	if (!(irq_stat & NV_INT_DEV))
913 		return 0;
914 
915 	/* DEV interrupt w/ no active qc? */
916 	if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
917 		ata_check_status(ap);
918 		return 1;
919 	}
920 
921 	/* handle interrupt */
922 	return ata_host_intr(ap, qc);
923 }
924 
925 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
926 {
927 	struct ata_host *host = dev_instance;
928 	int i, handled = 0;
929 	u32 notifier_clears[2];
930 
931 	spin_lock(&host->lock);
932 
933 	for (i = 0; i < host->n_ports; i++) {
934 		struct ata_port *ap = host->ports[i];
935 		notifier_clears[i] = 0;
936 
937 		if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
938 			struct nv_adma_port_priv *pp = ap->private_data;
939 			void __iomem *mmio = pp->ctl_block;
940 			u16 status;
941 			u32 gen_ctl;
942 			u32 notifier, notifier_error;
943 
944 			/* if ADMA is disabled, use standard ata interrupt handler */
945 			if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
946 				u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
947 					>> (NV_INT_PORT_SHIFT * i);
948 				handled += nv_host_intr(ap, irq_stat);
949 				continue;
950 			}
951 
952 			/* if in ATA register mode, check for standard interrupts */
953 			if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
954 				u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
955 					>> (NV_INT_PORT_SHIFT * i);
956 				if (ata_tag_valid(ap->link.active_tag))
957 					/** NV_INT_DEV indication seems unreliable at times
958 					    at least in ADMA mode. Force it on always when a
959 					    command is active, to prevent losing interrupts. */
960 					irq_stat |= NV_INT_DEV;
961 				handled += nv_host_intr(ap, irq_stat);
962 			}
963 
964 			notifier = readl(mmio + NV_ADMA_NOTIFIER);
965 			notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
966 			notifier_clears[i] = notifier | notifier_error;
967 
968 			gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
969 
970 			if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
971 			    !notifier_error)
972 				/* Nothing to do */
973 				continue;
974 
975 			status = readw(mmio + NV_ADMA_STAT);
976 
977 			/* Clear status. Ensure the controller sees the clearing before we start
978 			   looking at any of the CPB statuses, so that any CPB completions after
979 			   this point in the handler will raise another interrupt. */
980 			writew(status, mmio + NV_ADMA_STAT);
981 			readw(mmio + NV_ADMA_STAT); /* flush posted write */
982 			rmb();
983 
984 			handled++; /* irq handled if we got here */
985 
986 			/* freeze if hotplugged or controller error */
987 			if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
988 					       NV_ADMA_STAT_HOTUNPLUG |
989 					       NV_ADMA_STAT_TIMEOUT |
990 					       NV_ADMA_STAT_SERROR))) {
991 				struct ata_eh_info *ehi = &ap->link.eh_info;
992 
993 				ata_ehi_clear_desc(ehi);
994 				__ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
995 				if (status & NV_ADMA_STAT_TIMEOUT) {
996 					ehi->err_mask |= AC_ERR_SYSTEM;
997 					ata_ehi_push_desc(ehi, "timeout");
998 				} else if (status & NV_ADMA_STAT_HOTPLUG) {
999 					ata_ehi_hotplugged(ehi);
1000 					ata_ehi_push_desc(ehi, "hotplug");
1001 				} else if (status & NV_ADMA_STAT_HOTUNPLUG) {
1002 					ata_ehi_hotplugged(ehi);
1003 					ata_ehi_push_desc(ehi, "hot unplug");
1004 				} else if (status & NV_ADMA_STAT_SERROR) {
1005 					/* let libata analyze SError and figure out the cause */
1006 					ata_ehi_push_desc(ehi, "SError");
1007 				} else
1008 					ata_ehi_push_desc(ehi, "unknown");
1009 				ata_port_freeze(ap);
1010 				continue;
1011 			}
1012 
1013 			if (status & (NV_ADMA_STAT_DONE |
1014 				      NV_ADMA_STAT_CPBERR)) {
1015 				u32 check_commands;
1016 				int pos, error = 0;
1017 
1018 				if (ata_tag_valid(ap->link.active_tag))
1019 					check_commands = 1 << ap->link.active_tag;
1020 				else
1021 					check_commands = ap->link.sactive;
1022 
1023 				/** Check CPBs for completed commands */
1024 				while ((pos = ffs(check_commands)) && !error) {
1025 					pos--;
1026 					error = nv_adma_check_cpb(ap, pos,
1027 						notifier_error & (1 << pos));
1028 					check_commands &= ~(1 << pos);
1029 				}
1030 			}
1031 		}
1032 	}
1033 
1034 	if (notifier_clears[0] || notifier_clears[1]) {
1035 		/* Note: Both notifier clear registers must be written
1036 		   if either is set, even if one is zero, according to NVIDIA. */
1037 		struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1038 		writel(notifier_clears[0], pp->notifier_clear_block);
1039 		pp = host->ports[1]->private_data;
1040 		writel(notifier_clears[1], pp->notifier_clear_block);
1041 	}
1042 
1043 	spin_unlock(&host->lock);
1044 
1045 	return IRQ_RETVAL(handled);
1046 }
1047 
1048 static void nv_adma_freeze(struct ata_port *ap)
1049 {
1050 	struct nv_adma_port_priv *pp = ap->private_data;
1051 	void __iomem *mmio = pp->ctl_block;
1052 	u16 tmp;
1053 
1054 	nv_ck804_freeze(ap);
1055 
1056 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1057 		return;
1058 
1059 	/* clear any outstanding CK804 notifications */
1060 	writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1061 		ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1062 
1063 	/* Disable interrupt */
1064 	tmp = readw(mmio + NV_ADMA_CTL);
1065 	writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1066 		mmio + NV_ADMA_CTL);
1067 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1068 }
1069 
1070 static void nv_adma_thaw(struct ata_port *ap)
1071 {
1072 	struct nv_adma_port_priv *pp = ap->private_data;
1073 	void __iomem *mmio = pp->ctl_block;
1074 	u16 tmp;
1075 
1076 	nv_ck804_thaw(ap);
1077 
1078 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1079 		return;
1080 
1081 	/* Enable interrupt */
1082 	tmp = readw(mmio + NV_ADMA_CTL);
1083 	writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1084 		mmio + NV_ADMA_CTL);
1085 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1086 }
1087 
1088 static void nv_adma_irq_clear(struct ata_port *ap)
1089 {
1090 	struct nv_adma_port_priv *pp = ap->private_data;
1091 	void __iomem *mmio = pp->ctl_block;
1092 	u32 notifier_clears[2];
1093 
1094 	if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1095 		ata_bmdma_irq_clear(ap);
1096 		return;
1097 	}
1098 
1099 	/* clear any outstanding CK804 notifications */
1100 	writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1101 		ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1102 
1103 	/* clear ADMA status */
1104 	writew(0xffff, mmio + NV_ADMA_STAT);
1105 
1106 	/* clear notifiers - note both ports need to be written with
1107 	   something even though we are only clearing on one */
1108 	if (ap->port_no == 0) {
1109 		notifier_clears[0] = 0xFFFFFFFF;
1110 		notifier_clears[1] = 0;
1111 	} else {
1112 		notifier_clears[0] = 0;
1113 		notifier_clears[1] = 0xFFFFFFFF;
1114 	}
1115 	pp = ap->host->ports[0]->private_data;
1116 	writel(notifier_clears[0], pp->notifier_clear_block);
1117 	pp = ap->host->ports[1]->private_data;
1118 	writel(notifier_clears[1], pp->notifier_clear_block);
1119 }
1120 
1121 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1122 {
1123 	struct nv_adma_port_priv *pp = qc->ap->private_data;
1124 
1125 	if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1126 		ata_bmdma_post_internal_cmd(qc);
1127 }
1128 
1129 static int nv_adma_port_start(struct ata_port *ap)
1130 {
1131 	struct device *dev = ap->host->dev;
1132 	struct nv_adma_port_priv *pp;
1133 	int rc;
1134 	void *mem;
1135 	dma_addr_t mem_dma;
1136 	void __iomem *mmio;
1137 	u16 tmp;
1138 
1139 	VPRINTK("ENTER\n");
1140 
1141 	rc = ata_port_start(ap);
1142 	if (rc)
1143 		return rc;
1144 
1145 	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1146 	if (!pp)
1147 		return -ENOMEM;
1148 
1149 	mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1150 	       ap->port_no * NV_ADMA_PORT_SIZE;
1151 	pp->ctl_block = mmio;
1152 	pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1153 	pp->notifier_clear_block = pp->gen_block +
1154 	       NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1155 
1156 	mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1157 				  &mem_dma, GFP_KERNEL);
1158 	if (!mem)
1159 		return -ENOMEM;
1160 	memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1161 
1162 	/*
1163 	 * First item in chunk of DMA memory:
1164 	 * 128-byte command parameter block (CPB)
1165 	 * one for each command tag
1166 	 */
1167 	pp->cpb     = mem;
1168 	pp->cpb_dma = mem_dma;
1169 
1170 	writel(mem_dma & 0xFFFFFFFF, 	mmio + NV_ADMA_CPB_BASE_LOW);
1171 	writel((mem_dma >> 16) >> 16,	mmio + NV_ADMA_CPB_BASE_HIGH);
1172 
1173 	mem     += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1174 	mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1175 
1176 	/*
1177 	 * Second item: block of ADMA_SGTBL_LEN s/g entries
1178 	 */
1179 	pp->aprd = mem;
1180 	pp->aprd_dma = mem_dma;
1181 
1182 	ap->private_data = pp;
1183 
1184 	/* clear any outstanding interrupt conditions */
1185 	writew(0xffff, mmio + NV_ADMA_STAT);
1186 
1187 	/* initialize port variables */
1188 	pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1189 
1190 	/* clear CPB fetch count */
1191 	writew(0, mmio + NV_ADMA_CPB_COUNT);
1192 
1193 	/* clear GO for register mode, enable interrupt */
1194 	tmp = readw(mmio + NV_ADMA_CTL);
1195 	writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1196 		NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1197 
1198 	tmp = readw(mmio + NV_ADMA_CTL);
1199 	writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1200 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1201 	udelay(1);
1202 	writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1203 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1204 
1205 	return 0;
1206 }
1207 
1208 static void nv_adma_port_stop(struct ata_port *ap)
1209 {
1210 	struct nv_adma_port_priv *pp = ap->private_data;
1211 	void __iomem *mmio = pp->ctl_block;
1212 
1213 	VPRINTK("ENTER\n");
1214 	writew(0, mmio + NV_ADMA_CTL);
1215 }
1216 
1217 #ifdef CONFIG_PM
1218 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1219 {
1220 	struct nv_adma_port_priv *pp = ap->private_data;
1221 	void __iomem *mmio = pp->ctl_block;
1222 
1223 	/* Go to register mode - clears GO */
1224 	nv_adma_register_mode(ap);
1225 
1226 	/* clear CPB fetch count */
1227 	writew(0, mmio + NV_ADMA_CPB_COUNT);
1228 
1229 	/* disable interrupt, shut down port */
1230 	writew(0, mmio + NV_ADMA_CTL);
1231 
1232 	return 0;
1233 }
1234 
1235 static int nv_adma_port_resume(struct ata_port *ap)
1236 {
1237 	struct nv_adma_port_priv *pp = ap->private_data;
1238 	void __iomem *mmio = pp->ctl_block;
1239 	u16 tmp;
1240 
1241 	/* set CPB block location */
1242 	writel(pp->cpb_dma & 0xFFFFFFFF, 	mmio + NV_ADMA_CPB_BASE_LOW);
1243 	writel((pp->cpb_dma >> 16) >> 16,	mmio + NV_ADMA_CPB_BASE_HIGH);
1244 
1245 	/* clear any outstanding interrupt conditions */
1246 	writew(0xffff, mmio + NV_ADMA_STAT);
1247 
1248 	/* initialize port variables */
1249 	pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1250 
1251 	/* clear CPB fetch count */
1252 	writew(0, mmio + NV_ADMA_CPB_COUNT);
1253 
1254 	/* clear GO for register mode, enable interrupt */
1255 	tmp = readw(mmio + NV_ADMA_CTL);
1256 	writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1257 		NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1258 
1259 	tmp = readw(mmio + NV_ADMA_CTL);
1260 	writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1261 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1262 	udelay(1);
1263 	writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1264 	readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1265 
1266 	return 0;
1267 }
1268 #endif
1269 
1270 static void nv_adma_setup_port(struct ata_port *ap)
1271 {
1272 	void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1273 	struct ata_ioports *ioport = &ap->ioaddr;
1274 
1275 	VPRINTK("ENTER\n");
1276 
1277 	mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1278 
1279 	ioport->cmd_addr	= mmio;
1280 	ioport->data_addr	= mmio + (ATA_REG_DATA * 4);
1281 	ioport->error_addr	=
1282 	ioport->feature_addr	= mmio + (ATA_REG_ERR * 4);
1283 	ioport->nsect_addr	= mmio + (ATA_REG_NSECT * 4);
1284 	ioport->lbal_addr	= mmio + (ATA_REG_LBAL * 4);
1285 	ioport->lbam_addr	= mmio + (ATA_REG_LBAM * 4);
1286 	ioport->lbah_addr	= mmio + (ATA_REG_LBAH * 4);
1287 	ioport->device_addr	= mmio + (ATA_REG_DEVICE * 4);
1288 	ioport->status_addr	=
1289 	ioport->command_addr	= mmio + (ATA_REG_STATUS * 4);
1290 	ioport->altstatus_addr	=
1291 	ioport->ctl_addr	= mmio + 0x20;
1292 }
1293 
1294 static int nv_adma_host_init(struct ata_host *host)
1295 {
1296 	struct pci_dev *pdev = to_pci_dev(host->dev);
1297 	unsigned int i;
1298 	u32 tmp32;
1299 
1300 	VPRINTK("ENTER\n");
1301 
1302 	/* enable ADMA on the ports */
1303 	pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1304 	tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1305 		 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1306 		 NV_MCP_SATA_CFG_20_PORT1_EN |
1307 		 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1308 
1309 	pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1310 
1311 	for (i = 0; i < host->n_ports; i++)
1312 		nv_adma_setup_port(host->ports[i]);
1313 
1314 	return 0;
1315 }
1316 
1317 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1318 			      struct scatterlist *sg,
1319 			      int idx,
1320 			      struct nv_adma_prd *aprd)
1321 {
1322 	u8 flags = 0;
1323 	if (qc->tf.flags & ATA_TFLAG_WRITE)
1324 		flags |= NV_APRD_WRITE;
1325 	if (idx == qc->n_elem - 1)
1326 		flags |= NV_APRD_END;
1327 	else if (idx != 4)
1328 		flags |= NV_APRD_CONT;
1329 
1330 	aprd->addr  = cpu_to_le64(((u64)sg_dma_address(sg)));
1331 	aprd->len   = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1332 	aprd->flags = flags;
1333 	aprd->packet_len = 0;
1334 }
1335 
1336 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1337 {
1338 	struct nv_adma_port_priv *pp = qc->ap->private_data;
1339 	struct nv_adma_prd *aprd;
1340 	struct scatterlist *sg;
1341 	unsigned int si;
1342 
1343 	VPRINTK("ENTER\n");
1344 
1345 	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1346 		aprd = (si < 5) ? &cpb->aprd[si] :
1347 			       &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1348 		nv_adma_fill_aprd(qc, sg, si, aprd);
1349 	}
1350 	if (si > 5)
1351 		cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1352 	else
1353 		cpb->next_aprd = cpu_to_le64(0);
1354 }
1355 
1356 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1357 {
1358 	struct nv_adma_port_priv *pp = qc->ap->private_data;
1359 
1360 	/* ADMA engine can only be used for non-ATAPI DMA commands,
1361 	   or interrupt-driven no-data commands. */
1362 	if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1363 	   (qc->tf.flags & ATA_TFLAG_POLLING))
1364 		return 1;
1365 
1366 	if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1367 	   (qc->tf.protocol == ATA_PROT_NODATA))
1368 		return 0;
1369 
1370 	return 1;
1371 }
1372 
1373 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1374 {
1375 	struct nv_adma_port_priv *pp = qc->ap->private_data;
1376 	struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1377 	u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1378 		       NV_CPB_CTL_IEN;
1379 
1380 	if (nv_adma_use_reg_mode(qc)) {
1381 		BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1382 			(qc->flags & ATA_QCFLAG_DMAMAP));
1383 		nv_adma_register_mode(qc->ap);
1384 		ata_qc_prep(qc);
1385 		return;
1386 	}
1387 
1388 	cpb->resp_flags = NV_CPB_RESP_DONE;
1389 	wmb();
1390 	cpb->ctl_flags = 0;
1391 	wmb();
1392 
1393 	cpb->len		= 3;
1394 	cpb->tag		= qc->tag;
1395 	cpb->next_cpb_idx	= 0;
1396 
1397 	/* turn on NCQ flags for NCQ commands */
1398 	if (qc->tf.protocol == ATA_PROT_NCQ)
1399 		ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1400 
1401 	VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1402 
1403 	nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1404 
1405 	if (qc->flags & ATA_QCFLAG_DMAMAP) {
1406 		nv_adma_fill_sg(qc, cpb);
1407 		ctl_flags |= NV_CPB_CTL_APRD_VALID;
1408 	} else
1409 		memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1410 
1411 	/* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1412 	   until we are finished filling in all of the contents */
1413 	wmb();
1414 	cpb->ctl_flags = ctl_flags;
1415 	wmb();
1416 	cpb->resp_flags = 0;
1417 }
1418 
1419 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1420 {
1421 	struct nv_adma_port_priv *pp = qc->ap->private_data;
1422 	void __iomem *mmio = pp->ctl_block;
1423 	int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1424 
1425 	VPRINTK("ENTER\n");
1426 
1427 	/* We can't handle result taskfile with NCQ commands, since
1428 	   retrieving the taskfile switches us out of ADMA mode and would abort
1429 	   existing commands. */
1430 	if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1431 		     (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1432 		ata_dev_printk(qc->dev, KERN_ERR,
1433 			"NCQ w/ RESULT_TF not allowed\n");
1434 		return AC_ERR_SYSTEM;
1435 	}
1436 
1437 	if (nv_adma_use_reg_mode(qc)) {
1438 		/* use ATA register mode */
1439 		VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1440 		BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1441 			(qc->flags & ATA_QCFLAG_DMAMAP));
1442 		nv_adma_register_mode(qc->ap);
1443 		return ata_qc_issue_prot(qc);
1444 	} else
1445 		nv_adma_mode(qc->ap);
1446 
1447 	/* write append register, command tag in lower 8 bits
1448 	   and (number of cpbs to append -1) in top 8 bits */
1449 	wmb();
1450 
1451 	if (curr_ncq != pp->last_issue_ncq) {
1452 		/* Seems to need some delay before switching between NCQ and
1453 		   non-NCQ commands, else we get command timeouts and such. */
1454 		udelay(20);
1455 		pp->last_issue_ncq = curr_ncq;
1456 	}
1457 
1458 	writew(qc->tag, mmio + NV_ADMA_APPEND);
1459 
1460 	DPRINTK("Issued tag %u\n", qc->tag);
1461 
1462 	return 0;
1463 }
1464 
1465 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1466 {
1467 	struct ata_host *host = dev_instance;
1468 	unsigned int i;
1469 	unsigned int handled = 0;
1470 	unsigned long flags;
1471 
1472 	spin_lock_irqsave(&host->lock, flags);
1473 
1474 	for (i = 0; i < host->n_ports; i++) {
1475 		struct ata_port *ap;
1476 
1477 		ap = host->ports[i];
1478 		if (ap &&
1479 		    !(ap->flags & ATA_FLAG_DISABLED)) {
1480 			struct ata_queued_cmd *qc;
1481 
1482 			qc = ata_qc_from_tag(ap, ap->link.active_tag);
1483 			if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1484 				handled += ata_host_intr(ap, qc);
1485 			else
1486 				// No request pending?  Clear interrupt status
1487 				// anyway, in case there's one pending.
1488 				ap->ops->check_status(ap);
1489 		}
1490 
1491 	}
1492 
1493 	spin_unlock_irqrestore(&host->lock, flags);
1494 
1495 	return IRQ_RETVAL(handled);
1496 }
1497 
1498 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1499 {
1500 	int i, handled = 0;
1501 
1502 	for (i = 0; i < host->n_ports; i++) {
1503 		struct ata_port *ap = host->ports[i];
1504 
1505 		if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1506 			handled += nv_host_intr(ap, irq_stat);
1507 
1508 		irq_stat >>= NV_INT_PORT_SHIFT;
1509 	}
1510 
1511 	return IRQ_RETVAL(handled);
1512 }
1513 
1514 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1515 {
1516 	struct ata_host *host = dev_instance;
1517 	u8 irq_stat;
1518 	irqreturn_t ret;
1519 
1520 	spin_lock(&host->lock);
1521 	irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1522 	ret = nv_do_interrupt(host, irq_stat);
1523 	spin_unlock(&host->lock);
1524 
1525 	return ret;
1526 }
1527 
1528 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1529 {
1530 	struct ata_host *host = dev_instance;
1531 	u8 irq_stat;
1532 	irqreturn_t ret;
1533 
1534 	spin_lock(&host->lock);
1535 	irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1536 	ret = nv_do_interrupt(host, irq_stat);
1537 	spin_unlock(&host->lock);
1538 
1539 	return ret;
1540 }
1541 
1542 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1543 {
1544 	if (sc_reg > SCR_CONTROL)
1545 		return -EINVAL;
1546 
1547 	*val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1548 	return 0;
1549 }
1550 
1551 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1552 {
1553 	if (sc_reg > SCR_CONTROL)
1554 		return -EINVAL;
1555 
1556 	iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
1557 	return 0;
1558 }
1559 
1560 static void nv_nf2_freeze(struct ata_port *ap)
1561 {
1562 	void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1563 	int shift = ap->port_no * NV_INT_PORT_SHIFT;
1564 	u8 mask;
1565 
1566 	mask = ioread8(scr_addr + NV_INT_ENABLE);
1567 	mask &= ~(NV_INT_ALL << shift);
1568 	iowrite8(mask, scr_addr + NV_INT_ENABLE);
1569 }
1570 
1571 static void nv_nf2_thaw(struct ata_port *ap)
1572 {
1573 	void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1574 	int shift = ap->port_no * NV_INT_PORT_SHIFT;
1575 	u8 mask;
1576 
1577 	iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1578 
1579 	mask = ioread8(scr_addr + NV_INT_ENABLE);
1580 	mask |= (NV_INT_MASK << shift);
1581 	iowrite8(mask, scr_addr + NV_INT_ENABLE);
1582 }
1583 
1584 static void nv_ck804_freeze(struct ata_port *ap)
1585 {
1586 	void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1587 	int shift = ap->port_no * NV_INT_PORT_SHIFT;
1588 	u8 mask;
1589 
1590 	mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1591 	mask &= ~(NV_INT_ALL << shift);
1592 	writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1593 }
1594 
1595 static void nv_ck804_thaw(struct ata_port *ap)
1596 {
1597 	void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1598 	int shift = ap->port_no * NV_INT_PORT_SHIFT;
1599 	u8 mask;
1600 
1601 	writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1602 
1603 	mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1604 	mask |= (NV_INT_MASK << shift);
1605 	writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1606 }
1607 
1608 static void nv_mcp55_freeze(struct ata_port *ap)
1609 {
1610 	void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1611 	int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1612 	u32 mask;
1613 
1614 	writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1615 
1616 	mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1617 	mask &= ~(NV_INT_ALL_MCP55 << shift);
1618 	writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1619 	ata_bmdma_freeze(ap);
1620 }
1621 
1622 static void nv_mcp55_thaw(struct ata_port *ap)
1623 {
1624 	void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1625 	int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1626 	u32 mask;
1627 
1628 	writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1629 
1630 	mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1631 	mask |= (NV_INT_MASK_MCP55 << shift);
1632 	writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1633 	ata_bmdma_thaw(ap);
1634 }
1635 
1636 static int nv_hardreset(struct ata_link *link, unsigned int *class,
1637 			unsigned long deadline)
1638 {
1639 	unsigned int dummy;
1640 
1641 	/* SATA hardreset fails to retrieve proper device signature on
1642 	 * some controllers.  Don't classify on hardreset.  For more
1643 	 * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
1644 	 */
1645 	return sata_std_hardreset(link, &dummy, deadline);
1646 }
1647 
1648 static void nv_error_handler(struct ata_port *ap)
1649 {
1650 	ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1651 			   nv_hardreset, ata_std_postreset);
1652 }
1653 
1654 static void nv_adma_error_handler(struct ata_port *ap)
1655 {
1656 	struct nv_adma_port_priv *pp = ap->private_data;
1657 	if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1658 		void __iomem *mmio = pp->ctl_block;
1659 		int i;
1660 		u16 tmp;
1661 
1662 		if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1663 			u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1664 			u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1665 			u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1666 			u32 status = readw(mmio + NV_ADMA_STAT);
1667 			u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1668 			u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1669 
1670 			ata_port_printk(ap, KERN_ERR,
1671 				"EH in ADMA mode, notifier 0x%X "
1672 				"notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1673 				"next cpb count 0x%X next cpb idx 0x%x\n",
1674 				notifier, notifier_error, gen_ctl, status,
1675 				cpb_count, next_cpb_idx);
1676 
1677 			for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1678 				struct nv_adma_cpb *cpb = &pp->cpb[i];
1679 				if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1680 				    ap->link.sactive & (1 << i))
1681 					ata_port_printk(ap, KERN_ERR,
1682 						"CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1683 						i, cpb->ctl_flags, cpb->resp_flags);
1684 			}
1685 		}
1686 
1687 		/* Push us back into port register mode for error handling. */
1688 		nv_adma_register_mode(ap);
1689 
1690 		/* Mark all of the CPBs as invalid to prevent them from
1691 		   being executed */
1692 		for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1693 			pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1694 
1695 		/* clear CPB fetch count */
1696 		writew(0, mmio + NV_ADMA_CPB_COUNT);
1697 
1698 		/* Reset channel */
1699 		tmp = readw(mmio + NV_ADMA_CTL);
1700 		writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1701 		readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1702 		udelay(1);
1703 		writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1704 		readw(mmio + NV_ADMA_CTL);	/* flush posted write */
1705 	}
1706 
1707 	ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1708 			   nv_hardreset, ata_std_postreset);
1709 }
1710 
1711 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1712 {
1713 	struct nv_swncq_port_priv *pp = ap->private_data;
1714 	struct defer_queue *dq = &pp->defer_queue;
1715 
1716 	/* queue is full */
1717 	WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1718 	dq->defer_bits |= (1 << qc->tag);
1719 	dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1720 }
1721 
1722 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1723 {
1724 	struct nv_swncq_port_priv *pp = ap->private_data;
1725 	struct defer_queue *dq = &pp->defer_queue;
1726 	unsigned int tag;
1727 
1728 	if (dq->head == dq->tail)	/* null queue */
1729 		return NULL;
1730 
1731 	tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1732 	dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1733 	WARN_ON(!(dq->defer_bits & (1 << tag)));
1734 	dq->defer_bits &= ~(1 << tag);
1735 
1736 	return ata_qc_from_tag(ap, tag);
1737 }
1738 
1739 static void nv_swncq_fis_reinit(struct ata_port *ap)
1740 {
1741 	struct nv_swncq_port_priv *pp = ap->private_data;
1742 
1743 	pp->dhfis_bits = 0;
1744 	pp->dmafis_bits = 0;
1745 	pp->sdbfis_bits = 0;
1746 	pp->ncq_flags = 0;
1747 }
1748 
1749 static void nv_swncq_pp_reinit(struct ata_port *ap)
1750 {
1751 	struct nv_swncq_port_priv *pp = ap->private_data;
1752 	struct defer_queue *dq = &pp->defer_queue;
1753 
1754 	dq->head = 0;
1755 	dq->tail = 0;
1756 	dq->defer_bits = 0;
1757 	pp->qc_active = 0;
1758 	pp->last_issue_tag = ATA_TAG_POISON;
1759 	nv_swncq_fis_reinit(ap);
1760 }
1761 
1762 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1763 {
1764 	struct nv_swncq_port_priv *pp = ap->private_data;
1765 
1766 	writew(fis, pp->irq_block);
1767 }
1768 
1769 static void __ata_bmdma_stop(struct ata_port *ap)
1770 {
1771 	struct ata_queued_cmd qc;
1772 
1773 	qc.ap = ap;
1774 	ata_bmdma_stop(&qc);
1775 }
1776 
1777 static void nv_swncq_ncq_stop(struct ata_port *ap)
1778 {
1779 	struct nv_swncq_port_priv *pp = ap->private_data;
1780 	unsigned int i;
1781 	u32 sactive;
1782 	u32 done_mask;
1783 
1784 	ata_port_printk(ap, KERN_ERR,
1785 			"EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1786 			ap->qc_active, ap->link.sactive);
1787 	ata_port_printk(ap, KERN_ERR,
1788 		"SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n  "
1789 		"dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1790 		pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1791 		pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1792 
1793 	ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1794 			ap->ops->check_status(ap),
1795 			ioread8(ap->ioaddr.error_addr));
1796 
1797 	sactive = readl(pp->sactive_block);
1798 	done_mask = pp->qc_active ^ sactive;
1799 
1800 	ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1801 	for (i = 0; i < ATA_MAX_QUEUE; i++) {
1802 		u8 err = 0;
1803 		if (pp->qc_active & (1 << i))
1804 			err = 0;
1805 		else if (done_mask & (1 << i))
1806 			err = 1;
1807 		else
1808 			continue;
1809 
1810 		ata_port_printk(ap, KERN_ERR,
1811 				"tag 0x%x: %01x %01x %01x %01x %s\n", i,
1812 				(pp->dhfis_bits >> i) & 0x1,
1813 				(pp->dmafis_bits >> i) & 0x1,
1814 				(pp->sdbfis_bits >> i) & 0x1,
1815 				(sactive >> i) & 0x1,
1816 				(err ? "error! tag doesn't exit" : " "));
1817 	}
1818 
1819 	nv_swncq_pp_reinit(ap);
1820 	ap->ops->irq_clear(ap);
1821 	__ata_bmdma_stop(ap);
1822 	nv_swncq_irq_clear(ap, 0xffff);
1823 }
1824 
1825 static void nv_swncq_error_handler(struct ata_port *ap)
1826 {
1827 	struct ata_eh_context *ehc = &ap->link.eh_context;
1828 
1829 	if (ap->link.sactive) {
1830 		nv_swncq_ncq_stop(ap);
1831 		ehc->i.action |= ATA_EH_HARDRESET;
1832 	}
1833 
1834 	ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1835 			   nv_hardreset, ata_std_postreset);
1836 }
1837 
1838 #ifdef CONFIG_PM
1839 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1840 {
1841 	void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1842 	u32 tmp;
1843 
1844 	/* clear irq */
1845 	writel(~0, mmio + NV_INT_STATUS_MCP55);
1846 
1847 	/* disable irq */
1848 	writel(0, mmio + NV_INT_ENABLE_MCP55);
1849 
1850 	/* disable swncq */
1851 	tmp = readl(mmio + NV_CTL_MCP55);
1852 	tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1853 	writel(tmp, mmio + NV_CTL_MCP55);
1854 
1855 	return 0;
1856 }
1857 
1858 static int nv_swncq_port_resume(struct ata_port *ap)
1859 {
1860 	void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1861 	u32 tmp;
1862 
1863 	/* clear irq */
1864 	writel(~0, mmio + NV_INT_STATUS_MCP55);
1865 
1866 	/* enable irq */
1867 	writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1868 
1869 	/* enable swncq */
1870 	tmp = readl(mmio + NV_CTL_MCP55);
1871 	writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1872 
1873 	return 0;
1874 }
1875 #endif
1876 
1877 static void nv_swncq_host_init(struct ata_host *host)
1878 {
1879 	u32 tmp;
1880 	void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1881 	struct pci_dev *pdev = to_pci_dev(host->dev);
1882 	u8 regval;
1883 
1884 	/* disable  ECO 398 */
1885 	pci_read_config_byte(pdev, 0x7f, &regval);
1886 	regval &= ~(1 << 7);
1887 	pci_write_config_byte(pdev, 0x7f, regval);
1888 
1889 	/* enable swncq */
1890 	tmp = readl(mmio + NV_CTL_MCP55);
1891 	VPRINTK("HOST_CTL:0x%X\n", tmp);
1892 	writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1893 
1894 	/* enable irq intr */
1895 	tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1896 	VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1897 	writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1898 
1899 	/*  clear port irq */
1900 	writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1901 }
1902 
1903 static int nv_swncq_slave_config(struct scsi_device *sdev)
1904 {
1905 	struct ata_port *ap = ata_shost_to_port(sdev->host);
1906 	struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1907 	struct ata_device *dev;
1908 	int rc;
1909 	u8 rev;
1910 	u8 check_maxtor = 0;
1911 	unsigned char model_num[ATA_ID_PROD_LEN + 1];
1912 
1913 	rc = ata_scsi_slave_config(sdev);
1914 	if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1915 		/* Not a proper libata device, ignore */
1916 		return rc;
1917 
1918 	dev = &ap->link.device[sdev->id];
1919 	if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1920 		return rc;
1921 
1922 	/* if MCP51 and Maxtor, then disable ncq */
1923 	if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1924 		pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1925 		check_maxtor = 1;
1926 
1927 	/* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1928 	if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1929 		pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1930 		pci_read_config_byte(pdev, 0x8, &rev);
1931 		if (rev <= 0xa2)
1932 			check_maxtor = 1;
1933 	}
1934 
1935 	if (!check_maxtor)
1936 		return rc;
1937 
1938 	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1939 
1940 	if (strncmp(model_num, "Maxtor", 6) == 0) {
1941 		ata_scsi_change_queue_depth(sdev, 1);
1942 		ata_dev_printk(dev, KERN_NOTICE,
1943 			"Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1944 	}
1945 
1946 	return rc;
1947 }
1948 
1949 static int nv_swncq_port_start(struct ata_port *ap)
1950 {
1951 	struct device *dev = ap->host->dev;
1952 	void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1953 	struct nv_swncq_port_priv *pp;
1954 	int rc;
1955 
1956 	rc = ata_port_start(ap);
1957 	if (rc)
1958 		return rc;
1959 
1960 	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1961 	if (!pp)
1962 		return -ENOMEM;
1963 
1964 	pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1965 				      &pp->prd_dma, GFP_KERNEL);
1966 	if (!pp->prd)
1967 		return -ENOMEM;
1968 	memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1969 
1970 	ap->private_data = pp;
1971 	pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1972 	pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1973 	pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1974 
1975 	return 0;
1976 }
1977 
1978 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1979 {
1980 	if (qc->tf.protocol != ATA_PROT_NCQ) {
1981 		ata_qc_prep(qc);
1982 		return;
1983 	}
1984 
1985 	if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1986 		return;
1987 
1988 	nv_swncq_fill_sg(qc);
1989 }
1990 
1991 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1992 {
1993 	struct ata_port *ap = qc->ap;
1994 	struct scatterlist *sg;
1995 	struct nv_swncq_port_priv *pp = ap->private_data;
1996 	struct ata_prd *prd;
1997 	unsigned int si, idx;
1998 
1999 	prd = pp->prd + ATA_MAX_PRD * qc->tag;
2000 
2001 	idx = 0;
2002 	for_each_sg(qc->sg, sg, qc->n_elem, si) {
2003 		u32 addr, offset;
2004 		u32 sg_len, len;
2005 
2006 		addr = (u32)sg_dma_address(sg);
2007 		sg_len = sg_dma_len(sg);
2008 
2009 		while (sg_len) {
2010 			offset = addr & 0xffff;
2011 			len = sg_len;
2012 			if ((offset + sg_len) > 0x10000)
2013 				len = 0x10000 - offset;
2014 
2015 			prd[idx].addr = cpu_to_le32(addr);
2016 			prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2017 
2018 			idx++;
2019 			sg_len -= len;
2020 			addr += len;
2021 		}
2022 	}
2023 
2024 	prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2025 }
2026 
2027 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2028 					  struct ata_queued_cmd *qc)
2029 {
2030 	struct nv_swncq_port_priv *pp = ap->private_data;
2031 
2032 	if (qc == NULL)
2033 		return 0;
2034 
2035 	DPRINTK("Enter\n");
2036 
2037 	writel((1 << qc->tag), pp->sactive_block);
2038 	pp->last_issue_tag = qc->tag;
2039 	pp->dhfis_bits &= ~(1 << qc->tag);
2040 	pp->dmafis_bits &= ~(1 << qc->tag);
2041 	pp->qc_active |= (0x1 << qc->tag);
2042 
2043 	ap->ops->tf_load(ap, &qc->tf);	 /* load tf registers */
2044 	ap->ops->exec_command(ap, &qc->tf);
2045 
2046 	DPRINTK("Issued tag %u\n", qc->tag);
2047 
2048 	return 0;
2049 }
2050 
2051 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2052 {
2053 	struct ata_port *ap = qc->ap;
2054 	struct nv_swncq_port_priv *pp = ap->private_data;
2055 
2056 	if (qc->tf.protocol != ATA_PROT_NCQ)
2057 		return ata_qc_issue_prot(qc);
2058 
2059 	DPRINTK("Enter\n");
2060 
2061 	if (!pp->qc_active)
2062 		nv_swncq_issue_atacmd(ap, qc);
2063 	else
2064 		nv_swncq_qc_to_dq(ap, qc);	/* add qc to defer queue */
2065 
2066 	return 0;
2067 }
2068 
2069 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2070 {
2071 	u32 serror;
2072 	struct ata_eh_info *ehi = &ap->link.eh_info;
2073 
2074 	ata_ehi_clear_desc(ehi);
2075 
2076 	/* AHCI needs SError cleared; otherwise, it might lock up */
2077 	sata_scr_read(&ap->link, SCR_ERROR, &serror);
2078 	sata_scr_write(&ap->link, SCR_ERROR, serror);
2079 
2080 	/* analyze @irq_stat */
2081 	if (fis & NV_SWNCQ_IRQ_ADDED)
2082 		ata_ehi_push_desc(ehi, "hot plug");
2083 	else if (fis & NV_SWNCQ_IRQ_REMOVED)
2084 		ata_ehi_push_desc(ehi, "hot unplug");
2085 
2086 	ata_ehi_hotplugged(ehi);
2087 
2088 	/* okay, let's hand over to EH */
2089 	ehi->serror |= serror;
2090 
2091 	ata_port_freeze(ap);
2092 }
2093 
2094 static int nv_swncq_sdbfis(struct ata_port *ap)
2095 {
2096 	struct ata_queued_cmd *qc;
2097 	struct nv_swncq_port_priv *pp = ap->private_data;
2098 	struct ata_eh_info *ehi = &ap->link.eh_info;
2099 	u32 sactive;
2100 	int nr_done = 0;
2101 	u32 done_mask;
2102 	int i;
2103 	u8 host_stat;
2104 	u8 lack_dhfis = 0;
2105 
2106 	host_stat = ap->ops->bmdma_status(ap);
2107 	if (unlikely(host_stat & ATA_DMA_ERR)) {
2108 		/* error when transfering data to/from memory */
2109 		ata_ehi_clear_desc(ehi);
2110 		ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2111 		ehi->err_mask |= AC_ERR_HOST_BUS;
2112 		ehi->action |= ATA_EH_SOFTRESET;
2113 		return -EINVAL;
2114 	}
2115 
2116 	ap->ops->irq_clear(ap);
2117 	__ata_bmdma_stop(ap);
2118 
2119 	sactive = readl(pp->sactive_block);
2120 	done_mask = pp->qc_active ^ sactive;
2121 
2122 	if (unlikely(done_mask & sactive)) {
2123 		ata_ehi_clear_desc(ehi);
2124 		ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2125 				  "(%08x->%08x)", pp->qc_active, sactive);
2126 		ehi->err_mask |= AC_ERR_HSM;
2127 		ehi->action |= ATA_EH_HARDRESET;
2128 		return -EINVAL;
2129 	}
2130 	for (i = 0; i < ATA_MAX_QUEUE; i++) {
2131 		if (!(done_mask & (1 << i)))
2132 			continue;
2133 
2134 		qc = ata_qc_from_tag(ap, i);
2135 		if (qc) {
2136 			ata_qc_complete(qc);
2137 			pp->qc_active &= ~(1 << i);
2138 			pp->dhfis_bits &= ~(1 << i);
2139 			pp->dmafis_bits &= ~(1 << i);
2140 			pp->sdbfis_bits |= (1 << i);
2141 			nr_done++;
2142 		}
2143 	}
2144 
2145 	if (!ap->qc_active) {
2146 		DPRINTK("over\n");
2147 		nv_swncq_pp_reinit(ap);
2148 		return nr_done;
2149 	}
2150 
2151 	if (pp->qc_active & pp->dhfis_bits)
2152 		return nr_done;
2153 
2154 	if ((pp->ncq_flags & ncq_saw_backout) ||
2155 	    (pp->qc_active ^ pp->dhfis_bits))
2156 		/* if the controller cann't get a device to host register FIS,
2157 		 * The driver needs to reissue the new command.
2158 		 */
2159 		lack_dhfis = 1;
2160 
2161 	DPRINTK("id 0x%x QC: qc_active 0x%x,"
2162 		"SWNCQ:qc_active 0x%X defer_bits %X "
2163 		"dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2164 		ap->print_id, ap->qc_active, pp->qc_active,
2165 		pp->defer_queue.defer_bits, pp->dhfis_bits,
2166 		pp->dmafis_bits, pp->last_issue_tag);
2167 
2168 	nv_swncq_fis_reinit(ap);
2169 
2170 	if (lack_dhfis) {
2171 		qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2172 		nv_swncq_issue_atacmd(ap, qc);
2173 		return nr_done;
2174 	}
2175 
2176 	if (pp->defer_queue.defer_bits) {
2177 		/* send deferral queue command */
2178 		qc = nv_swncq_qc_from_dq(ap);
2179 		WARN_ON(qc == NULL);
2180 		nv_swncq_issue_atacmd(ap, qc);
2181 	}
2182 
2183 	return nr_done;
2184 }
2185 
2186 static inline u32 nv_swncq_tag(struct ata_port *ap)
2187 {
2188 	struct nv_swncq_port_priv *pp = ap->private_data;
2189 	u32 tag;
2190 
2191 	tag = readb(pp->tag_block) >> 2;
2192 	return (tag & 0x1f);
2193 }
2194 
2195 static int nv_swncq_dmafis(struct ata_port *ap)
2196 {
2197 	struct ata_queued_cmd *qc;
2198 	unsigned int rw;
2199 	u8 dmactl;
2200 	u32 tag;
2201 	struct nv_swncq_port_priv *pp = ap->private_data;
2202 
2203 	__ata_bmdma_stop(ap);
2204 	tag = nv_swncq_tag(ap);
2205 
2206 	DPRINTK("dma setup tag 0x%x\n", tag);
2207 	qc = ata_qc_from_tag(ap, tag);
2208 
2209 	if (unlikely(!qc))
2210 		return 0;
2211 
2212 	rw = qc->tf.flags & ATA_TFLAG_WRITE;
2213 
2214 	/* load PRD table addr. */
2215 	iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2216 		  ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2217 
2218 	/* specify data direction, triple-check start bit is clear */
2219 	dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2220 	dmactl &= ~ATA_DMA_WR;
2221 	if (!rw)
2222 		dmactl |= ATA_DMA_WR;
2223 
2224 	iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2225 
2226 	return 1;
2227 }
2228 
2229 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2230 {
2231 	struct nv_swncq_port_priv *pp = ap->private_data;
2232 	struct ata_queued_cmd *qc;
2233 	struct ata_eh_info *ehi = &ap->link.eh_info;
2234 	u32 serror;
2235 	u8 ata_stat;
2236 	int rc = 0;
2237 
2238 	ata_stat = ap->ops->check_status(ap);
2239 	nv_swncq_irq_clear(ap, fis);
2240 	if (!fis)
2241 		return;
2242 
2243 	if (ap->pflags & ATA_PFLAG_FROZEN)
2244 		return;
2245 
2246 	if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2247 		nv_swncq_hotplug(ap, fis);
2248 		return;
2249 	}
2250 
2251 	if (!pp->qc_active)
2252 		return;
2253 
2254 	if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2255 		return;
2256 	ap->ops->scr_write(ap, SCR_ERROR, serror);
2257 
2258 	if (ata_stat & ATA_ERR) {
2259 		ata_ehi_clear_desc(ehi);
2260 		ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2261 		ehi->err_mask |= AC_ERR_DEV;
2262 		ehi->serror |= serror;
2263 		ehi->action |= ATA_EH_SOFTRESET;
2264 		ata_port_freeze(ap);
2265 		return;
2266 	}
2267 
2268 	if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2269 		/* If the IRQ is backout, driver must issue
2270 		 * the new command again some time later.
2271 		 */
2272 		pp->ncq_flags |= ncq_saw_backout;
2273 	}
2274 
2275 	if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2276 		pp->ncq_flags |= ncq_saw_sdb;
2277 		DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2278 			"dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2279 			ap->print_id, pp->qc_active, pp->dhfis_bits,
2280 			pp->dmafis_bits, readl(pp->sactive_block));
2281 		rc = nv_swncq_sdbfis(ap);
2282 		if (rc < 0)
2283 			goto irq_error;
2284 	}
2285 
2286 	if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2287 		/* The interrupt indicates the new command
2288 		 * was transmitted correctly to the drive.
2289 		 */
2290 		pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2291 		pp->ncq_flags |= ncq_saw_d2h;
2292 		if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2293 			ata_ehi_push_desc(ehi, "illegal fis transaction");
2294 			ehi->err_mask |= AC_ERR_HSM;
2295 			ehi->action |= ATA_EH_HARDRESET;
2296 			goto irq_error;
2297 		}
2298 
2299 		if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2300 		    !(pp->ncq_flags & ncq_saw_dmas)) {
2301 			ata_stat = ap->ops->check_status(ap);
2302 			if (ata_stat & ATA_BUSY)
2303 				goto irq_exit;
2304 
2305 			if (pp->defer_queue.defer_bits) {
2306 				DPRINTK("send next command\n");
2307 				qc = nv_swncq_qc_from_dq(ap);
2308 				nv_swncq_issue_atacmd(ap, qc);
2309 			}
2310 		}
2311 	}
2312 
2313 	if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2314 		/* program the dma controller with appropriate PRD buffers
2315 		 * and start the DMA transfer for requested command.
2316 		 */
2317 		pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2318 		pp->ncq_flags |= ncq_saw_dmas;
2319 		rc = nv_swncq_dmafis(ap);
2320 	}
2321 
2322 irq_exit:
2323 	return;
2324 irq_error:
2325 	ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2326 	ata_port_freeze(ap);
2327 	return;
2328 }
2329 
2330 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2331 {
2332 	struct ata_host *host = dev_instance;
2333 	unsigned int i;
2334 	unsigned int handled = 0;
2335 	unsigned long flags;
2336 	u32 irq_stat;
2337 
2338 	spin_lock_irqsave(&host->lock, flags);
2339 
2340 	irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2341 
2342 	for (i = 0; i < host->n_ports; i++) {
2343 		struct ata_port *ap = host->ports[i];
2344 
2345 		if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2346 			if (ap->link.sactive) {
2347 				nv_swncq_host_interrupt(ap, (u16)irq_stat);
2348 				handled = 1;
2349 			} else {
2350 				if (irq_stat)	/* reserve Hotplug */
2351 					nv_swncq_irq_clear(ap, 0xfff0);
2352 
2353 				handled += nv_host_intr(ap, (u8)irq_stat);
2354 			}
2355 		}
2356 		irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2357 	}
2358 
2359 	spin_unlock_irqrestore(&host->lock, flags);
2360 
2361 	return IRQ_RETVAL(handled);
2362 }
2363 
2364 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2365 {
2366 	static int printed_version;
2367 	const struct ata_port_info *ppi[] = { NULL, NULL };
2368 	struct ata_host *host;
2369 	struct nv_host_priv *hpriv;
2370 	int rc;
2371 	u32 bar;
2372 	void __iomem *base;
2373 	unsigned long type = ent->driver_data;
2374 
2375         // Make sure this is a SATA controller by counting the number of bars
2376         // (NVIDIA SATA controllers will always have six bars).  Otherwise,
2377         // it's an IDE controller and we ignore it.
2378 	for (bar = 0; bar < 6; bar++)
2379 		if (pci_resource_start(pdev, bar) == 0)
2380 			return -ENODEV;
2381 
2382 	if (!printed_version++)
2383 		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
2384 
2385 	rc = pcim_enable_device(pdev);
2386 	if (rc)
2387 		return rc;
2388 
2389 	/* determine type and allocate host */
2390 	if (type == CK804 && adma_enabled) {
2391 		dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2392 		type = ADMA;
2393 	}
2394 
2395 	if (type == SWNCQ) {
2396 		if (swncq_enabled)
2397 			dev_printk(KERN_NOTICE, &pdev->dev,
2398 				   "Using SWNCQ mode\n");
2399 		else
2400 			type = GENERIC;
2401 	}
2402 
2403 	ppi[0] = &nv_port_info[type];
2404 	rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
2405 	if (rc)
2406 		return rc;
2407 
2408 	hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2409 	if (!hpriv)
2410 		return -ENOMEM;
2411 	hpriv->type = type;
2412 	host->private_data = hpriv;
2413 
2414 	/* set 64bit dma masks, may fail */
2415 	if (type == ADMA) {
2416 		if (pci_set_dma_mask(pdev, DMA_64BIT_MASK) == 0)
2417 			pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2418 	}
2419 
2420 	/* request and iomap NV_MMIO_BAR */
2421 	rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2422 	if (rc)
2423 		return rc;
2424 
2425 	/* configure SCR access */
2426 	base = host->iomap[NV_MMIO_BAR];
2427 	host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2428 	host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2429 
2430 	/* enable SATA space for CK804 */
2431 	if (type >= CK804) {
2432 		u8 regval;
2433 
2434 		pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2435 		regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2436 		pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2437 	}
2438 
2439 	/* init ADMA */
2440 	if (type == ADMA) {
2441 		rc = nv_adma_host_init(host);
2442 		if (rc)
2443 			return rc;
2444 	} else if (type == SWNCQ)
2445 		nv_swncq_host_init(host);
2446 
2447 	pci_set_master(pdev);
2448 	return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2449 				 IRQF_SHARED, ppi[0]->sht);
2450 }
2451 
2452 #ifdef CONFIG_PM
2453 static int nv_pci_device_resume(struct pci_dev *pdev)
2454 {
2455 	struct ata_host *host = dev_get_drvdata(&pdev->dev);
2456 	struct nv_host_priv *hpriv = host->private_data;
2457 	int rc;
2458 
2459 	rc = ata_pci_device_do_resume(pdev);
2460 	if (rc)
2461 		return rc;
2462 
2463 	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2464 		if (hpriv->type >= CK804) {
2465 			u8 regval;
2466 
2467 			pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2468 			regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2469 			pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2470 		}
2471 		if (hpriv->type == ADMA) {
2472 			u32 tmp32;
2473 			struct nv_adma_port_priv *pp;
2474 			/* enable/disable ADMA on the ports appropriately */
2475 			pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2476 
2477 			pp = host->ports[0]->private_data;
2478 			if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2479 				tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2480 					   NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2481 			else
2482 				tmp32 |=  (NV_MCP_SATA_CFG_20_PORT0_EN |
2483 					   NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2484 			pp = host->ports[1]->private_data;
2485 			if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2486 				tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2487 					   NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2488 			else
2489 				tmp32 |=  (NV_MCP_SATA_CFG_20_PORT1_EN |
2490 					   NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2491 
2492 			pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2493 		}
2494 	}
2495 
2496 	ata_host_resume(host);
2497 
2498 	return 0;
2499 }
2500 #endif
2501 
2502 static void nv_ck804_host_stop(struct ata_host *host)
2503 {
2504 	struct pci_dev *pdev = to_pci_dev(host->dev);
2505 	u8 regval;
2506 
2507 	/* disable SATA space for CK804 */
2508 	pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2509 	regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2510 	pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2511 }
2512 
2513 static void nv_adma_host_stop(struct ata_host *host)
2514 {
2515 	struct pci_dev *pdev = to_pci_dev(host->dev);
2516 	u32 tmp32;
2517 
2518 	/* disable ADMA on the ports */
2519 	pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2520 	tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2521 		   NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2522 		   NV_MCP_SATA_CFG_20_PORT1_EN |
2523 		   NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2524 
2525 	pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2526 
2527 	nv_ck804_host_stop(host);
2528 }
2529 
2530 static int __init nv_init(void)
2531 {
2532 	return pci_register_driver(&nv_pci_driver);
2533 }
2534 
2535 static void __exit nv_exit(void)
2536 {
2537 	pci_unregister_driver(&nv_pci_driver);
2538 }
2539 
2540 module_init(nv_init);
2541 module_exit(nv_exit);
2542 module_param_named(adma, adma_enabled, bool, 0444);
2543 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
2544 module_param_named(swncq, swncq_enabled, bool, 0444);
2545 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2546 
2547