xref: /linux/drivers/block/mtip32xx/mtip32xx.c (revision 680e6ffa15103ab610c0fc1241d2f98c801b13e2)
1 /*
2  * Driver for the Micron P320 SSD
3  *   Copyright (C) 2011 Micron Technology, Inc.
4  *
5  * Portions of this code were derived from works subjected to the
6  * following copyright:
7  *    Copyright (C) 2009 Integrated Device Technology, Inc.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  */
20 
21 #include <linux/pci.h>
22 #include <linux/interrupt.h>
23 #include <linux/ata.h>
24 #include <linux/delay.h>
25 #include <linux/hdreg.h>
26 #include <linux/uaccess.h>
27 #include <linux/random.h>
28 #include <linux/smp.h>
29 #include <linux/compat.h>
30 #include <linux/fs.h>
31 #include <linux/module.h>
32 #include <linux/genhd.h>
33 #include <linux/blkdev.h>
34 #include <linux/blk-mq.h>
35 #include <linux/bio.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/idr.h>
38 #include <linux/kthread.h>
39 #include <../drivers/ata/ahci.h>
40 #include <linux/export.h>
41 #include <linux/debugfs.h>
42 #include <linux/prefetch.h>
43 #include <linux/numa.h>
44 #include "mtip32xx.h"
45 
46 #define HW_CMD_SLOT_SZ		(MTIP_MAX_COMMAND_SLOTS * 32)
47 
48 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
49 #define AHCI_RX_FIS_SZ          0x100
50 #define AHCI_RX_FIS_OFFSET      0x0
51 #define AHCI_IDFY_SZ            ATA_SECT_SIZE
52 #define AHCI_IDFY_OFFSET        0x400
53 #define AHCI_SECTBUF_SZ         ATA_SECT_SIZE
54 #define AHCI_SECTBUF_OFFSET     0x800
55 #define AHCI_SMARTBUF_SZ        ATA_SECT_SIZE
56 #define AHCI_SMARTBUF_OFFSET    0xC00
57 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */
58 #define BLOCK_DMA_ALLOC_SZ      4096
59 
60 /* DMA region containing command table (should be 8192 bytes) */
61 #define AHCI_CMD_SLOT_SZ        sizeof(struct mtip_cmd_hdr)
62 #define AHCI_CMD_TBL_SZ         (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ)
63 #define AHCI_CMD_TBL_OFFSET     0x0
64 
65 /* DMA region per command (contains header and SGL) */
66 #define AHCI_CMD_TBL_HDR_SZ     0x80
67 #define AHCI_CMD_TBL_HDR_OFFSET 0x0
68 #define AHCI_CMD_TBL_SGL_SZ     (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg))
69 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ
70 #define CMD_DMA_ALLOC_SZ        (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ)
71 
72 
73 #define HOST_CAP_NZDMA		(1 << 19)
74 #define HOST_HSORG		0xFC
75 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
76 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
77 #define HSORG_HWREV		0xFF00
78 #define HSORG_STYLE		0x8
79 #define HSORG_SLOTGROUPS	0x7
80 
81 #define PORT_COMMAND_ISSUE	0x38
82 #define PORT_SDBV		0x7C
83 
84 #define PORT_OFFSET		0x100
85 #define PORT_MEM_SIZE		0x80
86 
87 #define PORT_IRQ_ERR \
88 	(PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
89 	 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
90 	 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
91 	 PORT_IRQ_OVERFLOW)
92 #define PORT_IRQ_LEGACY \
93 	(PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
94 #define PORT_IRQ_HANDLED \
95 	(PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
96 	 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
97 	 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
98 #define DEF_PORT_IRQ \
99 	(PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
100 
101 /* product numbers */
102 #define MTIP_PRODUCT_UNKNOWN	0x00
103 #define MTIP_PRODUCT_ASICFPGA	0x11
104 
105 /* Device instance number, incremented each time a device is probed. */
106 static int instance;
107 
108 static struct list_head online_list;
109 static struct list_head removing_list;
110 static spinlock_t dev_lock;
111 
112 /*
113  * Global variable used to hold the major block device number
114  * allocated in mtip_init().
115  */
116 static int mtip_major;
117 static struct dentry *dfs_parent;
118 static struct dentry *dfs_device_status;
119 
120 static u32 cpu_use[NR_CPUS];
121 
122 static DEFINE_IDA(rssd_index_ida);
123 
124 static int mtip_block_initialize(struct driver_data *dd);
125 
126 #ifdef CONFIG_COMPAT
127 struct mtip_compat_ide_task_request_s {
128 	__u8		io_ports[8];
129 	__u8		hob_ports[8];
130 	ide_reg_valid_t	out_flags;
131 	ide_reg_valid_t	in_flags;
132 	int		data_phase;
133 	int		req_cmd;
134 	compat_ulong_t	out_size;
135 	compat_ulong_t	in_size;
136 };
137 #endif
138 
139 /*
140  * This function check_for_surprise_removal is called
141  * while card is removed from the system and it will
142  * read the vendor id from the configration space
143  *
144  * @pdev Pointer to the pci_dev structure.
145  *
146  * return value
147  *	 true if device removed, else false
148  */
149 static bool mtip_check_surprise_removal(struct pci_dev *pdev)
150 {
151 	u16 vendor_id = 0;
152 	struct driver_data *dd = pci_get_drvdata(pdev);
153 
154 	if (dd->sr)
155 		return true;
156 
157        /* Read the vendorID from the configuration space */
158 	pci_read_config_word(pdev, 0x00, &vendor_id);
159 	if (vendor_id == 0xFFFF) {
160 		dd->sr = true;
161 		if (dd->queue)
162 			blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue);
163 		else
164 			dev_warn(&dd->pdev->dev,
165 				"%s: dd->queue is NULL\n", __func__);
166 		return true; /* device removed */
167 	}
168 
169 	return false; /* device present */
170 }
171 
172 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd,
173 					  unsigned int tag)
174 {
175 	struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0];
176 
177 	return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag));
178 }
179 
180 /*
181  * Reset the HBA (without sleeping)
182  *
183  * @dd Pointer to the driver data structure.
184  *
185  * return value
186  *	0	The reset was successful.
187  *	-1	The HBA Reset bit did not clear.
188  */
189 static int mtip_hba_reset(struct driver_data *dd)
190 {
191 	unsigned long timeout;
192 
193 	/* Set the reset bit */
194 	writel(HOST_RESET, dd->mmio + HOST_CTL);
195 
196 	/* Flush */
197 	readl(dd->mmio + HOST_CTL);
198 
199 	/*
200 	 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec
201 	 * is 1 sec but in LUN failure conditions, up to 10 secs are required
202 	 */
203 	timeout = jiffies + msecs_to_jiffies(10000);
204 	do {
205 		mdelay(10);
206 		if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
207 			return -1;
208 
209 	} while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
210 		 && time_before(jiffies, timeout));
211 
212 	if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
213 		return -1;
214 
215 	return 0;
216 }
217 
218 /*
219  * Issue a command to the hardware.
220  *
221  * Set the appropriate bit in the s_active and Command Issue hardware
222  * registers, causing hardware command processing to begin.
223  *
224  * @port Pointer to the port structure.
225  * @tag  The tag of the command to be issued.
226  *
227  * return value
228  *      None
229  */
230 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
231 {
232 	int group = tag >> 5;
233 
234 	/* guard SACT and CI registers */
235 	spin_lock(&port->cmd_issue_lock[group]);
236 	writel((1 << MTIP_TAG_BIT(tag)),
237 			port->s_active[MTIP_TAG_INDEX(tag)]);
238 	writel((1 << MTIP_TAG_BIT(tag)),
239 			port->cmd_issue[MTIP_TAG_INDEX(tag)]);
240 	spin_unlock(&port->cmd_issue_lock[group]);
241 }
242 
243 /*
244  * Enable/disable the reception of FIS
245  *
246  * @port   Pointer to the port data structure
247  * @enable 1 to enable, 0 to disable
248  *
249  * return value
250  *	Previous state: 1 enabled, 0 disabled
251  */
252 static int mtip_enable_fis(struct mtip_port *port, int enable)
253 {
254 	u32 tmp;
255 
256 	/* enable FIS reception */
257 	tmp = readl(port->mmio + PORT_CMD);
258 	if (enable)
259 		writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
260 	else
261 		writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
262 
263 	/* Flush */
264 	readl(port->mmio + PORT_CMD);
265 
266 	return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
267 }
268 
269 /*
270  * Enable/disable the DMA engine
271  *
272  * @port   Pointer to the port data structure
273  * @enable 1 to enable, 0 to disable
274  *
275  * return value
276  *	Previous state: 1 enabled, 0 disabled.
277  */
278 static int mtip_enable_engine(struct mtip_port *port, int enable)
279 {
280 	u32 tmp;
281 
282 	/* enable FIS reception */
283 	tmp = readl(port->mmio + PORT_CMD);
284 	if (enable)
285 		writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
286 	else
287 		writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
288 
289 	readl(port->mmio + PORT_CMD);
290 	return (((tmp & PORT_CMD_START) == PORT_CMD_START));
291 }
292 
293 /*
294  * Enables the port DMA engine and FIS reception.
295  *
296  * return value
297  *	None
298  */
299 static inline void mtip_start_port(struct mtip_port *port)
300 {
301 	/* Enable FIS reception */
302 	mtip_enable_fis(port, 1);
303 
304 	/* Enable the DMA engine */
305 	mtip_enable_engine(port, 1);
306 }
307 
308 /*
309  * Deinitialize a port by disabling port interrupts, the DMA engine,
310  * and FIS reception.
311  *
312  * @port Pointer to the port structure
313  *
314  * return value
315  *	None
316  */
317 static inline void mtip_deinit_port(struct mtip_port *port)
318 {
319 	/* Disable interrupts on this port */
320 	writel(0, port->mmio + PORT_IRQ_MASK);
321 
322 	/* Disable the DMA engine */
323 	mtip_enable_engine(port, 0);
324 
325 	/* Disable FIS reception */
326 	mtip_enable_fis(port, 0);
327 }
328 
329 /*
330  * Initialize a port.
331  *
332  * This function deinitializes the port by calling mtip_deinit_port() and
333  * then initializes it by setting the command header and RX FIS addresses,
334  * clearing the SError register and any pending port interrupts before
335  * re-enabling the default set of port interrupts.
336  *
337  * @port Pointer to the port structure.
338  *
339  * return value
340  *	None
341  */
342 static void mtip_init_port(struct mtip_port *port)
343 {
344 	int i;
345 	mtip_deinit_port(port);
346 
347 	/* Program the command list base and FIS base addresses */
348 	if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
349 		writel((port->command_list_dma >> 16) >> 16,
350 			 port->mmio + PORT_LST_ADDR_HI);
351 		writel((port->rxfis_dma >> 16) >> 16,
352 			 port->mmio + PORT_FIS_ADDR_HI);
353 		set_bit(MTIP_PF_HOST_CAP_64, &port->flags);
354 	}
355 
356 	writel(port->command_list_dma & 0xFFFFFFFF,
357 			port->mmio + PORT_LST_ADDR);
358 	writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
359 
360 	/* Clear SError */
361 	writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
362 
363 	/* reset the completed registers.*/
364 	for (i = 0; i < port->dd->slot_groups; i++)
365 		writel(0xFFFFFFFF, port->completed[i]);
366 
367 	/* Clear any pending interrupts for this port */
368 	writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
369 
370 	/* Clear any pending interrupts on the HBA. */
371 	writel(readl(port->dd->mmio + HOST_IRQ_STAT),
372 					port->dd->mmio + HOST_IRQ_STAT);
373 
374 	/* Enable port interrupts */
375 	writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
376 }
377 
378 /*
379  * Restart a port
380  *
381  * @port Pointer to the port data structure.
382  *
383  * return value
384  *	None
385  */
386 static void mtip_restart_port(struct mtip_port *port)
387 {
388 	unsigned long timeout;
389 
390 	/* Disable the DMA engine */
391 	mtip_enable_engine(port, 0);
392 
393 	/* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
394 	timeout = jiffies + msecs_to_jiffies(500);
395 	while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
396 		 && time_before(jiffies, timeout))
397 		;
398 
399 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
400 		return;
401 
402 	/*
403 	 * Chip quirk: escalate to hba reset if
404 	 * PxCMD.CR not clear after 500 ms
405 	 */
406 	if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
407 		dev_warn(&port->dd->pdev->dev,
408 			"PxCMD.CR not clear, escalating reset\n");
409 
410 		if (mtip_hba_reset(port->dd))
411 			dev_err(&port->dd->pdev->dev,
412 				"HBA reset escalation failed.\n");
413 
414 		/* 30 ms delay before com reset to quiesce chip */
415 		mdelay(30);
416 	}
417 
418 	dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
419 
420 	/* Set PxSCTL.DET */
421 	writel(readl(port->mmio + PORT_SCR_CTL) |
422 			 1, port->mmio + PORT_SCR_CTL);
423 	readl(port->mmio + PORT_SCR_CTL);
424 
425 	/* Wait 1 ms to quiesce chip function */
426 	timeout = jiffies + msecs_to_jiffies(1);
427 	while (time_before(jiffies, timeout))
428 		;
429 
430 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
431 		return;
432 
433 	/* Clear PxSCTL.DET */
434 	writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
435 			 port->mmio + PORT_SCR_CTL);
436 	readl(port->mmio + PORT_SCR_CTL);
437 
438 	/* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
439 	timeout = jiffies + msecs_to_jiffies(500);
440 	while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
441 			 && time_before(jiffies, timeout))
442 		;
443 
444 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
445 		return;
446 
447 	if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
448 		dev_warn(&port->dd->pdev->dev,
449 			"COM reset failed\n");
450 
451 	mtip_init_port(port);
452 	mtip_start_port(port);
453 
454 }
455 
456 static int mtip_device_reset(struct driver_data *dd)
457 {
458 	int rv = 0;
459 
460 	if (mtip_check_surprise_removal(dd->pdev))
461 		return 0;
462 
463 	if (mtip_hba_reset(dd) < 0)
464 		rv = -EFAULT;
465 
466 	mdelay(1);
467 	mtip_init_port(dd->port);
468 	mtip_start_port(dd->port);
469 
470 	/* Enable interrupts on the HBA. */
471 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
472 					dd->mmio + HOST_CTL);
473 	return rv;
474 }
475 
476 /*
477  * Helper function for tag logging
478  */
479 static void print_tags(struct driver_data *dd,
480 			char *msg,
481 			unsigned long *tagbits,
482 			int cnt)
483 {
484 	unsigned char tagmap[128];
485 	int group, tagmap_len = 0;
486 
487 	memset(tagmap, 0, sizeof(tagmap));
488 	for (group = SLOTBITS_IN_LONGS; group > 0; group--)
489 		tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ",
490 						tagbits[group-1]);
491 	dev_warn(&dd->pdev->dev,
492 			"%d command(s) %s: tagmap [%s]", cnt, msg, tagmap);
493 }
494 
495 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
496 				dma_addr_t buffer_dma, unsigned int sectors);
497 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
498 						struct smart_attr *attrib);
499 
500 static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status)
501 {
502 	struct request *req = blk_mq_rq_from_pdu(cmd);
503 
504 	cmd->status = status;
505 	blk_mq_complete_request(req);
506 }
507 
508 /*
509  * Handle an error.
510  *
511  * @dd Pointer to the DRIVER_DATA structure.
512  *
513  * return value
514  *	None
515  */
516 static void mtip_handle_tfe(struct driver_data *dd)
517 {
518 	int group, tag, bit, reissue, rv;
519 	struct mtip_port *port;
520 	struct mtip_cmd  *cmd;
521 	u32 completed;
522 	struct host_to_dev_fis *fis;
523 	unsigned long tagaccum[SLOTBITS_IN_LONGS];
524 	unsigned int cmd_cnt = 0;
525 	unsigned char *buf;
526 	char *fail_reason = NULL;
527 	int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0;
528 
529 	dev_warn(&dd->pdev->dev, "Taskfile error\n");
530 
531 	port = dd->port;
532 
533 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) {
534 		cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
535 		dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n");
536 		mtip_complete_command(cmd, BLK_STS_IOERR);
537 		return;
538 	}
539 
540 	/* clear the tag accumulator */
541 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
542 
543 	/* Loop through all the groups */
544 	for (group = 0; group < dd->slot_groups; group++) {
545 		completed = readl(port->completed[group]);
546 
547 		dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed);
548 
549 		/* clear completed status register in the hardware.*/
550 		writel(completed, port->completed[group]);
551 
552 		/* Process successfully completed commands */
553 		for (bit = 0; bit < 32 && completed; bit++) {
554 			if (!(completed & (1<<bit)))
555 				continue;
556 			tag = (group << 5) + bit;
557 
558 			/* Skip the internal command slot */
559 			if (tag == MTIP_TAG_INTERNAL)
560 				continue;
561 
562 			cmd = mtip_cmd_from_tag(dd, tag);
563 			mtip_complete_command(cmd, 0);
564 			set_bit(tag, tagaccum);
565 			cmd_cnt++;
566 		}
567 	}
568 
569 	print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt);
570 
571 	/* Restart the port */
572 	mdelay(20);
573 	mtip_restart_port(port);
574 
575 	/* Trying to determine the cause of the error */
576 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
577 				dd->port->log_buf,
578 				dd->port->log_buf_dma, 1);
579 	if (rv) {
580 		dev_warn(&dd->pdev->dev,
581 			"Error in READ LOG EXT (10h) command\n");
582 		/* non-critical error, don't fail the load */
583 	} else {
584 		buf = (unsigned char *)dd->port->log_buf;
585 		if (buf[259] & 0x1) {
586 			dev_info(&dd->pdev->dev,
587 				"Write protect bit is set.\n");
588 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
589 			fail_all_ncq_write = 1;
590 			fail_reason = "write protect";
591 		}
592 		if (buf[288] == 0xF7) {
593 			dev_info(&dd->pdev->dev,
594 				"Exceeded Tmax, drive in thermal shutdown.\n");
595 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
596 			fail_all_ncq_cmds = 1;
597 			fail_reason = "thermal shutdown";
598 		}
599 		if (buf[288] == 0xBF) {
600 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
601 			dev_info(&dd->pdev->dev,
602 				"Drive indicates rebuild has failed. Secure erase required.\n");
603 			fail_all_ncq_cmds = 1;
604 			fail_reason = "rebuild failed";
605 		}
606 	}
607 
608 	/* clear the tag accumulator */
609 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
610 
611 	/* Loop through all the groups */
612 	for (group = 0; group < dd->slot_groups; group++) {
613 		for (bit = 0; bit < 32; bit++) {
614 			reissue = 1;
615 			tag = (group << 5) + bit;
616 			cmd = mtip_cmd_from_tag(dd, tag);
617 
618 			fis = (struct host_to_dev_fis *)cmd->command;
619 
620 			/* Should re-issue? */
621 			if (tag == MTIP_TAG_INTERNAL ||
622 			    fis->command == ATA_CMD_SET_FEATURES)
623 				reissue = 0;
624 			else {
625 				if (fail_all_ncq_cmds ||
626 					(fail_all_ncq_write &&
627 					fis->command == ATA_CMD_FPDMA_WRITE)) {
628 					dev_warn(&dd->pdev->dev,
629 					"  Fail: %s w/tag %d [%s].\n",
630 					fis->command == ATA_CMD_FPDMA_WRITE ?
631 						"write" : "read",
632 					tag,
633 					fail_reason != NULL ?
634 						fail_reason : "unknown");
635 					mtip_complete_command(cmd, BLK_STS_MEDIUM);
636 					continue;
637 				}
638 			}
639 
640 			/*
641 			 * First check if this command has
642 			 *  exceeded its retries.
643 			 */
644 			if (reissue && (cmd->retries-- > 0)) {
645 
646 				set_bit(tag, tagaccum);
647 
648 				/* Re-issue the command. */
649 				mtip_issue_ncq_command(port, tag);
650 
651 				continue;
652 			}
653 
654 			/* Retire a command that will not be reissued */
655 			dev_warn(&port->dd->pdev->dev,
656 				"retiring tag %d\n", tag);
657 
658 			mtip_complete_command(cmd, BLK_STS_IOERR);
659 		}
660 	}
661 	print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt);
662 }
663 
664 /*
665  * Handle a set device bits interrupt
666  */
667 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
668 							u32 completed)
669 {
670 	struct driver_data *dd = port->dd;
671 	int tag, bit;
672 	struct mtip_cmd *command;
673 
674 	if (!completed) {
675 		WARN_ON_ONCE(!completed);
676 		return;
677 	}
678 	/* clear completed status register in the hardware.*/
679 	writel(completed, port->completed[group]);
680 
681 	/* Process completed commands. */
682 	for (bit = 0; (bit < 32) && completed; bit++) {
683 		if (completed & 0x01) {
684 			tag = (group << 5) | bit;
685 
686 			/* skip internal command slot. */
687 			if (unlikely(tag == MTIP_TAG_INTERNAL))
688 				continue;
689 
690 			command = mtip_cmd_from_tag(dd, tag);
691 			mtip_complete_command(command, 0);
692 		}
693 		completed >>= 1;
694 	}
695 
696 	/* If last, re-enable interrupts */
697 	if (atomic_dec_return(&dd->irq_workers_active) == 0)
698 		writel(0xffffffff, dd->mmio + HOST_IRQ_STAT);
699 }
700 
701 /*
702  * Process legacy pio and d2h interrupts
703  */
704 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
705 {
706 	struct mtip_port *port = dd->port;
707 	struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
708 
709 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) {
710 		int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL);
711 		int status = readl(port->cmd_issue[group]);
712 
713 		if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))))
714 			mtip_complete_command(cmd, 0);
715 	}
716 }
717 
718 /*
719  * Demux and handle errors
720  */
721 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
722 {
723 	if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
724 		dev_warn(&dd->pdev->dev,
725 			"Clearing PxSERR.DIAG.x\n");
726 		writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
727 	}
728 
729 	if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
730 		dev_warn(&dd->pdev->dev,
731 			"Clearing PxSERR.DIAG.n\n");
732 		writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
733 	}
734 
735 	if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
736 		dev_warn(&dd->pdev->dev,
737 			"Port stat errors %x unhandled\n",
738 			(port_stat & ~PORT_IRQ_HANDLED));
739 		if (mtip_check_surprise_removal(dd->pdev))
740 			return;
741 	}
742 	if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
743 		set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags);
744 		wake_up_interruptible(&dd->port->svc_wait);
745 	}
746 }
747 
748 static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
749 {
750 	struct driver_data *dd = (struct driver_data *) data;
751 	struct mtip_port *port = dd->port;
752 	u32 hba_stat, port_stat;
753 	int rv = IRQ_NONE;
754 	int do_irq_enable = 1, i, workers;
755 	struct mtip_work *twork;
756 
757 	hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
758 	if (hba_stat) {
759 		rv = IRQ_HANDLED;
760 
761 		/* Acknowledge the interrupt status on the port.*/
762 		port_stat = readl(port->mmio + PORT_IRQ_STAT);
763 		if (unlikely(port_stat == 0xFFFFFFFF)) {
764 			mtip_check_surprise_removal(dd->pdev);
765 			return IRQ_HANDLED;
766 		}
767 		writel(port_stat, port->mmio + PORT_IRQ_STAT);
768 
769 		/* Demux port status */
770 		if (likely(port_stat & PORT_IRQ_SDB_FIS)) {
771 			do_irq_enable = 0;
772 			WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0);
773 
774 			/* Start at 1: group zero is always local? */
775 			for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS;
776 									i++) {
777 				twork = &dd->work[i];
778 				twork->completed = readl(port->completed[i]);
779 				if (twork->completed)
780 					workers++;
781 			}
782 
783 			atomic_set(&dd->irq_workers_active, workers);
784 			if (workers) {
785 				for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) {
786 					twork = &dd->work[i];
787 					if (twork->completed)
788 						queue_work_on(
789 							twork->cpu_binding,
790 							dd->isr_workq,
791 							&twork->work);
792 				}
793 
794 				if (likely(dd->work[0].completed))
795 					mtip_workq_sdbfx(port, 0,
796 							dd->work[0].completed);
797 
798 			} else {
799 				/*
800 				 * Chip quirk: SDB interrupt but nothing
801 				 * to complete
802 				 */
803 				do_irq_enable = 1;
804 			}
805 		}
806 
807 		if (unlikely(port_stat & PORT_IRQ_ERR)) {
808 			if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
809 				/* don't proceed further */
810 				return IRQ_HANDLED;
811 			}
812 			if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
813 							&dd->dd_flag))
814 				return rv;
815 
816 			mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
817 		}
818 
819 		if (unlikely(port_stat & PORT_IRQ_LEGACY))
820 			mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
821 	}
822 
823 	/* acknowledge interrupt */
824 	if (unlikely(do_irq_enable))
825 		writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
826 
827 	return rv;
828 }
829 
830 /*
831  * HBA interrupt subroutine.
832  *
833  * @irq		IRQ number.
834  * @instance	Pointer to the driver data structure.
835  *
836  * return value
837  *	IRQ_HANDLED	A HBA interrupt was pending and handled.
838  *	IRQ_NONE	This interrupt was not for the HBA.
839  */
840 static irqreturn_t mtip_irq_handler(int irq, void *instance)
841 {
842 	struct driver_data *dd = instance;
843 
844 	return mtip_handle_irq(dd);
845 }
846 
847 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
848 {
849 	writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]);
850 }
851 
852 static bool mtip_pause_ncq(struct mtip_port *port,
853 				struct host_to_dev_fis *fis)
854 {
855 	unsigned long task_file_data;
856 
857 	task_file_data = readl(port->mmio+PORT_TFDATA);
858 	if ((task_file_data & 1))
859 		return false;
860 
861 	if (fis->command == ATA_CMD_SEC_ERASE_PREP) {
862 		port->ic_pause_timer = jiffies;
863 		return true;
864 	} else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) &&
865 					(fis->features == 0x03)) {
866 		set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
867 		port->ic_pause_timer = jiffies;
868 		return true;
869 	} else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) ||
870 		((fis->command == 0xFC) &&
871 			(fis->features == 0x27 || fis->features == 0x72 ||
872 			 fis->features == 0x62 || fis->features == 0x26))) {
873 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
874 		clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag);
875 		/* Com reset after secure erase or lowlevel format */
876 		mtip_restart_port(port);
877 		clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
878 		return false;
879 	}
880 
881 	return false;
882 }
883 
884 static bool mtip_commands_active(struct mtip_port *port)
885 {
886 	unsigned int active;
887 	unsigned int n;
888 
889 	/*
890 	 * Ignore s_active bit 0 of array element 0.
891 	 * This bit will always be set
892 	 */
893 	active = readl(port->s_active[0]) & 0xFFFFFFFE;
894 	for (n = 1; n < port->dd->slot_groups; n++)
895 		active |= readl(port->s_active[n]);
896 
897 	return active != 0;
898 }
899 
900 /*
901  * Wait for port to quiesce
902  *
903  * @port    Pointer to port data structure
904  * @timeout Max duration to wait (ms)
905  *
906  * return value
907  *	0	Success
908  *	-EBUSY  Commands still active
909  */
910 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
911 {
912 	unsigned long to;
913 	bool active = true;
914 
915 	blk_mq_quiesce_queue(port->dd->queue);
916 
917 	to = jiffies + msecs_to_jiffies(timeout);
918 	do {
919 		if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) &&
920 			test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
921 			msleep(20);
922 			continue; /* svc thd is actively issuing commands */
923 		}
924 
925 		msleep(100);
926 
927 		if (mtip_check_surprise_removal(port->dd->pdev))
928 			goto err_fault;
929 
930 		active = mtip_commands_active(port);
931 		if (!active)
932 			break;
933 	} while (time_before(jiffies, to));
934 
935 	blk_mq_unquiesce_queue(port->dd->queue);
936 	return active ? -EBUSY : 0;
937 err_fault:
938 	blk_mq_unquiesce_queue(port->dd->queue);
939 	return -EFAULT;
940 }
941 
942 struct mtip_int_cmd {
943 	int fis_len;
944 	dma_addr_t buffer;
945 	int buf_len;
946 	u32 opts;
947 };
948 
949 /*
950  * Execute an internal command and wait for the completion.
951  *
952  * @port    Pointer to the port data structure.
953  * @fis     Pointer to the FIS that describes the command.
954  * @fis_len  Length in WORDS of the FIS.
955  * @buffer  DMA accessible for command data.
956  * @buf_len  Length, in bytes, of the data buffer.
957  * @opts    Command header options, excluding the FIS length
958  *             and the number of PRD entries.
959  * @timeout Time in ms to wait for the command to complete.
960  *
961  * return value
962  *	0	 Command completed successfully.
963  *	-EFAULT  The buffer address is not correctly aligned.
964  *	-EBUSY   Internal command or other IO in progress.
965  *	-EAGAIN  Time out waiting for command to complete.
966  */
967 static int mtip_exec_internal_command(struct mtip_port *port,
968 					struct host_to_dev_fis *fis,
969 					int fis_len,
970 					dma_addr_t buffer,
971 					int buf_len,
972 					u32 opts,
973 					unsigned long timeout)
974 {
975 	struct mtip_cmd *int_cmd;
976 	struct driver_data *dd = port->dd;
977 	struct request *rq;
978 	struct mtip_int_cmd icmd = {
979 		.fis_len = fis_len,
980 		.buffer = buffer,
981 		.buf_len = buf_len,
982 		.opts = opts
983 	};
984 	int rv = 0;
985 
986 	/* Make sure the buffer is 8 byte aligned. This is asic specific. */
987 	if (buffer & 0x00000007) {
988 		dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n");
989 		return -EFAULT;
990 	}
991 
992 	if (mtip_check_surprise_removal(dd->pdev))
993 		return -EFAULT;
994 
995 	rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
996 	if (IS_ERR(rq)) {
997 		dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n");
998 		return -EFAULT;
999 	}
1000 
1001 	set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1002 
1003 	if (fis->command == ATA_CMD_SEC_ERASE_PREP)
1004 		set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
1005 
1006 	clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
1007 
1008 	if (fis->command != ATA_CMD_STANDBYNOW1) {
1009 		/* wait for io to complete if non atomic */
1010 		if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) {
1011 			dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n");
1012 			blk_mq_free_request(rq);
1013 			clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1014 			wake_up_interruptible(&port->svc_wait);
1015 			return -EBUSY;
1016 		}
1017 	}
1018 
1019 	/* Copy the command to the command table */
1020 	int_cmd = blk_mq_rq_to_pdu(rq);
1021 	int_cmd->icmd = &icmd;
1022 	memcpy(int_cmd->command, fis, fis_len*4);
1023 
1024 	rq->timeout = timeout;
1025 
1026 	/* insert request and run queue */
1027 	blk_execute_rq(rq->q, NULL, rq, true);
1028 
1029 	if (int_cmd->status) {
1030 		dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
1031 				fis->command, int_cmd->status);
1032 		rv = -EIO;
1033 
1034 		if (mtip_check_surprise_removal(dd->pdev) ||
1035 			test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
1036 					&dd->dd_flag)) {
1037 			dev_err(&dd->pdev->dev,
1038 				"Internal command [%02X] wait returned due to SR\n",
1039 				fis->command);
1040 			rv = -ENXIO;
1041 			goto exec_ic_exit;
1042 		}
1043 		mtip_device_reset(dd); /* recover from timeout issue */
1044 		rv = -EAGAIN;
1045 		goto exec_ic_exit;
1046 	}
1047 
1048 	if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)])
1049 			& (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) {
1050 		rv = -ENXIO;
1051 		if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
1052 			mtip_device_reset(dd);
1053 			rv = -EAGAIN;
1054 		}
1055 	}
1056 exec_ic_exit:
1057 	/* Clear the allocated and active bits for the internal command. */
1058 	blk_mq_free_request(rq);
1059 	clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1060 	if (rv >= 0 && mtip_pause_ncq(port, fis)) {
1061 		/* NCQ paused */
1062 		return rv;
1063 	}
1064 	wake_up_interruptible(&port->svc_wait);
1065 
1066 	return rv;
1067 }
1068 
1069 /*
1070  * Byte-swap ATA ID strings.
1071  *
1072  * ATA identify data contains strings in byte-swapped 16-bit words.
1073  * They must be swapped (on all architectures) to be usable as C strings.
1074  * This function swaps bytes in-place.
1075  *
1076  * @buf The buffer location of the string
1077  * @len The number of bytes to swap
1078  *
1079  * return value
1080  *	None
1081  */
1082 static inline void ata_swap_string(u16 *buf, unsigned int len)
1083 {
1084 	int i;
1085 	for (i = 0; i < (len/2); i++)
1086 		be16_to_cpus(&buf[i]);
1087 }
1088 
1089 static void mtip_set_timeout(struct driver_data *dd,
1090 					struct host_to_dev_fis *fis,
1091 					unsigned int *timeout, u8 erasemode)
1092 {
1093 	switch (fis->command) {
1094 	case ATA_CMD_DOWNLOAD_MICRO:
1095 		*timeout = 120000; /* 2 minutes */
1096 		break;
1097 	case ATA_CMD_SEC_ERASE_UNIT:
1098 	case 0xFC:
1099 		if (erasemode)
1100 			*timeout = ((*(dd->port->identify + 90) * 2) * 60000);
1101 		else
1102 			*timeout = ((*(dd->port->identify + 89) * 2) * 60000);
1103 		break;
1104 	case ATA_CMD_STANDBYNOW1:
1105 		*timeout = 120000;  /* 2 minutes */
1106 		break;
1107 	case 0xF7:
1108 	case 0xFA:
1109 		*timeout = 60000;  /* 60 seconds */
1110 		break;
1111 	case ATA_CMD_SMART:
1112 		*timeout = 15000;  /* 15 seconds */
1113 		break;
1114 	default:
1115 		*timeout = MTIP_IOCTL_CMD_TIMEOUT_MS;
1116 		break;
1117 	}
1118 }
1119 
1120 /*
1121  * Request the device identity information.
1122  *
1123  * If a user space buffer is not specified, i.e. is NULL, the
1124  * identify information is still read from the drive and placed
1125  * into the identify data buffer (@e port->identify) in the
1126  * port data structure.
1127  * When the identify buffer contains valid identify information @e
1128  * port->identify_valid is non-zero.
1129  *
1130  * @port	 Pointer to the port structure.
1131  * @user_buffer  A user space buffer where the identify data should be
1132  *                    copied.
1133  *
1134  * return value
1135  *	0	Command completed successfully.
1136  *	-EFAULT An error occurred while coping data to the user buffer.
1137  *	-1	Command failed.
1138  */
1139 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
1140 {
1141 	int rv = 0;
1142 	struct host_to_dev_fis fis;
1143 
1144 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
1145 		return -EFAULT;
1146 
1147 	/* Build the FIS. */
1148 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1149 	fis.type	= 0x27;
1150 	fis.opts	= 1 << 7;
1151 	fis.command	= ATA_CMD_ID_ATA;
1152 
1153 	/* Set the identify information as invalid. */
1154 	port->identify_valid = 0;
1155 
1156 	/* Clear the identify information. */
1157 	memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
1158 
1159 	/* Execute the command. */
1160 	if (mtip_exec_internal_command(port,
1161 				&fis,
1162 				5,
1163 				port->identify_dma,
1164 				sizeof(u16) * ATA_ID_WORDS,
1165 				0,
1166 				MTIP_INT_CMD_TIMEOUT_MS)
1167 				< 0) {
1168 		rv = -1;
1169 		goto out;
1170 	}
1171 
1172 	/*
1173 	 * Perform any necessary byte-swapping.  Yes, the kernel does in fact
1174 	 * perform field-sensitive swapping on the string fields.
1175 	 * See the kernel use of ata_id_string() for proof of this.
1176 	 */
1177 #ifdef __LITTLE_ENDIAN
1178 	ata_swap_string(port->identify + 27, 40);  /* model string*/
1179 	ata_swap_string(port->identify + 23, 8);   /* firmware string*/
1180 	ata_swap_string(port->identify + 10, 20);  /* serial# string*/
1181 #else
1182 	{
1183 		int i;
1184 		for (i = 0; i < ATA_ID_WORDS; i++)
1185 			port->identify[i] = le16_to_cpu(port->identify[i]);
1186 	}
1187 #endif
1188 
1189 	/* Check security locked state */
1190 	if (port->identify[128] & 0x4)
1191 		set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1192 	else
1193 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1194 
1195 #ifdef MTIP_TRIM /* Disabling TRIM support temporarily */
1196 	/* Demux ID.DRAT & ID.RZAT to determine trim support */
1197 	if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5))
1198 		port->dd->trim_supp = true;
1199 	else
1200 #endif
1201 		port->dd->trim_supp = false;
1202 
1203 	/* Set the identify buffer as valid. */
1204 	port->identify_valid = 1;
1205 
1206 	if (user_buffer) {
1207 		if (copy_to_user(
1208 			user_buffer,
1209 			port->identify,
1210 			ATA_ID_WORDS * sizeof(u16))) {
1211 			rv = -EFAULT;
1212 			goto out;
1213 		}
1214 	}
1215 
1216 out:
1217 	return rv;
1218 }
1219 
1220 /*
1221  * Issue a standby immediate command to the device.
1222  *
1223  * @port Pointer to the port structure.
1224  *
1225  * return value
1226  *	0	Command was executed successfully.
1227  *	-1	An error occurred while executing the command.
1228  */
1229 static int mtip_standby_immediate(struct mtip_port *port)
1230 {
1231 	int rv;
1232 	struct host_to_dev_fis	fis;
1233 	unsigned long start;
1234 	unsigned int timeout;
1235 
1236 	/* Build the FIS. */
1237 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1238 	fis.type	= 0x27;
1239 	fis.opts	= 1 << 7;
1240 	fis.command	= ATA_CMD_STANDBYNOW1;
1241 
1242 	mtip_set_timeout(port->dd, &fis, &timeout, 0);
1243 
1244 	start = jiffies;
1245 	rv = mtip_exec_internal_command(port,
1246 					&fis,
1247 					5,
1248 					0,
1249 					0,
1250 					0,
1251 					timeout);
1252 	dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n",
1253 			jiffies_to_msecs(jiffies - start));
1254 	if (rv)
1255 		dev_warn(&port->dd->pdev->dev,
1256 			"STANDBY IMMEDIATE command failed.\n");
1257 
1258 	return rv;
1259 }
1260 
1261 /*
1262  * Issue a READ LOG EXT command to the device.
1263  *
1264  * @port	pointer to the port structure.
1265  * @page	page number to fetch
1266  * @buffer	pointer to buffer
1267  * @buffer_dma	dma address corresponding to @buffer
1268  * @sectors	page length to fetch, in sectors
1269  *
1270  * return value
1271  *	@rv	return value from mtip_exec_internal_command()
1272  */
1273 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
1274 				dma_addr_t buffer_dma, unsigned int sectors)
1275 {
1276 	struct host_to_dev_fis fis;
1277 
1278 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1279 	fis.type	= 0x27;
1280 	fis.opts	= 1 << 7;
1281 	fis.command	= ATA_CMD_READ_LOG_EXT;
1282 	fis.sect_count	= sectors & 0xFF;
1283 	fis.sect_cnt_ex	= (sectors >> 8) & 0xFF;
1284 	fis.lba_low	= page;
1285 	fis.lba_mid	= 0;
1286 	fis.device	= ATA_DEVICE_OBS;
1287 
1288 	memset(buffer, 0, sectors * ATA_SECT_SIZE);
1289 
1290 	return mtip_exec_internal_command(port,
1291 					&fis,
1292 					5,
1293 					buffer_dma,
1294 					sectors * ATA_SECT_SIZE,
1295 					0,
1296 					MTIP_INT_CMD_TIMEOUT_MS);
1297 }
1298 
1299 /*
1300  * Issue a SMART READ DATA command to the device.
1301  *
1302  * @port	pointer to the port structure.
1303  * @buffer	pointer to buffer
1304  * @buffer_dma	dma address corresponding to @buffer
1305  *
1306  * return value
1307  *	@rv	return value from mtip_exec_internal_command()
1308  */
1309 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer,
1310 					dma_addr_t buffer_dma)
1311 {
1312 	struct host_to_dev_fis fis;
1313 
1314 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1315 	fis.type	= 0x27;
1316 	fis.opts	= 1 << 7;
1317 	fis.command	= ATA_CMD_SMART;
1318 	fis.features	= 0xD0;
1319 	fis.sect_count	= 1;
1320 	fis.lba_mid	= 0x4F;
1321 	fis.lba_hi	= 0xC2;
1322 	fis.device	= ATA_DEVICE_OBS;
1323 
1324 	return mtip_exec_internal_command(port,
1325 					&fis,
1326 					5,
1327 					buffer_dma,
1328 					ATA_SECT_SIZE,
1329 					0,
1330 					15000);
1331 }
1332 
1333 /*
1334  * Get the value of a smart attribute
1335  *
1336  * @port	pointer to the port structure
1337  * @id		attribute number
1338  * @attrib	pointer to return attrib information corresponding to @id
1339  *
1340  * return value
1341  *	-EINVAL	NULL buffer passed or unsupported attribute @id.
1342  *	-EPERM	Identify data not valid, SMART not supported or not enabled
1343  */
1344 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
1345 						struct smart_attr *attrib)
1346 {
1347 	int rv, i;
1348 	struct smart_attr *pattr;
1349 
1350 	if (!attrib)
1351 		return -EINVAL;
1352 
1353 	if (!port->identify_valid) {
1354 		dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n");
1355 		return -EPERM;
1356 	}
1357 	if (!(port->identify[82] & 0x1)) {
1358 		dev_warn(&port->dd->pdev->dev, "SMART not supported\n");
1359 		return -EPERM;
1360 	}
1361 	if (!(port->identify[85] & 0x1)) {
1362 		dev_warn(&port->dd->pdev->dev, "SMART not enabled\n");
1363 		return -EPERM;
1364 	}
1365 
1366 	memset(port->smart_buf, 0, ATA_SECT_SIZE);
1367 	rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma);
1368 	if (rv) {
1369 		dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n");
1370 		return rv;
1371 	}
1372 
1373 	pattr = (struct smart_attr *)(port->smart_buf + 2);
1374 	for (i = 0; i < 29; i++, pattr++)
1375 		if (pattr->attr_id == id) {
1376 			memcpy(attrib, pattr, sizeof(struct smart_attr));
1377 			break;
1378 		}
1379 
1380 	if (i == 29) {
1381 		dev_warn(&port->dd->pdev->dev,
1382 			"Query for invalid SMART attribute ID\n");
1383 		rv = -EINVAL;
1384 	}
1385 
1386 	return rv;
1387 }
1388 
1389 /*
1390  * Trim unused sectors
1391  *
1392  * @dd		pointer to driver_data structure
1393  * @lba		starting lba
1394  * @len		# of 512b sectors to trim
1395  */
1396 static blk_status_t mtip_send_trim(struct driver_data *dd, unsigned int lba,
1397 		unsigned int len)
1398 {
1399 	u64 tlba, tlen, sect_left;
1400 	struct mtip_trim_entry *buf;
1401 	dma_addr_t dma_addr;
1402 	struct host_to_dev_fis fis;
1403 	blk_status_t ret = BLK_STS_OK;
1404 	int i;
1405 
1406 	if (!len || dd->trim_supp == false)
1407 		return BLK_STS_IOERR;
1408 
1409 	/* Trim request too big */
1410 	WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES));
1411 
1412 	/* Trim request not aligned on 4k boundary */
1413 	WARN_ON(len % 8 != 0);
1414 
1415 	/* Warn if vu_trim structure is too big */
1416 	WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE);
1417 
1418 	/* Allocate a DMA buffer for the trim structure */
1419 	buf = dma_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr,
1420 								GFP_KERNEL);
1421 	if (!buf)
1422 		return BLK_STS_RESOURCE;
1423 	memset(buf, 0, ATA_SECT_SIZE);
1424 
1425 	for (i = 0, sect_left = len, tlba = lba;
1426 			i < MTIP_MAX_TRIM_ENTRIES && sect_left;
1427 			i++) {
1428 		tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ?
1429 					MTIP_MAX_TRIM_ENTRY_LEN :
1430 					sect_left);
1431 		buf[i].lba = cpu_to_le32(tlba);
1432 		buf[i].range = cpu_to_le16(tlen);
1433 		tlba += tlen;
1434 		sect_left -= tlen;
1435 	}
1436 	WARN_ON(sect_left != 0);
1437 
1438 	/* Build the fis */
1439 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1440 	fis.type       = 0x27;
1441 	fis.opts       = 1 << 7;
1442 	fis.command    = 0xfb;
1443 	fis.features   = 0x60;
1444 	fis.sect_count = 1;
1445 	fis.device     = ATA_DEVICE_OBS;
1446 
1447 	if (mtip_exec_internal_command(dd->port,
1448 					&fis,
1449 					5,
1450 					dma_addr,
1451 					ATA_SECT_SIZE,
1452 					0,
1453 					MTIP_TRIM_TIMEOUT_MS) < 0)
1454 		ret = BLK_STS_IOERR;
1455 
1456 	dma_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr);
1457 	return ret;
1458 }
1459 
1460 /*
1461  * Get the drive capacity.
1462  *
1463  * @dd      Pointer to the device data structure.
1464  * @sectors Pointer to the variable that will receive the sector count.
1465  *
1466  * return value
1467  *	1 Capacity was returned successfully.
1468  *	0 The identify information is invalid.
1469  */
1470 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
1471 {
1472 	struct mtip_port *port = dd->port;
1473 	u64 total, raw0, raw1, raw2, raw3;
1474 	raw0 = port->identify[100];
1475 	raw1 = port->identify[101];
1476 	raw2 = port->identify[102];
1477 	raw3 = port->identify[103];
1478 	total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
1479 	*sectors = total;
1480 	return (bool) !!port->identify_valid;
1481 }
1482 
1483 /*
1484  * Display the identify command data.
1485  *
1486  * @port Pointer to the port data structure.
1487  *
1488  * return value
1489  *	None
1490  */
1491 static void mtip_dump_identify(struct mtip_port *port)
1492 {
1493 	sector_t sectors;
1494 	unsigned short revid;
1495 	char cbuf[42];
1496 
1497 	if (!port->identify_valid)
1498 		return;
1499 
1500 	strlcpy(cbuf, (char *)(port->identify+10), 21);
1501 	dev_info(&port->dd->pdev->dev,
1502 		"Serial No.: %s\n", cbuf);
1503 
1504 	strlcpy(cbuf, (char *)(port->identify+23), 9);
1505 	dev_info(&port->dd->pdev->dev,
1506 		"Firmware Ver.: %s\n", cbuf);
1507 
1508 	strlcpy(cbuf, (char *)(port->identify+27), 41);
1509 	dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
1510 
1511 	dev_info(&port->dd->pdev->dev, "Security: %04x %s\n",
1512 		port->identify[128],
1513 		port->identify[128] & 0x4 ? "(LOCKED)" : "");
1514 
1515 	if (mtip_hw_get_capacity(port->dd, &sectors))
1516 		dev_info(&port->dd->pdev->dev,
1517 			"Capacity: %llu sectors (%llu MB)\n",
1518 			 (u64)sectors,
1519 			 ((u64)sectors) * ATA_SECT_SIZE >> 20);
1520 
1521 	pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
1522 	switch (revid & 0xFF) {
1523 	case 0x1:
1524 		strlcpy(cbuf, "A0", 3);
1525 		break;
1526 	case 0x3:
1527 		strlcpy(cbuf, "A2", 3);
1528 		break;
1529 	default:
1530 		strlcpy(cbuf, "?", 2);
1531 		break;
1532 	}
1533 	dev_info(&port->dd->pdev->dev,
1534 		"Card Type: %s\n", cbuf);
1535 }
1536 
1537 /*
1538  * Map the commands scatter list into the command table.
1539  *
1540  * @command Pointer to the command.
1541  * @nents Number of scatter list entries.
1542  *
1543  * return value
1544  *	None
1545  */
1546 static inline void fill_command_sg(struct driver_data *dd,
1547 				struct mtip_cmd *command,
1548 				int nents)
1549 {
1550 	int n;
1551 	unsigned int dma_len;
1552 	struct mtip_cmd_sg *command_sg;
1553 	struct scatterlist *sg;
1554 
1555 	command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
1556 
1557 	for_each_sg(command->sg, sg, nents, n) {
1558 		dma_len = sg_dma_len(sg);
1559 		if (dma_len > 0x400000)
1560 			dev_err(&dd->pdev->dev,
1561 				"DMA segment length truncated\n");
1562 		command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF);
1563 		command_sg->dba	=  cpu_to_le32(sg_dma_address(sg));
1564 		command_sg->dba_upper =
1565 			cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
1566 		command_sg++;
1567 	}
1568 }
1569 
1570 /*
1571  * @brief Execute a drive command.
1572  *
1573  * return value 0 The command completed successfully.
1574  * return value -1 An error occurred while executing the command.
1575  */
1576 static int exec_drive_task(struct mtip_port *port, u8 *command)
1577 {
1578 	struct host_to_dev_fis	fis;
1579 	struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1580 	unsigned int to;
1581 
1582 	/* Build the FIS. */
1583 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1584 	fis.type	= 0x27;
1585 	fis.opts	= 1 << 7;
1586 	fis.command	= command[0];
1587 	fis.features	= command[1];
1588 	fis.sect_count	= command[2];
1589 	fis.sector	= command[3];
1590 	fis.cyl_low	= command[4];
1591 	fis.cyl_hi	= command[5];
1592 	fis.device	= command[6] & ~0x10; /* Clear the dev bit*/
1593 
1594 	mtip_set_timeout(port->dd, &fis, &to, 0);
1595 
1596 	dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n",
1597 		__func__,
1598 		command[0],
1599 		command[1],
1600 		command[2],
1601 		command[3],
1602 		command[4],
1603 		command[5],
1604 		command[6]);
1605 
1606 	/* Execute the command. */
1607 	if (mtip_exec_internal_command(port,
1608 				 &fis,
1609 				 5,
1610 				 0,
1611 				 0,
1612 				 0,
1613 				 to) < 0) {
1614 		return -1;
1615 	}
1616 
1617 	command[0] = reply->command; /* Status*/
1618 	command[1] = reply->features; /* Error*/
1619 	command[4] = reply->cyl_low;
1620 	command[5] = reply->cyl_hi;
1621 
1622 	dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n",
1623 		__func__,
1624 		command[0],
1625 		command[1],
1626 		command[4],
1627 		command[5]);
1628 
1629 	return 0;
1630 }
1631 
1632 /*
1633  * @brief Execute a drive command.
1634  *
1635  * @param port Pointer to the port data structure.
1636  * @param command Pointer to the user specified command parameters.
1637  * @param user_buffer Pointer to the user space buffer where read sector
1638  *                   data should be copied.
1639  *
1640  * return value 0 The command completed successfully.
1641  * return value -EFAULT An error occurred while copying the completion
1642  *                 data to the user space buffer.
1643  * return value -1 An error occurred while executing the command.
1644  */
1645 static int exec_drive_command(struct mtip_port *port, u8 *command,
1646 				void __user *user_buffer)
1647 {
1648 	struct host_to_dev_fis	fis;
1649 	struct host_to_dev_fis *reply;
1650 	u8 *buf = NULL;
1651 	dma_addr_t dma_addr = 0;
1652 	int rv = 0, xfer_sz = command[3];
1653 	unsigned int to;
1654 
1655 	if (xfer_sz) {
1656 		if (!user_buffer)
1657 			return -EFAULT;
1658 
1659 		buf = dma_alloc_coherent(&port->dd->pdev->dev,
1660 				ATA_SECT_SIZE * xfer_sz,
1661 				&dma_addr,
1662 				GFP_KERNEL);
1663 		if (!buf) {
1664 			dev_err(&port->dd->pdev->dev,
1665 				"Memory allocation failed (%d bytes)\n",
1666 				ATA_SECT_SIZE * xfer_sz);
1667 			return -ENOMEM;
1668 		}
1669 		memset(buf, 0, ATA_SECT_SIZE * xfer_sz);
1670 	}
1671 
1672 	/* Build the FIS. */
1673 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1674 	fis.type	= 0x27;
1675 	fis.opts	= 1 << 7;
1676 	fis.command	= command[0];
1677 	fis.features	= command[2];
1678 	fis.sect_count	= command[3];
1679 	if (fis.command == ATA_CMD_SMART) {
1680 		fis.sector	= command[1];
1681 		fis.cyl_low	= 0x4F;
1682 		fis.cyl_hi	= 0xC2;
1683 	}
1684 
1685 	mtip_set_timeout(port->dd, &fis, &to, 0);
1686 
1687 	if (xfer_sz)
1688 		reply = (port->rxfis + RX_FIS_PIO_SETUP);
1689 	else
1690 		reply = (port->rxfis + RX_FIS_D2H_REG);
1691 
1692 	dbg_printk(MTIP_DRV_NAME
1693 		" %s: User Command: cmd %x, sect %x, "
1694 		"feat %x, sectcnt %x\n",
1695 		__func__,
1696 		command[0],
1697 		command[1],
1698 		command[2],
1699 		command[3]);
1700 
1701 	/* Execute the command. */
1702 	if (mtip_exec_internal_command(port,
1703 				&fis,
1704 				 5,
1705 				 (xfer_sz ? dma_addr : 0),
1706 				 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0),
1707 				 0,
1708 				 to)
1709 				 < 0) {
1710 		rv = -EFAULT;
1711 		goto exit_drive_command;
1712 	}
1713 
1714 	/* Collect the completion status. */
1715 	command[0] = reply->command; /* Status*/
1716 	command[1] = reply->features; /* Error*/
1717 	command[2] = reply->sect_count;
1718 
1719 	dbg_printk(MTIP_DRV_NAME
1720 		" %s: Completion Status: stat %x, "
1721 		"err %x, nsect %x\n",
1722 		__func__,
1723 		command[0],
1724 		command[1],
1725 		command[2]);
1726 
1727 	if (xfer_sz) {
1728 		if (copy_to_user(user_buffer,
1729 				 buf,
1730 				 ATA_SECT_SIZE * command[3])) {
1731 			rv = -EFAULT;
1732 			goto exit_drive_command;
1733 		}
1734 	}
1735 exit_drive_command:
1736 	if (buf)
1737 		dma_free_coherent(&port->dd->pdev->dev,
1738 				ATA_SECT_SIZE * xfer_sz, buf, dma_addr);
1739 	return rv;
1740 }
1741 
1742 /*
1743  *  Indicates whether a command has a single sector payload.
1744  *
1745  *  @command passed to the device to perform the certain event.
1746  *  @features passed to the device to perform the certain event.
1747  *
1748  *  return value
1749  *	1	command is one that always has a single sector payload,
1750  *		regardless of the value in the Sector Count field.
1751  *      0       otherwise
1752  *
1753  */
1754 static unsigned int implicit_sector(unsigned char command,
1755 				    unsigned char features)
1756 {
1757 	unsigned int rv = 0;
1758 
1759 	/* list of commands that have an implicit sector count of 1 */
1760 	switch (command) {
1761 	case ATA_CMD_SEC_SET_PASS:
1762 	case ATA_CMD_SEC_UNLOCK:
1763 	case ATA_CMD_SEC_ERASE_PREP:
1764 	case ATA_CMD_SEC_ERASE_UNIT:
1765 	case ATA_CMD_SEC_FREEZE_LOCK:
1766 	case ATA_CMD_SEC_DISABLE_PASS:
1767 	case ATA_CMD_PMP_READ:
1768 	case ATA_CMD_PMP_WRITE:
1769 		rv = 1;
1770 		break;
1771 	case ATA_CMD_SET_MAX:
1772 		if (features == ATA_SET_MAX_UNLOCK)
1773 			rv = 1;
1774 		break;
1775 	case ATA_CMD_SMART:
1776 		if ((features == ATA_SMART_READ_VALUES) ||
1777 				(features == ATA_SMART_READ_THRESHOLDS))
1778 			rv = 1;
1779 		break;
1780 	case ATA_CMD_CONF_OVERLAY:
1781 		if ((features == ATA_DCO_IDENTIFY) ||
1782 				(features == ATA_DCO_SET))
1783 			rv = 1;
1784 		break;
1785 	}
1786 	return rv;
1787 }
1788 
1789 /*
1790  * Executes a taskfile
1791  * See ide_taskfile_ioctl() for derivation
1792  */
1793 static int exec_drive_taskfile(struct driver_data *dd,
1794 			       void __user *buf,
1795 			       ide_task_request_t *req_task,
1796 			       int outtotal)
1797 {
1798 	struct host_to_dev_fis	fis;
1799 	struct host_to_dev_fis *reply;
1800 	u8 *outbuf = NULL;
1801 	u8 *inbuf = NULL;
1802 	dma_addr_t outbuf_dma = 0;
1803 	dma_addr_t inbuf_dma = 0;
1804 	dma_addr_t dma_buffer = 0;
1805 	int err = 0;
1806 	unsigned int taskin = 0;
1807 	unsigned int taskout = 0;
1808 	u8 nsect = 0;
1809 	unsigned int timeout;
1810 	unsigned int force_single_sector;
1811 	unsigned int transfer_size;
1812 	unsigned long task_file_data;
1813 	int intotal = outtotal + req_task->out_size;
1814 	int erasemode = 0;
1815 
1816 	taskout = req_task->out_size;
1817 	taskin = req_task->in_size;
1818 	/* 130560 = 512 * 0xFF*/
1819 	if (taskin > 130560 || taskout > 130560)
1820 		return -EINVAL;
1821 
1822 	if (taskout) {
1823 		outbuf = memdup_user(buf + outtotal, taskout);
1824 		if (IS_ERR(outbuf))
1825 			return PTR_ERR(outbuf);
1826 
1827 		outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf,
1828 					    taskout, DMA_TO_DEVICE);
1829 		if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) {
1830 			err = -ENOMEM;
1831 			goto abort;
1832 		}
1833 		dma_buffer = outbuf_dma;
1834 	}
1835 
1836 	if (taskin) {
1837 		inbuf = memdup_user(buf + intotal, taskin);
1838 		if (IS_ERR(inbuf)) {
1839 			err = PTR_ERR(inbuf);
1840 			inbuf = NULL;
1841 			goto abort;
1842 		}
1843 		inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf,
1844 					   taskin, DMA_FROM_DEVICE);
1845 		if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) {
1846 			err = -ENOMEM;
1847 			goto abort;
1848 		}
1849 		dma_buffer = inbuf_dma;
1850 	}
1851 
1852 	/* only supports PIO and non-data commands from this ioctl. */
1853 	switch (req_task->data_phase) {
1854 	case TASKFILE_OUT:
1855 		nsect = taskout / ATA_SECT_SIZE;
1856 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1857 		break;
1858 	case TASKFILE_IN:
1859 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1860 		break;
1861 	case TASKFILE_NO_DATA:
1862 		reply = (dd->port->rxfis + RX_FIS_D2H_REG);
1863 		break;
1864 	default:
1865 		err = -EINVAL;
1866 		goto abort;
1867 	}
1868 
1869 	/* Build the FIS. */
1870 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1871 
1872 	fis.type	= 0x27;
1873 	fis.opts	= 1 << 7;
1874 	fis.command	= req_task->io_ports[7];
1875 	fis.features	= req_task->io_ports[1];
1876 	fis.sect_count	= req_task->io_ports[2];
1877 	fis.lba_low	= req_task->io_ports[3];
1878 	fis.lba_mid	= req_task->io_ports[4];
1879 	fis.lba_hi	= req_task->io_ports[5];
1880 	 /* Clear the dev bit*/
1881 	fis.device	= req_task->io_ports[6] & ~0x10;
1882 
1883 	if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
1884 		req_task->in_flags.all	=
1885 			IDE_TASKFILE_STD_IN_FLAGS |
1886 			(IDE_HOB_STD_IN_FLAGS << 8);
1887 		fis.lba_low_ex		= req_task->hob_ports[3];
1888 		fis.lba_mid_ex		= req_task->hob_ports[4];
1889 		fis.lba_hi_ex		= req_task->hob_ports[5];
1890 		fis.features_ex		= req_task->hob_ports[1];
1891 		fis.sect_cnt_ex		= req_task->hob_ports[2];
1892 
1893 	} else {
1894 		req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
1895 	}
1896 
1897 	force_single_sector = implicit_sector(fis.command, fis.features);
1898 
1899 	if ((taskin || taskout) && (!fis.sect_count)) {
1900 		if (nsect)
1901 			fis.sect_count = nsect;
1902 		else {
1903 			if (!force_single_sector) {
1904 				dev_warn(&dd->pdev->dev,
1905 					"data movement but "
1906 					"sect_count is 0\n");
1907 				err = -EINVAL;
1908 				goto abort;
1909 			}
1910 		}
1911 	}
1912 
1913 	dbg_printk(MTIP_DRV_NAME
1914 		" %s: cmd %x, feat %x, nsect %x,"
1915 		" sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
1916 		" head/dev %x\n",
1917 		__func__,
1918 		fis.command,
1919 		fis.features,
1920 		fis.sect_count,
1921 		fis.lba_low,
1922 		fis.lba_mid,
1923 		fis.lba_hi,
1924 		fis.device);
1925 
1926 	/* check for erase mode support during secure erase.*/
1927 	if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf &&
1928 					(outbuf[0] & MTIP_SEC_ERASE_MODE)) {
1929 		erasemode = 1;
1930 	}
1931 
1932 	mtip_set_timeout(dd, &fis, &timeout, erasemode);
1933 
1934 	/* Determine the correct transfer size.*/
1935 	if (force_single_sector)
1936 		transfer_size = ATA_SECT_SIZE;
1937 	else
1938 		transfer_size = ATA_SECT_SIZE * fis.sect_count;
1939 
1940 	/* Execute the command.*/
1941 	if (mtip_exec_internal_command(dd->port,
1942 				 &fis,
1943 				 5,
1944 				 dma_buffer,
1945 				 transfer_size,
1946 				 0,
1947 				 timeout) < 0) {
1948 		err = -EIO;
1949 		goto abort;
1950 	}
1951 
1952 	task_file_data = readl(dd->port->mmio+PORT_TFDATA);
1953 
1954 	if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
1955 		reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
1956 		req_task->io_ports[7] = reply->control;
1957 	} else {
1958 		reply = dd->port->rxfis + RX_FIS_D2H_REG;
1959 		req_task->io_ports[7] = reply->command;
1960 	}
1961 
1962 	/* reclaim the DMA buffers.*/
1963 	if (inbuf_dma)
1964 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1965 				 DMA_FROM_DEVICE);
1966 	if (outbuf_dma)
1967 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1968 				 DMA_TO_DEVICE);
1969 	inbuf_dma  = 0;
1970 	outbuf_dma = 0;
1971 
1972 	/* return the ATA registers to the caller.*/
1973 	req_task->io_ports[1] = reply->features;
1974 	req_task->io_ports[2] = reply->sect_count;
1975 	req_task->io_ports[3] = reply->lba_low;
1976 	req_task->io_ports[4] = reply->lba_mid;
1977 	req_task->io_ports[5] = reply->lba_hi;
1978 	req_task->io_ports[6] = reply->device;
1979 
1980 	if (req_task->out_flags.all & 1)  {
1981 
1982 		req_task->hob_ports[3] = reply->lba_low_ex;
1983 		req_task->hob_ports[4] = reply->lba_mid_ex;
1984 		req_task->hob_ports[5] = reply->lba_hi_ex;
1985 		req_task->hob_ports[1] = reply->features_ex;
1986 		req_task->hob_ports[2] = reply->sect_cnt_ex;
1987 	}
1988 	dbg_printk(MTIP_DRV_NAME
1989 		" %s: Completion: stat %x,"
1990 		"err %x, sect_cnt %x, lbalo %x,"
1991 		"lbamid %x, lbahi %x, dev %x\n",
1992 		__func__,
1993 		req_task->io_ports[7],
1994 		req_task->io_ports[1],
1995 		req_task->io_ports[2],
1996 		req_task->io_ports[3],
1997 		req_task->io_ports[4],
1998 		req_task->io_ports[5],
1999 		req_task->io_ports[6]);
2000 
2001 	if (taskout) {
2002 		if (copy_to_user(buf + outtotal, outbuf, taskout)) {
2003 			err = -EFAULT;
2004 			goto abort;
2005 		}
2006 	}
2007 	if (taskin) {
2008 		if (copy_to_user(buf + intotal, inbuf, taskin)) {
2009 			err = -EFAULT;
2010 			goto abort;
2011 		}
2012 	}
2013 abort:
2014 	if (inbuf_dma)
2015 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
2016 				 DMA_FROM_DEVICE);
2017 	if (outbuf_dma)
2018 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
2019 				 DMA_TO_DEVICE);
2020 	kfree(outbuf);
2021 	kfree(inbuf);
2022 
2023 	return err;
2024 }
2025 
2026 /*
2027  * Handle IOCTL calls from the Block Layer.
2028  *
2029  * This function is called by the Block Layer when it receives an IOCTL
2030  * command that it does not understand. If the IOCTL command is not supported
2031  * this function returns -ENOTTY.
2032  *
2033  * @dd  Pointer to the driver data structure.
2034  * @cmd IOCTL command passed from the Block Layer.
2035  * @arg IOCTL argument passed from the Block Layer.
2036  *
2037  * return value
2038  *	0	The IOCTL completed successfully.
2039  *	-ENOTTY The specified command is not supported.
2040  *	-EFAULT An error occurred copying data to a user space buffer.
2041  *	-EIO	An error occurred while executing the command.
2042  */
2043 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
2044 			 unsigned long arg)
2045 {
2046 	switch (cmd) {
2047 	case HDIO_GET_IDENTITY:
2048 	{
2049 		if (copy_to_user((void __user *)arg, dd->port->identify,
2050 						sizeof(u16) * ATA_ID_WORDS))
2051 			return -EFAULT;
2052 		break;
2053 	}
2054 	case HDIO_DRIVE_CMD:
2055 	{
2056 		u8 drive_command[4];
2057 
2058 		/* Copy the user command info to our buffer. */
2059 		if (copy_from_user(drive_command,
2060 					 (void __user *) arg,
2061 					 sizeof(drive_command)))
2062 			return -EFAULT;
2063 
2064 		/* Execute the drive command. */
2065 		if (exec_drive_command(dd->port,
2066 					 drive_command,
2067 					 (void __user *) (arg+4)))
2068 			return -EIO;
2069 
2070 		/* Copy the status back to the users buffer. */
2071 		if (copy_to_user((void __user *) arg,
2072 					 drive_command,
2073 					 sizeof(drive_command)))
2074 			return -EFAULT;
2075 
2076 		break;
2077 	}
2078 	case HDIO_DRIVE_TASK:
2079 	{
2080 		u8 drive_command[7];
2081 
2082 		/* Copy the user command info to our buffer. */
2083 		if (copy_from_user(drive_command,
2084 					 (void __user *) arg,
2085 					 sizeof(drive_command)))
2086 			return -EFAULT;
2087 
2088 		/* Execute the drive command. */
2089 		if (exec_drive_task(dd->port, drive_command))
2090 			return -EIO;
2091 
2092 		/* Copy the status back to the users buffer. */
2093 		if (copy_to_user((void __user *) arg,
2094 					 drive_command,
2095 					 sizeof(drive_command)))
2096 			return -EFAULT;
2097 
2098 		break;
2099 	}
2100 	case HDIO_DRIVE_TASKFILE: {
2101 		ide_task_request_t req_task;
2102 		int ret, outtotal;
2103 
2104 		if (copy_from_user(&req_task, (void __user *) arg,
2105 					sizeof(req_task)))
2106 			return -EFAULT;
2107 
2108 		outtotal = sizeof(req_task);
2109 
2110 		ret = exec_drive_taskfile(dd, (void __user *) arg,
2111 						&req_task, outtotal);
2112 
2113 		if (copy_to_user((void __user *) arg, &req_task,
2114 							sizeof(req_task)))
2115 			return -EFAULT;
2116 
2117 		return ret;
2118 	}
2119 
2120 	default:
2121 		return -EINVAL;
2122 	}
2123 	return 0;
2124 }
2125 
2126 /*
2127  * Submit an IO to the hw
2128  *
2129  * This function is called by the block layer to issue an io
2130  * to the device. Upon completion, the callback function will
2131  * be called with the data parameter passed as the callback data.
2132  *
2133  * @dd       Pointer to the driver data structure.
2134  * @start    First sector to read.
2135  * @nsect    Number of sectors to read.
2136  * @tag      The tag of this read command.
2137  * @callback Pointer to the function that should be called
2138  *	     when the read completes.
2139  * @data     Callback data passed to the callback function
2140  *	     when the read completes.
2141  * @dir      Direction (read or write)
2142  *
2143  * return value
2144  *	None
2145  */
2146 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq,
2147 			      struct mtip_cmd *command,
2148 			      struct blk_mq_hw_ctx *hctx)
2149 {
2150 	struct mtip_cmd_hdr *hdr =
2151 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
2152 	struct host_to_dev_fis	*fis;
2153 	struct mtip_port *port = dd->port;
2154 	int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
2155 	u64 start = blk_rq_pos(rq);
2156 	unsigned int nsect = blk_rq_sectors(rq);
2157 	unsigned int nents;
2158 
2159 	/* Map the scatter list for DMA access */
2160 	nents = blk_rq_map_sg(hctx->queue, rq, command->sg);
2161 	nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir);
2162 
2163 	prefetch(&port->flags);
2164 
2165 	command->scatter_ents = nents;
2166 
2167 	/*
2168 	 * The number of retries for this command before it is
2169 	 * reported as a failure to the upper layers.
2170 	 */
2171 	command->retries = MTIP_MAX_RETRIES;
2172 
2173 	/* Fill out fis */
2174 	fis = command->command;
2175 	fis->type        = 0x27;
2176 	fis->opts        = 1 << 7;
2177 	if (dma_dir == DMA_FROM_DEVICE)
2178 		fis->command = ATA_CMD_FPDMA_READ;
2179 	else
2180 		fis->command = ATA_CMD_FPDMA_WRITE;
2181 	fis->lba_low     = start & 0xFF;
2182 	fis->lba_mid     = (start >> 8) & 0xFF;
2183 	fis->lba_hi      = (start >> 16) & 0xFF;
2184 	fis->lba_low_ex  = (start >> 24) & 0xFF;
2185 	fis->lba_mid_ex  = (start >> 32) & 0xFF;
2186 	fis->lba_hi_ex   = (start >> 40) & 0xFF;
2187 	fis->device	 = 1 << 6;
2188 	fis->features    = nsect & 0xFF;
2189 	fis->features_ex = (nsect >> 8) & 0xFF;
2190 	fis->sect_count  = ((rq->tag << 3) | (rq->tag >> 5));
2191 	fis->sect_cnt_ex = 0;
2192 	fis->control     = 0;
2193 	fis->res2        = 0;
2194 	fis->res3        = 0;
2195 	fill_command_sg(dd, command, nents);
2196 
2197 	if (unlikely(command->unaligned))
2198 		fis->device |= 1 << 7;
2199 
2200 	/* Populate the command header */
2201 	hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF);
2202 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
2203 		hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16);
2204 	hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH);
2205 	hdr->byte_count = 0;
2206 
2207 	command->direction = dma_dir;
2208 
2209 	/*
2210 	 * To prevent this command from being issued
2211 	 * if an internal command is in progress or error handling is active.
2212 	 */
2213 	if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) {
2214 		set_bit(rq->tag, port->cmds_to_issue);
2215 		set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2216 		return;
2217 	}
2218 
2219 	/* Issue the command to the hardware */
2220 	mtip_issue_ncq_command(port, rq->tag);
2221 }
2222 
2223 /*
2224  * Sysfs status dump.
2225  *
2226  * @dev  Pointer to the device structure, passed by the kernrel.
2227  * @attr Pointer to the device_attribute structure passed by the kernel.
2228  * @buf  Pointer to the char buffer that will receive the stats info.
2229  *
2230  * return value
2231  *	The size, in bytes, of the data copied into buf.
2232  */
2233 static ssize_t mtip_hw_show_status(struct device *dev,
2234 				struct device_attribute *attr,
2235 				char *buf)
2236 {
2237 	struct driver_data *dd = dev_to_disk(dev)->private_data;
2238 	int size = 0;
2239 
2240 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
2241 		size += sprintf(buf, "%s", "thermal_shutdown\n");
2242 	else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag))
2243 		size += sprintf(buf, "%s", "write_protect\n");
2244 	else
2245 		size += sprintf(buf, "%s", "online\n");
2246 
2247 	return size;
2248 }
2249 
2250 static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
2251 
2252 /* debugsfs entries */
2253 
2254 static ssize_t show_device_status(struct device_driver *drv, char *buf)
2255 {
2256 	int size = 0;
2257 	struct driver_data *dd, *tmp;
2258 	unsigned long flags;
2259 	char id_buf[42];
2260 	u16 status = 0;
2261 
2262 	spin_lock_irqsave(&dev_lock, flags);
2263 	size += sprintf(&buf[size], "Devices Present:\n");
2264 	list_for_each_entry_safe(dd, tmp, &online_list, online_list) {
2265 		if (dd->pdev) {
2266 			if (dd->port &&
2267 			    dd->port->identify &&
2268 			    dd->port->identify_valid) {
2269 				strlcpy(id_buf,
2270 					(char *) (dd->port->identify + 10), 21);
2271 				status = *(dd->port->identify + 141);
2272 			} else {
2273 				memset(id_buf, 0, 42);
2274 				status = 0;
2275 			}
2276 
2277 			if (dd->port &&
2278 			    test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2279 				size += sprintf(&buf[size],
2280 					" device %s %s (ftl rebuild %d %%)\n",
2281 					dev_name(&dd->pdev->dev),
2282 					id_buf,
2283 					status);
2284 			} else {
2285 				size += sprintf(&buf[size],
2286 					" device %s %s\n",
2287 					dev_name(&dd->pdev->dev),
2288 					id_buf);
2289 			}
2290 		}
2291 	}
2292 
2293 	size += sprintf(&buf[size], "Devices Being Removed:\n");
2294 	list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) {
2295 		if (dd->pdev) {
2296 			if (dd->port &&
2297 			    dd->port->identify &&
2298 			    dd->port->identify_valid) {
2299 				strlcpy(id_buf,
2300 					(char *) (dd->port->identify+10), 21);
2301 				status = *(dd->port->identify + 141);
2302 			} else {
2303 				memset(id_buf, 0, 42);
2304 				status = 0;
2305 			}
2306 
2307 			if (dd->port &&
2308 			    test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2309 				size += sprintf(&buf[size],
2310 					" device %s %s (ftl rebuild %d %%)\n",
2311 					dev_name(&dd->pdev->dev),
2312 					id_buf,
2313 					status);
2314 			} else {
2315 				size += sprintf(&buf[size],
2316 					" device %s %s\n",
2317 					dev_name(&dd->pdev->dev),
2318 					id_buf);
2319 			}
2320 		}
2321 	}
2322 	spin_unlock_irqrestore(&dev_lock, flags);
2323 
2324 	return size;
2325 }
2326 
2327 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
2328 						size_t len, loff_t *offset)
2329 {
2330 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2331 	int size = *offset;
2332 	char *buf;
2333 	int rv = 0;
2334 
2335 	if (!len || *offset)
2336 		return 0;
2337 
2338 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2339 	if (!buf) {
2340 		dev_err(&dd->pdev->dev,
2341 			"Memory allocation: status buffer\n");
2342 		return -ENOMEM;
2343 	}
2344 
2345 	size += show_device_status(NULL, buf);
2346 
2347 	*offset = size <= len ? size : len;
2348 	size = copy_to_user(ubuf, buf, *offset);
2349 	if (size)
2350 		rv = -EFAULT;
2351 
2352 	kfree(buf);
2353 	return rv ? rv : *offset;
2354 }
2355 
2356 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
2357 				  size_t len, loff_t *offset)
2358 {
2359 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2360 	char *buf;
2361 	u32 group_allocated;
2362 	int size = *offset;
2363 	int n, rv = 0;
2364 
2365 	if (!len || size)
2366 		return 0;
2367 
2368 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2369 	if (!buf) {
2370 		dev_err(&dd->pdev->dev,
2371 			"Memory allocation: register buffer\n");
2372 		return -ENOMEM;
2373 	}
2374 
2375 	size += sprintf(&buf[size], "H/ S ACTive      : [ 0x");
2376 
2377 	for (n = dd->slot_groups-1; n >= 0; n--)
2378 		size += sprintf(&buf[size], "%08X ",
2379 					 readl(dd->port->s_active[n]));
2380 
2381 	size += sprintf(&buf[size], "]\n");
2382 	size += sprintf(&buf[size], "H/ Command Issue : [ 0x");
2383 
2384 	for (n = dd->slot_groups-1; n >= 0; n--)
2385 		size += sprintf(&buf[size], "%08X ",
2386 					readl(dd->port->cmd_issue[n]));
2387 
2388 	size += sprintf(&buf[size], "]\n");
2389 	size += sprintf(&buf[size], "H/ Completed     : [ 0x");
2390 
2391 	for (n = dd->slot_groups-1; n >= 0; n--)
2392 		size += sprintf(&buf[size], "%08X ",
2393 				readl(dd->port->completed[n]));
2394 
2395 	size += sprintf(&buf[size], "]\n");
2396 	size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n",
2397 				readl(dd->port->mmio + PORT_IRQ_STAT));
2398 	size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n",
2399 				readl(dd->mmio + HOST_IRQ_STAT));
2400 	size += sprintf(&buf[size], "\n");
2401 
2402 	size += sprintf(&buf[size], "L/ Commands in Q : [ 0x");
2403 
2404 	for (n = dd->slot_groups-1; n >= 0; n--) {
2405 		if (sizeof(long) > sizeof(u32))
2406 			group_allocated =
2407 				dd->port->cmds_to_issue[n/2] >> (32*(n&1));
2408 		else
2409 			group_allocated = dd->port->cmds_to_issue[n];
2410 		size += sprintf(&buf[size], "%08X ", group_allocated);
2411 	}
2412 	size += sprintf(&buf[size], "]\n");
2413 
2414 	*offset = size <= len ? size : len;
2415 	size = copy_to_user(ubuf, buf, *offset);
2416 	if (size)
2417 		rv = -EFAULT;
2418 
2419 	kfree(buf);
2420 	return rv ? rv : *offset;
2421 }
2422 
2423 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
2424 				  size_t len, loff_t *offset)
2425 {
2426 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2427 	char *buf;
2428 	int size = *offset;
2429 	int rv = 0;
2430 
2431 	if (!len || size)
2432 		return 0;
2433 
2434 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2435 	if (!buf) {
2436 		dev_err(&dd->pdev->dev,
2437 			"Memory allocation: flag buffer\n");
2438 		return -ENOMEM;
2439 	}
2440 
2441 	size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
2442 							dd->port->flags);
2443 	size += sprintf(&buf[size], "Flag-dd   : [ %08lX ]\n",
2444 							dd->dd_flag);
2445 
2446 	*offset = size <= len ? size : len;
2447 	size = copy_to_user(ubuf, buf, *offset);
2448 	if (size)
2449 		rv = -EFAULT;
2450 
2451 	kfree(buf);
2452 	return rv ? rv : *offset;
2453 }
2454 
2455 static const struct file_operations mtip_device_status_fops = {
2456 	.owner  = THIS_MODULE,
2457 	.open   = simple_open,
2458 	.read   = mtip_hw_read_device_status,
2459 	.llseek = no_llseek,
2460 };
2461 
2462 static const struct file_operations mtip_regs_fops = {
2463 	.owner  = THIS_MODULE,
2464 	.open   = simple_open,
2465 	.read   = mtip_hw_read_registers,
2466 	.llseek = no_llseek,
2467 };
2468 
2469 static const struct file_operations mtip_flags_fops = {
2470 	.owner  = THIS_MODULE,
2471 	.open   = simple_open,
2472 	.read   = mtip_hw_read_flags,
2473 	.llseek = no_llseek,
2474 };
2475 
2476 /*
2477  * Create the sysfs related attributes.
2478  *
2479  * @dd   Pointer to the driver data structure.
2480  * @kobj Pointer to the kobj for the block device.
2481  *
2482  * return value
2483  *	0	Operation completed successfully.
2484  *	-EINVAL Invalid parameter.
2485  */
2486 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj)
2487 {
2488 	if (!kobj || !dd)
2489 		return -EINVAL;
2490 
2491 	if (sysfs_create_file(kobj, &dev_attr_status.attr))
2492 		dev_warn(&dd->pdev->dev,
2493 			"Error creating 'status' sysfs entry\n");
2494 	return 0;
2495 }
2496 
2497 /*
2498  * Remove the sysfs related attributes.
2499  *
2500  * @dd   Pointer to the driver data structure.
2501  * @kobj Pointer to the kobj for the block device.
2502  *
2503  * return value
2504  *	0	Operation completed successfully.
2505  *	-EINVAL Invalid parameter.
2506  */
2507 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj)
2508 {
2509 	if (!kobj || !dd)
2510 		return -EINVAL;
2511 
2512 	sysfs_remove_file(kobj, &dev_attr_status.attr);
2513 
2514 	return 0;
2515 }
2516 
2517 static int mtip_hw_debugfs_init(struct driver_data *dd)
2518 {
2519 	if (!dfs_parent)
2520 		return -1;
2521 
2522 	dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent);
2523 	if (IS_ERR_OR_NULL(dd->dfs_node)) {
2524 		dev_warn(&dd->pdev->dev,
2525 			"Error creating node %s under debugfs\n",
2526 						dd->disk->disk_name);
2527 		dd->dfs_node = NULL;
2528 		return -1;
2529 	}
2530 
2531 	debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
2532 	debugfs_create_file("registers", 0444, dd->dfs_node, dd,
2533 			    &mtip_regs_fops);
2534 
2535 	return 0;
2536 }
2537 
2538 static void mtip_hw_debugfs_exit(struct driver_data *dd)
2539 {
2540 	debugfs_remove_recursive(dd->dfs_node);
2541 }
2542 
2543 /*
2544  * Perform any init/resume time hardware setup
2545  *
2546  * @dd Pointer to the driver data structure.
2547  *
2548  * return value
2549  *	None
2550  */
2551 static inline void hba_setup(struct driver_data *dd)
2552 {
2553 	u32 hwdata;
2554 	hwdata = readl(dd->mmio + HOST_HSORG);
2555 
2556 	/* interrupt bug workaround: use only 1 IS bit.*/
2557 	writel(hwdata |
2558 		HSORG_DISABLE_SLOTGRP_INTR |
2559 		HSORG_DISABLE_SLOTGRP_PXIS,
2560 		dd->mmio + HOST_HSORG);
2561 }
2562 
2563 static int mtip_device_unaligned_constrained(struct driver_data *dd)
2564 {
2565 	return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0);
2566 }
2567 
2568 /*
2569  * Detect the details of the product, and store anything needed
2570  * into the driver data structure.  This includes product type and
2571  * version and number of slot groups.
2572  *
2573  * @dd Pointer to the driver data structure.
2574  *
2575  * return value
2576  *	None
2577  */
2578 static void mtip_detect_product(struct driver_data *dd)
2579 {
2580 	u32 hwdata;
2581 	unsigned int rev, slotgroups;
2582 
2583 	/*
2584 	 * HBA base + 0xFC [15:0] - vendor-specific hardware interface
2585 	 * info register:
2586 	 * [15:8] hardware/software interface rev#
2587 	 * [   3] asic-style interface
2588 	 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
2589 	 */
2590 	hwdata = readl(dd->mmio + HOST_HSORG);
2591 
2592 	dd->product_type = MTIP_PRODUCT_UNKNOWN;
2593 	dd->slot_groups = 1;
2594 
2595 	if (hwdata & 0x8) {
2596 		dd->product_type = MTIP_PRODUCT_ASICFPGA;
2597 		rev = (hwdata & HSORG_HWREV) >> 8;
2598 		slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
2599 		dev_info(&dd->pdev->dev,
2600 			"ASIC-FPGA design, HS rev 0x%x, "
2601 			"%i slot groups [%i slots]\n",
2602 			 rev,
2603 			 slotgroups,
2604 			 slotgroups * 32);
2605 
2606 		if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
2607 			dev_warn(&dd->pdev->dev,
2608 				"Warning: driver only supports "
2609 				"%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
2610 			slotgroups = MTIP_MAX_SLOT_GROUPS;
2611 		}
2612 		dd->slot_groups = slotgroups;
2613 		return;
2614 	}
2615 
2616 	dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
2617 }
2618 
2619 /*
2620  * Blocking wait for FTL rebuild to complete
2621  *
2622  * @dd Pointer to the DRIVER_DATA structure.
2623  *
2624  * return value
2625  *	0	FTL rebuild completed successfully
2626  *	-EFAULT FTL rebuild error/timeout/interruption
2627  */
2628 static int mtip_ftl_rebuild_poll(struct driver_data *dd)
2629 {
2630 	unsigned long timeout, cnt = 0, start;
2631 
2632 	dev_warn(&dd->pdev->dev,
2633 		"FTL rebuild in progress. Polling for completion.\n");
2634 
2635 	start = jiffies;
2636 	timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
2637 
2638 	do {
2639 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2640 				&dd->dd_flag)))
2641 			return -EFAULT;
2642 		if (mtip_check_surprise_removal(dd->pdev))
2643 			return -EFAULT;
2644 
2645 		if (mtip_get_identify(dd->port, NULL) < 0)
2646 			return -EFAULT;
2647 
2648 		if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2649 			MTIP_FTL_REBUILD_MAGIC) {
2650 			ssleep(1);
2651 			/* Print message every 3 minutes */
2652 			if (cnt++ >= 180) {
2653 				dev_warn(&dd->pdev->dev,
2654 				"FTL rebuild in progress (%d secs).\n",
2655 				jiffies_to_msecs(jiffies - start) / 1000);
2656 				cnt = 0;
2657 			}
2658 		} else {
2659 			dev_warn(&dd->pdev->dev,
2660 				"FTL rebuild complete (%d secs).\n",
2661 			jiffies_to_msecs(jiffies - start) / 1000);
2662 			mtip_block_initialize(dd);
2663 			return 0;
2664 		}
2665 	} while (time_before(jiffies, timeout));
2666 
2667 	/* Check for timeout */
2668 	dev_err(&dd->pdev->dev,
2669 		"Timed out waiting for FTL rebuild to complete (%d secs).\n",
2670 		jiffies_to_msecs(jiffies - start) / 1000);
2671 	return -EFAULT;
2672 }
2673 
2674 static void mtip_softirq_done_fn(struct request *rq)
2675 {
2676 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
2677 	struct driver_data *dd = rq->q->queuedata;
2678 
2679 	/* Unmap the DMA scatter list entries */
2680 	dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents,
2681 							cmd->direction);
2682 
2683 	if (unlikely(cmd->unaligned))
2684 		atomic_inc(&dd->port->cmd_slot_unal);
2685 
2686 	blk_mq_end_request(rq, cmd->status);
2687 }
2688 
2689 static bool mtip_abort_cmd(struct request *req, void *data, bool reserved)
2690 {
2691 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
2692 	struct driver_data *dd = data;
2693 
2694 	dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
2695 
2696 	clear_bit(req->tag, dd->port->cmds_to_issue);
2697 	cmd->status = BLK_STS_IOERR;
2698 	mtip_softirq_done_fn(req);
2699 	return true;
2700 }
2701 
2702 static bool mtip_queue_cmd(struct request *req, void *data, bool reserved)
2703 {
2704 	struct driver_data *dd = data;
2705 
2706 	set_bit(req->tag, dd->port->cmds_to_issue);
2707 	blk_abort_request(req);
2708 	return true;
2709 }
2710 
2711 /*
2712  * service thread to issue queued commands
2713  *
2714  * @data Pointer to the driver data structure.
2715  *
2716  * return value
2717  *	0
2718  */
2719 
2720 static int mtip_service_thread(void *data)
2721 {
2722 	struct driver_data *dd = (struct driver_data *)data;
2723 	unsigned long slot, slot_start, slot_wrap, to;
2724 	unsigned int num_cmd_slots = dd->slot_groups * 32;
2725 	struct mtip_port *port = dd->port;
2726 
2727 	while (1) {
2728 		if (kthread_should_stop() ||
2729 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2730 			goto st_out;
2731 		clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2732 
2733 		/*
2734 		 * the condition is to check neither an internal command is
2735 		 * is in progress nor error handling is active
2736 		 */
2737 		wait_event_interruptible(port->svc_wait, (port->flags) &&
2738 			(port->flags & MTIP_PF_SVC_THD_WORK));
2739 
2740 		if (kthread_should_stop() ||
2741 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2742 			goto st_out;
2743 
2744 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2745 				&dd->dd_flag)))
2746 			goto st_out;
2747 
2748 		set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2749 
2750 restart_eh:
2751 		/* Demux bits: start with error handling */
2752 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) {
2753 			mtip_handle_tfe(dd);
2754 			clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags);
2755 		}
2756 
2757 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags))
2758 			goto restart_eh;
2759 
2760 		if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) {
2761 			to = jiffies + msecs_to_jiffies(5000);
2762 
2763 			do {
2764 				mdelay(100);
2765 			} while (atomic_read(&dd->irq_workers_active) != 0 &&
2766 				time_before(jiffies, to));
2767 
2768 			if (atomic_read(&dd->irq_workers_active) != 0)
2769 				dev_warn(&dd->pdev->dev,
2770 					"Completion workers still active!");
2771 
2772 			blk_mq_quiesce_queue(dd->queue);
2773 
2774 			blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd);
2775 
2776 			set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags);
2777 
2778 			if (mtip_device_reset(dd))
2779 				blk_mq_tagset_busy_iter(&dd->tags,
2780 							mtip_abort_cmd, dd);
2781 
2782 			clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);
2783 
2784 			blk_mq_unquiesce_queue(dd->queue);
2785 		}
2786 
2787 		if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
2788 			slot = 1;
2789 			/* used to restrict the loop to one iteration */
2790 			slot_start = num_cmd_slots;
2791 			slot_wrap = 0;
2792 			while (1) {
2793 				slot = find_next_bit(port->cmds_to_issue,
2794 						num_cmd_slots, slot);
2795 				if (slot_wrap == 1) {
2796 					if ((slot_start >= slot) ||
2797 						(slot >= num_cmd_slots))
2798 						break;
2799 				}
2800 				if (unlikely(slot_start == num_cmd_slots))
2801 					slot_start = slot;
2802 
2803 				if (unlikely(slot == num_cmd_slots)) {
2804 					slot = 1;
2805 					slot_wrap = 1;
2806 					continue;
2807 				}
2808 
2809 				/* Issue the command to the hardware */
2810 				mtip_issue_ncq_command(port, slot);
2811 
2812 				clear_bit(slot, port->cmds_to_issue);
2813 			}
2814 
2815 			clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2816 		}
2817 
2818 		if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
2819 			if (mtip_ftl_rebuild_poll(dd) == 0)
2820 				clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
2821 		}
2822 	}
2823 
2824 st_out:
2825 	return 0;
2826 }
2827 
2828 /*
2829  * DMA region teardown
2830  *
2831  * @dd Pointer to driver_data structure
2832  *
2833  * return value
2834  *      None
2835  */
2836 static void mtip_dma_free(struct driver_data *dd)
2837 {
2838 	struct mtip_port *port = dd->port;
2839 
2840 	if (port->block1)
2841 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2842 					port->block1, port->block1_dma);
2843 
2844 	if (port->command_list) {
2845 		dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2846 				port->command_list, port->command_list_dma);
2847 	}
2848 }
2849 
2850 /*
2851  * DMA region setup
2852  *
2853  * @dd Pointer to driver_data structure
2854  *
2855  * return value
2856  *      -ENOMEM Not enough free DMA region space to initialize driver
2857  */
2858 static int mtip_dma_alloc(struct driver_data *dd)
2859 {
2860 	struct mtip_port *port = dd->port;
2861 
2862 	/* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */
2863 	port->block1 =
2864 		dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2865 					&port->block1_dma, GFP_KERNEL);
2866 	if (!port->block1)
2867 		return -ENOMEM;
2868 	memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ);
2869 
2870 	/* Allocate dma memory for command list */
2871 	port->command_list =
2872 		dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2873 					&port->command_list_dma, GFP_KERNEL);
2874 	if (!port->command_list) {
2875 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2876 					port->block1, port->block1_dma);
2877 		port->block1 = NULL;
2878 		port->block1_dma = 0;
2879 		return -ENOMEM;
2880 	}
2881 	memset(port->command_list, 0, AHCI_CMD_TBL_SZ);
2882 
2883 	/* Setup all pointers into first DMA region */
2884 	port->rxfis         = port->block1 + AHCI_RX_FIS_OFFSET;
2885 	port->rxfis_dma     = port->block1_dma + AHCI_RX_FIS_OFFSET;
2886 	port->identify      = port->block1 + AHCI_IDFY_OFFSET;
2887 	port->identify_dma  = port->block1_dma + AHCI_IDFY_OFFSET;
2888 	port->log_buf       = port->block1 + AHCI_SECTBUF_OFFSET;
2889 	port->log_buf_dma   = port->block1_dma + AHCI_SECTBUF_OFFSET;
2890 	port->smart_buf     = port->block1 + AHCI_SMARTBUF_OFFSET;
2891 	port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET;
2892 
2893 	return 0;
2894 }
2895 
2896 static int mtip_hw_get_identify(struct driver_data *dd)
2897 {
2898 	struct smart_attr attr242;
2899 	unsigned char *buf;
2900 	int rv;
2901 
2902 	if (mtip_get_identify(dd->port, NULL) < 0)
2903 		return -EFAULT;
2904 
2905 	if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2906 		MTIP_FTL_REBUILD_MAGIC) {
2907 		set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
2908 		return MTIP_FTL_REBUILD_MAGIC;
2909 	}
2910 	mtip_dump_identify(dd->port);
2911 
2912 	/* check write protect, over temp and rebuild statuses */
2913 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
2914 				dd->port->log_buf,
2915 				dd->port->log_buf_dma, 1);
2916 	if (rv) {
2917 		dev_warn(&dd->pdev->dev,
2918 			"Error in READ LOG EXT (10h) command\n");
2919 		/* non-critical error, don't fail the load */
2920 	} else {
2921 		buf = (unsigned char *)dd->port->log_buf;
2922 		if (buf[259] & 0x1) {
2923 			dev_info(&dd->pdev->dev,
2924 				"Write protect bit is set.\n");
2925 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
2926 		}
2927 		if (buf[288] == 0xF7) {
2928 			dev_info(&dd->pdev->dev,
2929 				"Exceeded Tmax, drive in thermal shutdown.\n");
2930 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
2931 		}
2932 		if (buf[288] == 0xBF) {
2933 			dev_info(&dd->pdev->dev,
2934 				"Drive indicates rebuild has failed.\n");
2935 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
2936 		}
2937 	}
2938 
2939 	/* get write protect progess */
2940 	memset(&attr242, 0, sizeof(struct smart_attr));
2941 	if (mtip_get_smart_attr(dd->port, 242, &attr242))
2942 		dev_warn(&dd->pdev->dev,
2943 				"Unable to check write protect progress\n");
2944 	else
2945 		dev_info(&dd->pdev->dev,
2946 				"Write protect progress: %u%% (%u blocks)\n",
2947 				attr242.cur, le32_to_cpu(attr242.data));
2948 
2949 	return rv;
2950 }
2951 
2952 /*
2953  * Called once for each card.
2954  *
2955  * @dd Pointer to the driver data structure.
2956  *
2957  * return value
2958  *	0 on success, else an error code.
2959  */
2960 static int mtip_hw_init(struct driver_data *dd)
2961 {
2962 	int i;
2963 	int rv;
2964 	unsigned long timeout, timetaken;
2965 
2966 	dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
2967 
2968 	mtip_detect_product(dd);
2969 	if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
2970 		rv = -EIO;
2971 		goto out1;
2972 	}
2973 
2974 	hba_setup(dd);
2975 
2976 	dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL,
2977 				dd->numa_node);
2978 	if (!dd->port) {
2979 		dev_err(&dd->pdev->dev,
2980 			"Memory allocation: port structure\n");
2981 		return -ENOMEM;
2982 	}
2983 
2984 	/* Continue workqueue setup */
2985 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2986 		dd->work[i].port = dd->port;
2987 
2988 	/* Enable unaligned IO constraints for some devices */
2989 	if (mtip_device_unaligned_constrained(dd))
2990 		dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS;
2991 	else
2992 		dd->unal_qdepth = 0;
2993 
2994 	atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth);
2995 
2996 	/* Spinlock to prevent concurrent issue */
2997 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2998 		spin_lock_init(&dd->port->cmd_issue_lock[i]);
2999 
3000 	/* Set the port mmio base address. */
3001 	dd->port->mmio	= dd->mmio + PORT_OFFSET;
3002 	dd->port->dd	= dd;
3003 
3004 	/* DMA allocations */
3005 	rv = mtip_dma_alloc(dd);
3006 	if (rv < 0)
3007 		goto out1;
3008 
3009 	/* Setup the pointers to the extended s_active and CI registers. */
3010 	for (i = 0; i < dd->slot_groups; i++) {
3011 		dd->port->s_active[i] =
3012 			dd->port->mmio + i*0x80 + PORT_SCR_ACT;
3013 		dd->port->cmd_issue[i] =
3014 			dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
3015 		dd->port->completed[i] =
3016 			dd->port->mmio + i*0x80 + PORT_SDBV;
3017 	}
3018 
3019 	timetaken = jiffies;
3020 	timeout = jiffies + msecs_to_jiffies(30000);
3021 	while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) &&
3022 		 time_before(jiffies, timeout)) {
3023 		mdelay(100);
3024 	}
3025 	if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
3026 		timetaken = jiffies - timetaken;
3027 		dev_warn(&dd->pdev->dev,
3028 			"Surprise removal detected at %u ms\n",
3029 			jiffies_to_msecs(timetaken));
3030 		rv = -ENODEV;
3031 		goto out2 ;
3032 	}
3033 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) {
3034 		timetaken = jiffies - timetaken;
3035 		dev_warn(&dd->pdev->dev,
3036 			"Removal detected at %u ms\n",
3037 			jiffies_to_msecs(timetaken));
3038 		rv = -EFAULT;
3039 		goto out2;
3040 	}
3041 
3042 	/* Conditionally reset the HBA. */
3043 	if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) {
3044 		if (mtip_hba_reset(dd) < 0) {
3045 			dev_err(&dd->pdev->dev,
3046 				"Card did not reset within timeout\n");
3047 			rv = -EIO;
3048 			goto out2;
3049 		}
3050 	} else {
3051 		/* Clear any pending interrupts on the HBA */
3052 		writel(readl(dd->mmio + HOST_IRQ_STAT),
3053 			dd->mmio + HOST_IRQ_STAT);
3054 	}
3055 
3056 	mtip_init_port(dd->port);
3057 	mtip_start_port(dd->port);
3058 
3059 	/* Setup the ISR and enable interrupts. */
3060 	rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED,
3061 			 dev_driver_string(&dd->pdev->dev), dd);
3062 	if (rv) {
3063 		dev_err(&dd->pdev->dev,
3064 			"Unable to allocate IRQ %d\n", dd->pdev->irq);
3065 		goto out2;
3066 	}
3067 	irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding));
3068 
3069 	/* Enable interrupts on the HBA. */
3070 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
3071 					dd->mmio + HOST_CTL);
3072 
3073 	init_waitqueue_head(&dd->port->svc_wait);
3074 
3075 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
3076 		rv = -EFAULT;
3077 		goto out3;
3078 	}
3079 
3080 	return rv;
3081 
3082 out3:
3083 	/* Disable interrupts on the HBA. */
3084 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3085 			dd->mmio + HOST_CTL);
3086 
3087 	/* Release the IRQ. */
3088 	irq_set_affinity_hint(dd->pdev->irq, NULL);
3089 	free_irq(dd->pdev->irq, dd);
3090 
3091 out2:
3092 	mtip_deinit_port(dd->port);
3093 	mtip_dma_free(dd);
3094 
3095 out1:
3096 	/* Free the memory allocated for the for structure. */
3097 	kfree(dd->port);
3098 
3099 	return rv;
3100 }
3101 
3102 static int mtip_standby_drive(struct driver_data *dd)
3103 {
3104 	int rv = 0;
3105 
3106 	if (dd->sr || !dd->port)
3107 		return -ENODEV;
3108 	/*
3109 	 * Send standby immediate (E0h) to the drive so that it
3110 	 * saves its state.
3111 	 */
3112 	if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) &&
3113 	    !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) &&
3114 	    !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) {
3115 		rv = mtip_standby_immediate(dd->port);
3116 		if (rv)
3117 			dev_warn(&dd->pdev->dev,
3118 				"STANDBY IMMEDIATE failed\n");
3119 	}
3120 	return rv;
3121 }
3122 
3123 /*
3124  * Called to deinitialize an interface.
3125  *
3126  * @dd Pointer to the driver data structure.
3127  *
3128  * return value
3129  *	0
3130  */
3131 static int mtip_hw_exit(struct driver_data *dd)
3132 {
3133 	if (!dd->sr) {
3134 		/* de-initialize the port. */
3135 		mtip_deinit_port(dd->port);
3136 
3137 		/* Disable interrupts on the HBA. */
3138 		writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3139 				dd->mmio + HOST_CTL);
3140 	}
3141 
3142 	/* Release the IRQ. */
3143 	irq_set_affinity_hint(dd->pdev->irq, NULL);
3144 	free_irq(dd->pdev->irq, dd);
3145 	msleep(1000);
3146 
3147 	/* Free dma regions */
3148 	mtip_dma_free(dd);
3149 
3150 	/* Free the memory allocated for the for structure. */
3151 	kfree(dd->port);
3152 	dd->port = NULL;
3153 
3154 	return 0;
3155 }
3156 
3157 /*
3158  * Issue a Standby Immediate command to the device.
3159  *
3160  * This function is called by the Block Layer just before the
3161  * system powers off during a shutdown.
3162  *
3163  * @dd Pointer to the driver data structure.
3164  *
3165  * return value
3166  *	0
3167  */
3168 static int mtip_hw_shutdown(struct driver_data *dd)
3169 {
3170 	/*
3171 	 * Send standby immediate (E0h) to the drive so that it
3172 	 * saves its state.
3173 	 */
3174 	mtip_standby_drive(dd);
3175 
3176 	return 0;
3177 }
3178 
3179 /*
3180  * Suspend function
3181  *
3182  * This function is called by the Block Layer just before the
3183  * system hibernates.
3184  *
3185  * @dd Pointer to the driver data structure.
3186  *
3187  * return value
3188  *	0	Suspend was successful
3189  *	-EFAULT Suspend was not successful
3190  */
3191 static int mtip_hw_suspend(struct driver_data *dd)
3192 {
3193 	/*
3194 	 * Send standby immediate (E0h) to the drive
3195 	 * so that it saves its state.
3196 	 */
3197 	if (mtip_standby_drive(dd) != 0) {
3198 		dev_err(&dd->pdev->dev,
3199 			"Failed standby-immediate command\n");
3200 		return -EFAULT;
3201 	}
3202 
3203 	/* Disable interrupts on the HBA.*/
3204 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3205 			dd->mmio + HOST_CTL);
3206 	mtip_deinit_port(dd->port);
3207 
3208 	return 0;
3209 }
3210 
3211 /*
3212  * Resume function
3213  *
3214  * This function is called by the Block Layer as the
3215  * system resumes.
3216  *
3217  * @dd Pointer to the driver data structure.
3218  *
3219  * return value
3220  *	0	Resume was successful
3221  *      -EFAULT Resume was not successful
3222  */
3223 static int mtip_hw_resume(struct driver_data *dd)
3224 {
3225 	/* Perform any needed hardware setup steps */
3226 	hba_setup(dd);
3227 
3228 	/* Reset the HBA */
3229 	if (mtip_hba_reset(dd) != 0) {
3230 		dev_err(&dd->pdev->dev,
3231 			"Unable to reset the HBA\n");
3232 		return -EFAULT;
3233 	}
3234 
3235 	/*
3236 	 * Enable the port, DMA engine, and FIS reception specific
3237 	 * h/w in controller.
3238 	 */
3239 	mtip_init_port(dd->port);
3240 	mtip_start_port(dd->port);
3241 
3242 	/* Enable interrupts on the HBA.*/
3243 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
3244 			dd->mmio + HOST_CTL);
3245 
3246 	return 0;
3247 }
3248 
3249 /*
3250  * Helper function for reusing disk name
3251  * upon hot insertion.
3252  */
3253 static int rssd_disk_name_format(char *prefix,
3254 				 int index,
3255 				 char *buf,
3256 				 int buflen)
3257 {
3258 	const int base = 'z' - 'a' + 1;
3259 	char *begin = buf + strlen(prefix);
3260 	char *end = buf + buflen;
3261 	char *p;
3262 	int unit;
3263 
3264 	p = end - 1;
3265 	*p = '\0';
3266 	unit = base;
3267 	do {
3268 		if (p == begin)
3269 			return -EINVAL;
3270 		*--p = 'a' + (index % unit);
3271 		index = (index / unit) - 1;
3272 	} while (index >= 0);
3273 
3274 	memmove(begin, p, end - p);
3275 	memcpy(buf, prefix, strlen(prefix));
3276 
3277 	return 0;
3278 }
3279 
3280 /*
3281  * Block layer IOCTL handler.
3282  *
3283  * @dev Pointer to the block_device structure.
3284  * @mode ignored
3285  * @cmd IOCTL command passed from the user application.
3286  * @arg Argument passed from the user application.
3287  *
3288  * return value
3289  *	0        IOCTL completed successfully.
3290  *	-ENOTTY  IOCTL not supported or invalid driver data
3291  *                 structure pointer.
3292  */
3293 static int mtip_block_ioctl(struct block_device *dev,
3294 			    fmode_t mode,
3295 			    unsigned cmd,
3296 			    unsigned long arg)
3297 {
3298 	struct driver_data *dd = dev->bd_disk->private_data;
3299 
3300 	if (!capable(CAP_SYS_ADMIN))
3301 		return -EACCES;
3302 
3303 	if (!dd)
3304 		return -ENOTTY;
3305 
3306 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3307 		return -ENOTTY;
3308 
3309 	switch (cmd) {
3310 	case BLKFLSBUF:
3311 		return -ENOTTY;
3312 	default:
3313 		return mtip_hw_ioctl(dd, cmd, arg);
3314 	}
3315 }
3316 
3317 #ifdef CONFIG_COMPAT
3318 /*
3319  * Block layer compat IOCTL handler.
3320  *
3321  * @dev Pointer to the block_device structure.
3322  * @mode ignored
3323  * @cmd IOCTL command passed from the user application.
3324  * @arg Argument passed from the user application.
3325  *
3326  * return value
3327  *	0        IOCTL completed successfully.
3328  *	-ENOTTY  IOCTL not supported or invalid driver data
3329  *                 structure pointer.
3330  */
3331 static int mtip_block_compat_ioctl(struct block_device *dev,
3332 			    fmode_t mode,
3333 			    unsigned cmd,
3334 			    unsigned long arg)
3335 {
3336 	struct driver_data *dd = dev->bd_disk->private_data;
3337 
3338 	if (!capable(CAP_SYS_ADMIN))
3339 		return -EACCES;
3340 
3341 	if (!dd)
3342 		return -ENOTTY;
3343 
3344 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3345 		return -ENOTTY;
3346 
3347 	switch (cmd) {
3348 	case BLKFLSBUF:
3349 		return -ENOTTY;
3350 	case HDIO_DRIVE_TASKFILE: {
3351 		struct mtip_compat_ide_task_request_s __user *compat_req_task;
3352 		ide_task_request_t req_task;
3353 		int compat_tasksize, outtotal, ret;
3354 
3355 		compat_tasksize =
3356 			sizeof(struct mtip_compat_ide_task_request_s);
3357 
3358 		compat_req_task =
3359 			(struct mtip_compat_ide_task_request_s __user *) arg;
3360 
3361 		if (copy_from_user(&req_task, (void __user *) arg,
3362 			compat_tasksize - (2 * sizeof(compat_long_t))))
3363 			return -EFAULT;
3364 
3365 		if (get_user(req_task.out_size, &compat_req_task->out_size))
3366 			return -EFAULT;
3367 
3368 		if (get_user(req_task.in_size, &compat_req_task->in_size))
3369 			return -EFAULT;
3370 
3371 		outtotal = sizeof(struct mtip_compat_ide_task_request_s);
3372 
3373 		ret = exec_drive_taskfile(dd, (void __user *) arg,
3374 						&req_task, outtotal);
3375 
3376 		if (copy_to_user((void __user *) arg, &req_task,
3377 				compat_tasksize -
3378 				(2 * sizeof(compat_long_t))))
3379 			return -EFAULT;
3380 
3381 		if (put_user(req_task.out_size, &compat_req_task->out_size))
3382 			return -EFAULT;
3383 
3384 		if (put_user(req_task.in_size, &compat_req_task->in_size))
3385 			return -EFAULT;
3386 
3387 		return ret;
3388 	}
3389 	default:
3390 		return mtip_hw_ioctl(dd, cmd, arg);
3391 	}
3392 }
3393 #endif
3394 
3395 /*
3396  * Obtain the geometry of the device.
3397  *
3398  * You may think that this function is obsolete, but some applications,
3399  * fdisk for example still used CHS values. This function describes the
3400  * device as having 224 heads and 56 sectors per cylinder. These values are
3401  * chosen so that each cylinder is aligned on a 4KB boundary. Since a
3402  * partition is described in terms of a start and end cylinder this means
3403  * that each partition is also 4KB aligned. Non-aligned partitions adversely
3404  * affects performance.
3405  *
3406  * @dev Pointer to the block_device strucutre.
3407  * @geo Pointer to a hd_geometry structure.
3408  *
3409  * return value
3410  *	0       Operation completed successfully.
3411  *	-ENOTTY An error occurred while reading the drive capacity.
3412  */
3413 static int mtip_block_getgeo(struct block_device *dev,
3414 				struct hd_geometry *geo)
3415 {
3416 	struct driver_data *dd = dev->bd_disk->private_data;
3417 	sector_t capacity;
3418 
3419 	if (!dd)
3420 		return -ENOTTY;
3421 
3422 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3423 		dev_warn(&dd->pdev->dev,
3424 			"Could not get drive capacity.\n");
3425 		return -ENOTTY;
3426 	}
3427 
3428 	geo->heads = 224;
3429 	geo->sectors = 56;
3430 	sector_div(capacity, (geo->heads * geo->sectors));
3431 	geo->cylinders = capacity;
3432 	return 0;
3433 }
3434 
3435 static int mtip_block_open(struct block_device *dev, fmode_t mode)
3436 {
3437 	struct driver_data *dd;
3438 
3439 	if (dev && dev->bd_disk) {
3440 		dd = (struct driver_data *) dev->bd_disk->private_data;
3441 
3442 		if (dd) {
3443 			if (test_bit(MTIP_DDF_REMOVAL_BIT,
3444 							&dd->dd_flag)) {
3445 				return -ENODEV;
3446 			}
3447 			return 0;
3448 		}
3449 	}
3450 	return -ENODEV;
3451 }
3452 
3453 static void mtip_block_release(struct gendisk *disk, fmode_t mode)
3454 {
3455 }
3456 
3457 /*
3458  * Block device operation function.
3459  *
3460  * This structure contains pointers to the functions required by the block
3461  * layer.
3462  */
3463 static const struct block_device_operations mtip_block_ops = {
3464 	.open		= mtip_block_open,
3465 	.release	= mtip_block_release,
3466 	.ioctl		= mtip_block_ioctl,
3467 #ifdef CONFIG_COMPAT
3468 	.compat_ioctl	= mtip_block_compat_ioctl,
3469 #endif
3470 	.getgeo		= mtip_block_getgeo,
3471 	.owner		= THIS_MODULE
3472 };
3473 
3474 static inline bool is_se_active(struct driver_data *dd)
3475 {
3476 	if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) {
3477 		if (dd->port->ic_pause_timer) {
3478 			unsigned long to = dd->port->ic_pause_timer +
3479 							msecs_to_jiffies(1000);
3480 			if (time_after(jiffies, to)) {
3481 				clear_bit(MTIP_PF_SE_ACTIVE_BIT,
3482 							&dd->port->flags);
3483 				clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
3484 				dd->port->ic_pause_timer = 0;
3485 				wake_up_interruptible(&dd->port->svc_wait);
3486 				return false;
3487 			}
3488 		}
3489 		return true;
3490 	}
3491 	return false;
3492 }
3493 
3494 static inline bool is_stopped(struct driver_data *dd, struct request *rq)
3495 {
3496 	if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO)))
3497 		return false;
3498 
3499 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
3500 		return true;
3501 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
3502 		return true;
3503 	if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) &&
3504 	    rq_data_dir(rq))
3505 		return true;
3506 	if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))
3507 		return true;
3508 	if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
3509 		return true;
3510 
3511 	return false;
3512 }
3513 
3514 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx,
3515 				  struct request *rq)
3516 {
3517 	struct driver_data *dd = hctx->queue->queuedata;
3518 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3519 
3520 	if (rq_data_dir(rq) == READ || !dd->unal_qdepth)
3521 		return false;
3522 
3523 	/*
3524 	 * If unaligned depth must be limited on this controller, mark it
3525 	 * as unaligned if the IO isn't on a 4k boundary (start of length).
3526 	 */
3527 	if (blk_rq_sectors(rq) <= 64) {
3528 		if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7))
3529 			cmd->unaligned = 1;
3530 	}
3531 
3532 	if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0)
3533 		return true;
3534 
3535 	return false;
3536 }
3537 
3538 static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx,
3539 		struct request *rq)
3540 {
3541 	struct driver_data *dd = hctx->queue->queuedata;
3542 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3543 	struct mtip_int_cmd *icmd = cmd->icmd;
3544 	struct mtip_cmd_hdr *hdr =
3545 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
3546 	struct mtip_cmd_sg *command_sg;
3547 
3548 	if (mtip_commands_active(dd->port))
3549 		return BLK_STS_DEV_RESOURCE;
3550 
3551 	hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF);
3552 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
3553 		hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16);
3554 	/* Populate the SG list */
3555 	hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len);
3556 	if (icmd->buf_len) {
3557 		command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ;
3558 
3559 		command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF);
3560 		command_sg->dba	= cpu_to_le32(icmd->buffer & 0xFFFFFFFF);
3561 		command_sg->dba_upper =
3562 			cpu_to_le32((icmd->buffer >> 16) >> 16);
3563 
3564 		hdr->opts |= cpu_to_le32((1 << 16));
3565 	}
3566 
3567 	/* Populate the command header */
3568 	hdr->byte_count = 0;
3569 
3570 	blk_mq_start_request(rq);
3571 	mtip_issue_non_ncq_command(dd->port, rq->tag);
3572 	return 0;
3573 }
3574 
3575 static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx,
3576 			 const struct blk_mq_queue_data *bd)
3577 {
3578 	struct driver_data *dd = hctx->queue->queuedata;
3579 	struct request *rq = bd->rq;
3580 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3581 
3582 	if (blk_rq_is_passthrough(rq))
3583 		return mtip_issue_reserved_cmd(hctx, rq);
3584 
3585 	if (unlikely(mtip_check_unal_depth(hctx, rq)))
3586 		return BLK_STS_DEV_RESOURCE;
3587 
3588 	if (is_se_active(dd) || is_stopped(dd, rq))
3589 		return BLK_STS_IOERR;
3590 
3591 	blk_mq_start_request(rq);
3592 
3593 	if (req_op(rq) == REQ_OP_DISCARD)
3594 		return mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq));
3595 	mtip_hw_submit_io(dd, rq, cmd, hctx);
3596 	return BLK_STS_OK;
3597 }
3598 
3599 static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq,
3600 			  unsigned int hctx_idx)
3601 {
3602 	struct driver_data *dd = set->driver_data;
3603 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3604 
3605 	if (!cmd->command)
3606 		return;
3607 
3608 	dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command,
3609 			  cmd->command_dma);
3610 }
3611 
3612 static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
3613 			 unsigned int hctx_idx, unsigned int numa_node)
3614 {
3615 	struct driver_data *dd = set->driver_data;
3616 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3617 
3618 	cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ,
3619 			&cmd->command_dma, GFP_KERNEL);
3620 	if (!cmd->command)
3621 		return -ENOMEM;
3622 
3623 	memset(cmd->command, 0, CMD_DMA_ALLOC_SZ);
3624 
3625 	sg_init_table(cmd->sg, MTIP_MAX_SG);
3626 	return 0;
3627 }
3628 
3629 static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
3630 								bool reserved)
3631 {
3632 	struct driver_data *dd = req->q->queuedata;
3633 
3634 	if (reserved) {
3635 		struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
3636 
3637 		cmd->status = BLK_STS_TIMEOUT;
3638 		blk_mq_complete_request(req);
3639 		return BLK_EH_DONE;
3640 	}
3641 
3642 	if (test_bit(req->tag, dd->port->cmds_to_issue))
3643 		goto exit_handler;
3644 
3645 	if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags))
3646 		goto exit_handler;
3647 
3648 	wake_up_interruptible(&dd->port->svc_wait);
3649 exit_handler:
3650 	return BLK_EH_RESET_TIMER;
3651 }
3652 
3653 static const struct blk_mq_ops mtip_mq_ops = {
3654 	.queue_rq	= mtip_queue_rq,
3655 	.init_request	= mtip_init_cmd,
3656 	.exit_request	= mtip_free_cmd,
3657 	.complete	= mtip_softirq_done_fn,
3658 	.timeout        = mtip_cmd_timeout,
3659 };
3660 
3661 /*
3662  * Block layer initialization function.
3663  *
3664  * This function is called once by the PCI layer for each P320
3665  * device that is connected to the system.
3666  *
3667  * @dd Pointer to the driver data structure.
3668  *
3669  * return value
3670  *	0 on success else an error code.
3671  */
3672 static int mtip_block_initialize(struct driver_data *dd)
3673 {
3674 	int rv = 0, wait_for_rebuild = 0;
3675 	sector_t capacity;
3676 	unsigned int index = 0;
3677 	struct kobject *kobj;
3678 
3679 	if (dd->disk)
3680 		goto skip_create_disk; /* hw init done, before rebuild */
3681 
3682 	if (mtip_hw_init(dd)) {
3683 		rv = -EINVAL;
3684 		goto protocol_init_error;
3685 	}
3686 
3687 	dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node);
3688 	if (dd->disk  == NULL) {
3689 		dev_err(&dd->pdev->dev,
3690 			"Unable to allocate gendisk structure\n");
3691 		rv = -EINVAL;
3692 		goto alloc_disk_error;
3693 	}
3694 
3695 	rv = ida_alloc(&rssd_index_ida, GFP_KERNEL);
3696 	if (rv < 0)
3697 		goto ida_get_error;
3698 	index = rv;
3699 
3700 	rv = rssd_disk_name_format("rssd",
3701 				index,
3702 				dd->disk->disk_name,
3703 				DISK_NAME_LEN);
3704 	if (rv)
3705 		goto disk_index_error;
3706 
3707 	dd->disk->major		= dd->major;
3708 	dd->disk->first_minor	= index * MTIP_MAX_MINORS;
3709 	dd->disk->minors 	= MTIP_MAX_MINORS;
3710 	dd->disk->fops		= &mtip_block_ops;
3711 	dd->disk->private_data	= dd;
3712 	dd->index		= index;
3713 
3714 	mtip_hw_debugfs_init(dd);
3715 
3716 	memset(&dd->tags, 0, sizeof(dd->tags));
3717 	dd->tags.ops = &mtip_mq_ops;
3718 	dd->tags.nr_hw_queues = 1;
3719 	dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS;
3720 	dd->tags.reserved_tags = 1;
3721 	dd->tags.cmd_size = sizeof(struct mtip_cmd);
3722 	dd->tags.numa_node = dd->numa_node;
3723 	dd->tags.flags = BLK_MQ_F_SHOULD_MERGE;
3724 	dd->tags.driver_data = dd;
3725 	dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS;
3726 
3727 	rv = blk_mq_alloc_tag_set(&dd->tags);
3728 	if (rv) {
3729 		dev_err(&dd->pdev->dev,
3730 			"Unable to allocate request queue\n");
3731 		goto block_queue_alloc_tag_error;
3732 	}
3733 
3734 	/* Allocate the request queue. */
3735 	dd->queue = blk_mq_init_queue(&dd->tags);
3736 	if (IS_ERR(dd->queue)) {
3737 		dev_err(&dd->pdev->dev,
3738 			"Unable to allocate request queue\n");
3739 		rv = -ENOMEM;
3740 		goto block_queue_alloc_init_error;
3741 	}
3742 
3743 	dd->disk->queue		= dd->queue;
3744 	dd->queue->queuedata	= dd;
3745 
3746 skip_create_disk:
3747 	/* Initialize the protocol layer. */
3748 	wait_for_rebuild = mtip_hw_get_identify(dd);
3749 	if (wait_for_rebuild < 0) {
3750 		dev_err(&dd->pdev->dev,
3751 			"Protocol layer initialization failed\n");
3752 		rv = -EINVAL;
3753 		goto init_hw_cmds_error;
3754 	}
3755 
3756 	/*
3757 	 * if rebuild pending, start the service thread, and delay the block
3758 	 * queue creation and device_add_disk()
3759 	 */
3760 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3761 		goto start_service_thread;
3762 
3763 	/* Set device limits. */
3764 	blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue);
3765 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue);
3766 	blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
3767 	blk_queue_physical_block_size(dd->queue, 4096);
3768 	blk_queue_max_hw_sectors(dd->queue, 0xffff);
3769 	blk_queue_max_segment_size(dd->queue, 0x400000);
3770 	blk_queue_io_min(dd->queue, 4096);
3771 
3772 	/* Signal trim support */
3773 	if (dd->trim_supp == true) {
3774 		blk_queue_flag_set(QUEUE_FLAG_DISCARD, dd->queue);
3775 		dd->queue->limits.discard_granularity = 4096;
3776 		blk_queue_max_discard_sectors(dd->queue,
3777 			MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES);
3778 	}
3779 
3780 	/* Set the capacity of the device in 512 byte sectors. */
3781 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3782 		dev_warn(&dd->pdev->dev,
3783 			"Could not read drive capacity\n");
3784 		rv = -EIO;
3785 		goto read_capacity_error;
3786 	}
3787 	set_capacity(dd->disk, capacity);
3788 
3789 	/* Enable the block device and add it to /dev */
3790 	device_add_disk(&dd->pdev->dev, dd->disk, NULL);
3791 
3792 	dd->bdev = bdget_disk(dd->disk, 0);
3793 	/*
3794 	 * Now that the disk is active, initialize any sysfs attributes
3795 	 * managed by the protocol layer.
3796 	 */
3797 	kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3798 	if (kobj) {
3799 		mtip_hw_sysfs_init(dd, kobj);
3800 		kobject_put(kobj);
3801 	}
3802 
3803 	if (dd->mtip_svc_handler) {
3804 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
3805 		return rv; /* service thread created for handling rebuild */
3806 	}
3807 
3808 start_service_thread:
3809 	dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread,
3810 						dd, dd->numa_node,
3811 						"mtip_svc_thd_%02d", index);
3812 
3813 	if (IS_ERR(dd->mtip_svc_handler)) {
3814 		dev_err(&dd->pdev->dev, "service thread failed to start\n");
3815 		dd->mtip_svc_handler = NULL;
3816 		rv = -EFAULT;
3817 		goto kthread_run_error;
3818 	}
3819 	wake_up_process(dd->mtip_svc_handler);
3820 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3821 		rv = wait_for_rebuild;
3822 
3823 	return rv;
3824 
3825 kthread_run_error:
3826 	bdput(dd->bdev);
3827 	dd->bdev = NULL;
3828 
3829 	/* Delete our gendisk. This also removes the device from /dev */
3830 	del_gendisk(dd->disk);
3831 
3832 read_capacity_error:
3833 init_hw_cmds_error:
3834 	blk_cleanup_queue(dd->queue);
3835 block_queue_alloc_init_error:
3836 	blk_mq_free_tag_set(&dd->tags);
3837 block_queue_alloc_tag_error:
3838 	mtip_hw_debugfs_exit(dd);
3839 disk_index_error:
3840 	ida_free(&rssd_index_ida, index);
3841 
3842 ida_get_error:
3843 	put_disk(dd->disk);
3844 
3845 alloc_disk_error:
3846 	mtip_hw_exit(dd); /* De-initialize the protocol layer. */
3847 
3848 protocol_init_error:
3849 	return rv;
3850 }
3851 
3852 static bool mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv)
3853 {
3854 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3855 
3856 	cmd->status = BLK_STS_IOERR;
3857 	blk_mq_complete_request(rq);
3858 	return true;
3859 }
3860 
3861 /*
3862  * Block layer deinitialization function.
3863  *
3864  * Called by the PCI layer as each P320 device is removed.
3865  *
3866  * @dd Pointer to the driver data structure.
3867  *
3868  * return value
3869  *	0
3870  */
3871 static int mtip_block_remove(struct driver_data *dd)
3872 {
3873 	struct kobject *kobj;
3874 
3875 	mtip_hw_debugfs_exit(dd);
3876 
3877 	if (dd->mtip_svc_handler) {
3878 		set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3879 		wake_up_interruptible(&dd->port->svc_wait);
3880 		kthread_stop(dd->mtip_svc_handler);
3881 	}
3882 
3883 	/* Clean up the sysfs attributes, if created */
3884 	if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) {
3885 		kobj = kobject_get(&disk_to_dev(dd->disk)->kobj);
3886 		if (kobj) {
3887 			mtip_hw_sysfs_exit(dd, kobj);
3888 			kobject_put(kobj);
3889 		}
3890 	}
3891 
3892 	if (!dd->sr) {
3893 		/*
3894 		 * Explicitly wait here for IOs to quiesce,
3895 		 * as mtip_standby_drive usually won't wait for IOs.
3896 		 */
3897 		if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3898 			mtip_standby_drive(dd);
3899 	}
3900 	else
3901 		dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3902 						dd->disk->disk_name);
3903 
3904 	blk_freeze_queue_start(dd->queue);
3905 	blk_mq_quiesce_queue(dd->queue);
3906 	blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);
3907 	blk_mq_unquiesce_queue(dd->queue);
3908 
3909 	/*
3910 	 * Delete our gendisk structure. This also removes the device
3911 	 * from /dev
3912 	 */
3913 	if (dd->bdev) {
3914 		bdput(dd->bdev);
3915 		dd->bdev = NULL;
3916 	}
3917 	if (dd->disk) {
3918 		if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3919 			del_gendisk(dd->disk);
3920 		if (dd->disk->queue) {
3921 			blk_cleanup_queue(dd->queue);
3922 			blk_mq_free_tag_set(&dd->tags);
3923 			dd->queue = NULL;
3924 		}
3925 		put_disk(dd->disk);
3926 	}
3927 	dd->disk  = NULL;
3928 
3929 	ida_free(&rssd_index_ida, dd->index);
3930 
3931 	/* De-initialize the protocol layer. */
3932 	mtip_hw_exit(dd);
3933 
3934 	return 0;
3935 }
3936 
3937 /*
3938  * Function called by the PCI layer when just before the
3939  * machine shuts down.
3940  *
3941  * If a protocol layer shutdown function is present it will be called
3942  * by this function.
3943  *
3944  * @dd Pointer to the driver data structure.
3945  *
3946  * return value
3947  *	0
3948  */
3949 static int mtip_block_shutdown(struct driver_data *dd)
3950 {
3951 	mtip_hw_shutdown(dd);
3952 
3953 	/* Delete our gendisk structure, and cleanup the blk queue. */
3954 	if (dd->disk) {
3955 		dev_info(&dd->pdev->dev,
3956 			"Shutting down %s ...\n", dd->disk->disk_name);
3957 
3958 		if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3959 			del_gendisk(dd->disk);
3960 		if (dd->disk->queue) {
3961 			blk_cleanup_queue(dd->queue);
3962 			blk_mq_free_tag_set(&dd->tags);
3963 		}
3964 		put_disk(dd->disk);
3965 		dd->disk  = NULL;
3966 		dd->queue = NULL;
3967 	}
3968 
3969 	ida_free(&rssd_index_ida, dd->index);
3970 	return 0;
3971 }
3972 
3973 static int mtip_block_suspend(struct driver_data *dd)
3974 {
3975 	dev_info(&dd->pdev->dev,
3976 		"Suspending %s ...\n", dd->disk->disk_name);
3977 	mtip_hw_suspend(dd);
3978 	return 0;
3979 }
3980 
3981 static int mtip_block_resume(struct driver_data *dd)
3982 {
3983 	dev_info(&dd->pdev->dev, "Resuming %s ...\n",
3984 		dd->disk->disk_name);
3985 	mtip_hw_resume(dd);
3986 	return 0;
3987 }
3988 
3989 static void drop_cpu(int cpu)
3990 {
3991 	cpu_use[cpu]--;
3992 }
3993 
3994 static int get_least_used_cpu_on_node(int node)
3995 {
3996 	int cpu, least_used_cpu, least_cnt;
3997 	const struct cpumask *node_mask;
3998 
3999 	node_mask = cpumask_of_node(node);
4000 	least_used_cpu = cpumask_first(node_mask);
4001 	least_cnt = cpu_use[least_used_cpu];
4002 	cpu = least_used_cpu;
4003 
4004 	for_each_cpu(cpu, node_mask) {
4005 		if (cpu_use[cpu] < least_cnt) {
4006 			least_used_cpu = cpu;
4007 			least_cnt = cpu_use[cpu];
4008 		}
4009 	}
4010 	cpu_use[least_used_cpu]++;
4011 	return least_used_cpu;
4012 }
4013 
4014 /* Helper for selecting a node in round robin mode */
4015 static inline int mtip_get_next_rr_node(void)
4016 {
4017 	static int next_node = NUMA_NO_NODE;
4018 
4019 	if (next_node == NUMA_NO_NODE) {
4020 		next_node = first_online_node;
4021 		return next_node;
4022 	}
4023 
4024 	next_node = next_online_node(next_node);
4025 	if (next_node == MAX_NUMNODES)
4026 		next_node = first_online_node;
4027 	return next_node;
4028 }
4029 
4030 static DEFINE_HANDLER(0);
4031 static DEFINE_HANDLER(1);
4032 static DEFINE_HANDLER(2);
4033 static DEFINE_HANDLER(3);
4034 static DEFINE_HANDLER(4);
4035 static DEFINE_HANDLER(5);
4036 static DEFINE_HANDLER(6);
4037 static DEFINE_HANDLER(7);
4038 
4039 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev)
4040 {
4041 	int pos;
4042 	unsigned short pcie_dev_ctrl;
4043 
4044 	pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
4045 	if (pos) {
4046 		pci_read_config_word(pdev,
4047 			pos + PCI_EXP_DEVCTL,
4048 			&pcie_dev_ctrl);
4049 		if (pcie_dev_ctrl & (1 << 11) ||
4050 		    pcie_dev_ctrl & (1 << 4)) {
4051 			dev_info(&dd->pdev->dev,
4052 				"Disabling ERO/No-Snoop on bridge device %04x:%04x\n",
4053 					pdev->vendor, pdev->device);
4054 			pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN |
4055 						PCI_EXP_DEVCTL_RELAX_EN);
4056 			pci_write_config_word(pdev,
4057 				pos + PCI_EXP_DEVCTL,
4058 				pcie_dev_ctrl);
4059 		}
4060 	}
4061 }
4062 
4063 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev)
4064 {
4065 	/*
4066 	 * This workaround is specific to AMD/ATI chipset with a PCI upstream
4067 	 * device with device id 0x5aXX
4068 	 */
4069 	if (pdev->bus && pdev->bus->self) {
4070 		if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI &&
4071 		    ((pdev->bus->self->device & 0xff00) == 0x5a00)) {
4072 			mtip_disable_link_opts(dd, pdev->bus->self);
4073 		} else {
4074 			/* Check further up the topology */
4075 			struct pci_dev *parent_dev = pdev->bus->self;
4076 			if (parent_dev->bus &&
4077 				parent_dev->bus->parent &&
4078 				parent_dev->bus->parent->self &&
4079 				parent_dev->bus->parent->self->vendor ==
4080 					 PCI_VENDOR_ID_ATI &&
4081 				(parent_dev->bus->parent->self->device &
4082 					0xff00) == 0x5a00) {
4083 				mtip_disable_link_opts(dd,
4084 					parent_dev->bus->parent->self);
4085 			}
4086 		}
4087 	}
4088 }
4089 
4090 /*
4091  * Called for each supported PCI device detected.
4092  *
4093  * This function allocates the private data structure, enables the
4094  * PCI device and then calls the block layer initialization function.
4095  *
4096  * return value
4097  *	0 on success else an error code.
4098  */
4099 static int mtip_pci_probe(struct pci_dev *pdev,
4100 			const struct pci_device_id *ent)
4101 {
4102 	int rv = 0;
4103 	struct driver_data *dd = NULL;
4104 	char cpu_list[256];
4105 	const struct cpumask *node_mask;
4106 	int cpu, i = 0, j = 0;
4107 	int my_node = NUMA_NO_NODE;
4108 	unsigned long flags;
4109 
4110 	/* Allocate memory for this devices private data. */
4111 	my_node = pcibus_to_node(pdev->bus);
4112 	if (my_node != NUMA_NO_NODE) {
4113 		if (!node_online(my_node))
4114 			my_node = mtip_get_next_rr_node();
4115 	} else {
4116 		dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n");
4117 		my_node = mtip_get_next_rr_node();
4118 	}
4119 	dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n",
4120 		my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev),
4121 		cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id());
4122 
4123 	dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node);
4124 	if (dd == NULL) {
4125 		dev_err(&pdev->dev,
4126 			"Unable to allocate memory for driver data\n");
4127 		return -ENOMEM;
4128 	}
4129 
4130 	/* Attach the private data to this PCI device.  */
4131 	pci_set_drvdata(pdev, dd);
4132 
4133 	rv = pcim_enable_device(pdev);
4134 	if (rv < 0) {
4135 		dev_err(&pdev->dev, "Unable to enable device\n");
4136 		goto iomap_err;
4137 	}
4138 
4139 	/* Map BAR5 to memory. */
4140 	rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
4141 	if (rv < 0) {
4142 		dev_err(&pdev->dev, "Unable to map regions\n");
4143 		goto iomap_err;
4144 	}
4145 
4146 	rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4147 	if (rv) {
4148 		dev_warn(&pdev->dev, "64-bit DMA enable failed\n");
4149 		goto setmask_err;
4150 	}
4151 
4152 	/* Copy the info we may need later into the private data structure. */
4153 	dd->major	= mtip_major;
4154 	dd->instance	= instance;
4155 	dd->pdev	= pdev;
4156 	dd->numa_node	= my_node;
4157 
4158 	INIT_LIST_HEAD(&dd->online_list);
4159 	INIT_LIST_HEAD(&dd->remove_list);
4160 
4161 	memset(dd->workq_name, 0, 32);
4162 	snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
4163 
4164 	dd->isr_workq = create_workqueue(dd->workq_name);
4165 	if (!dd->isr_workq) {
4166 		dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
4167 		rv = -ENOMEM;
4168 		goto setmask_err;
4169 	}
4170 
4171 	memset(cpu_list, 0, sizeof(cpu_list));
4172 
4173 	node_mask = cpumask_of_node(dd->numa_node);
4174 	if (!cpumask_empty(node_mask)) {
4175 		for_each_cpu(cpu, node_mask)
4176 		{
4177 			snprintf(&cpu_list[j], 256 - j, "%d ", cpu);
4178 			j = strlen(cpu_list);
4179 		}
4180 
4181 		dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n",
4182 			dd->numa_node,
4183 			topology_physical_package_id(cpumask_first(node_mask)),
4184 			nr_cpus_node(dd->numa_node),
4185 			cpu_list);
4186 	} else
4187 		dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n");
4188 
4189 	dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node);
4190 	dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n",
4191 		cpu_to_node(dd->isr_binding), dd->isr_binding);
4192 
4193 	/* first worker context always runs in ISR */
4194 	dd->work[0].cpu_binding = dd->isr_binding;
4195 	dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
4196 	dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
4197 	dd->work[3].cpu_binding = dd->work[0].cpu_binding;
4198 	dd->work[4].cpu_binding = dd->work[1].cpu_binding;
4199 	dd->work[5].cpu_binding = dd->work[2].cpu_binding;
4200 	dd->work[6].cpu_binding = dd->work[2].cpu_binding;
4201 	dd->work[7].cpu_binding = dd->work[1].cpu_binding;
4202 
4203 	/* Log the bindings */
4204 	for_each_present_cpu(cpu) {
4205 		memset(cpu_list, 0, sizeof(cpu_list));
4206 		for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) {
4207 			if (dd->work[i].cpu_binding == cpu) {
4208 				snprintf(&cpu_list[j], 256 - j, "%d ", i);
4209 				j = strlen(cpu_list);
4210 			}
4211 		}
4212 		if (j)
4213 			dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list);
4214 	}
4215 
4216 	INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0);
4217 	INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1);
4218 	INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2);
4219 	INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3);
4220 	INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4);
4221 	INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5);
4222 	INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6);
4223 	INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7);
4224 
4225 	pci_set_master(pdev);
4226 	rv = pci_enable_msi(pdev);
4227 	if (rv) {
4228 		dev_warn(&pdev->dev,
4229 			"Unable to enable MSI interrupt.\n");
4230 		goto msi_initialize_err;
4231 	}
4232 
4233 	mtip_fix_ero_nosnoop(dd, pdev);
4234 
4235 	/* Initialize the block layer. */
4236 	rv = mtip_block_initialize(dd);
4237 	if (rv < 0) {
4238 		dev_err(&pdev->dev,
4239 			"Unable to initialize block layer\n");
4240 		goto block_initialize_err;
4241 	}
4242 
4243 	/*
4244 	 * Increment the instance count so that each device has a unique
4245 	 * instance number.
4246 	 */
4247 	instance++;
4248 	if (rv != MTIP_FTL_REBUILD_MAGIC)
4249 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
4250 	else
4251 		rv = 0; /* device in rebuild state, return 0 from probe */
4252 
4253 	/* Add to online list even if in ftl rebuild */
4254 	spin_lock_irqsave(&dev_lock, flags);
4255 	list_add(&dd->online_list, &online_list);
4256 	spin_unlock_irqrestore(&dev_lock, flags);
4257 
4258 	goto done;
4259 
4260 block_initialize_err:
4261 	pci_disable_msi(pdev);
4262 
4263 msi_initialize_err:
4264 	if (dd->isr_workq) {
4265 		flush_workqueue(dd->isr_workq);
4266 		destroy_workqueue(dd->isr_workq);
4267 		drop_cpu(dd->work[0].cpu_binding);
4268 		drop_cpu(dd->work[1].cpu_binding);
4269 		drop_cpu(dd->work[2].cpu_binding);
4270 	}
4271 setmask_err:
4272 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
4273 
4274 iomap_err:
4275 	kfree(dd);
4276 	pci_set_drvdata(pdev, NULL);
4277 	return rv;
4278 done:
4279 	return rv;
4280 }
4281 
4282 /*
4283  * Called for each probed device when the device is removed or the
4284  * driver is unloaded.
4285  *
4286  * return value
4287  *	None
4288  */
4289 static void mtip_pci_remove(struct pci_dev *pdev)
4290 {
4291 	struct driver_data *dd = pci_get_drvdata(pdev);
4292 	unsigned long flags, to;
4293 
4294 	set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag);
4295 
4296 	spin_lock_irqsave(&dev_lock, flags);
4297 	list_del_init(&dd->online_list);
4298 	list_add(&dd->remove_list, &removing_list);
4299 	spin_unlock_irqrestore(&dev_lock, flags);
4300 
4301 	mtip_check_surprise_removal(pdev);
4302 	synchronize_irq(dd->pdev->irq);
4303 
4304 	/* Spin until workers are done */
4305 	to = jiffies + msecs_to_jiffies(4000);
4306 	do {
4307 		msleep(20);
4308 	} while (atomic_read(&dd->irq_workers_active) != 0 &&
4309 		time_before(jiffies, to));
4310 
4311 	if (!dd->sr)
4312 		fsync_bdev(dd->bdev);
4313 
4314 	if (atomic_read(&dd->irq_workers_active) != 0) {
4315 		dev_warn(&dd->pdev->dev,
4316 			"Completion workers still active!\n");
4317 	}
4318 
4319 	blk_set_queue_dying(dd->queue);
4320 	set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
4321 
4322 	/* Clean up the block layer. */
4323 	mtip_block_remove(dd);
4324 
4325 	if (dd->isr_workq) {
4326 		flush_workqueue(dd->isr_workq);
4327 		destroy_workqueue(dd->isr_workq);
4328 		drop_cpu(dd->work[0].cpu_binding);
4329 		drop_cpu(dd->work[1].cpu_binding);
4330 		drop_cpu(dd->work[2].cpu_binding);
4331 	}
4332 
4333 	pci_disable_msi(pdev);
4334 
4335 	spin_lock_irqsave(&dev_lock, flags);
4336 	list_del_init(&dd->remove_list);
4337 	spin_unlock_irqrestore(&dev_lock, flags);
4338 
4339 	kfree(dd);
4340 
4341 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
4342 	pci_set_drvdata(pdev, NULL);
4343 }
4344 
4345 /*
4346  * Called for each probed device when the device is suspended.
4347  *
4348  * return value
4349  *	0  Success
4350  *	<0 Error
4351  */
4352 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
4353 {
4354 	int rv = 0;
4355 	struct driver_data *dd = pci_get_drvdata(pdev);
4356 
4357 	if (!dd) {
4358 		dev_err(&pdev->dev,
4359 			"Driver private datastructure is NULL\n");
4360 		return -EFAULT;
4361 	}
4362 
4363 	set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
4364 
4365 	/* Disable ports & interrupts then send standby immediate */
4366 	rv = mtip_block_suspend(dd);
4367 	if (rv < 0) {
4368 		dev_err(&pdev->dev,
4369 			"Failed to suspend controller\n");
4370 		return rv;
4371 	}
4372 
4373 	/*
4374 	 * Save the pci config space to pdev structure &
4375 	 * disable the device
4376 	 */
4377 	pci_save_state(pdev);
4378 	pci_disable_device(pdev);
4379 
4380 	/* Move to Low power state*/
4381 	pci_set_power_state(pdev, PCI_D3hot);
4382 
4383 	return rv;
4384 }
4385 
4386 /*
4387  * Called for each probed device when the device is resumed.
4388  *
4389  * return value
4390  *      0  Success
4391  *      <0 Error
4392  */
4393 static int mtip_pci_resume(struct pci_dev *pdev)
4394 {
4395 	int rv = 0;
4396 	struct driver_data *dd;
4397 
4398 	dd = pci_get_drvdata(pdev);
4399 	if (!dd) {
4400 		dev_err(&pdev->dev,
4401 			"Driver private datastructure is NULL\n");
4402 		return -EFAULT;
4403 	}
4404 
4405 	/* Move the device to active State */
4406 	pci_set_power_state(pdev, PCI_D0);
4407 
4408 	/* Restore PCI configuration space */
4409 	pci_restore_state(pdev);
4410 
4411 	/* Enable the PCI device*/
4412 	rv = pcim_enable_device(pdev);
4413 	if (rv < 0) {
4414 		dev_err(&pdev->dev,
4415 			"Failed to enable card during resume\n");
4416 		goto err;
4417 	}
4418 	pci_set_master(pdev);
4419 
4420 	/*
4421 	 * Calls hbaReset, initPort, & startPort function
4422 	 * then enables interrupts
4423 	 */
4424 	rv = mtip_block_resume(dd);
4425 	if (rv < 0)
4426 		dev_err(&pdev->dev, "Unable to resume\n");
4427 
4428 err:
4429 	clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
4430 
4431 	return rv;
4432 }
4433 
4434 /*
4435  * Shutdown routine
4436  *
4437  * return value
4438  *      None
4439  */
4440 static void mtip_pci_shutdown(struct pci_dev *pdev)
4441 {
4442 	struct driver_data *dd = pci_get_drvdata(pdev);
4443 	if (dd)
4444 		mtip_block_shutdown(dd);
4445 }
4446 
4447 /* Table of device ids supported by this driver. */
4448 static const struct pci_device_id mtip_pci_tbl[] = {
4449 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) },
4450 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) },
4451 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) },
4452 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) },
4453 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) },
4454 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) },
4455 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) },
4456 	{ 0 }
4457 };
4458 
4459 /* Structure that describes the PCI driver functions. */
4460 static struct pci_driver mtip_pci_driver = {
4461 	.name			= MTIP_DRV_NAME,
4462 	.id_table		= mtip_pci_tbl,
4463 	.probe			= mtip_pci_probe,
4464 	.remove			= mtip_pci_remove,
4465 	.suspend		= mtip_pci_suspend,
4466 	.resume			= mtip_pci_resume,
4467 	.shutdown		= mtip_pci_shutdown,
4468 };
4469 
4470 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
4471 
4472 /*
4473  * Module initialization function.
4474  *
4475  * Called once when the module is loaded. This function allocates a major
4476  * block device number to the Cyclone devices and registers the PCI layer
4477  * of the driver.
4478  *
4479  * Return value
4480  *      0 on success else error code.
4481  */
4482 static int __init mtip_init(void)
4483 {
4484 	int error;
4485 
4486 	pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
4487 
4488 	spin_lock_init(&dev_lock);
4489 
4490 	INIT_LIST_HEAD(&online_list);
4491 	INIT_LIST_HEAD(&removing_list);
4492 
4493 	/* Allocate a major block device number to use with this driver. */
4494 	error = register_blkdev(0, MTIP_DRV_NAME);
4495 	if (error <= 0) {
4496 		pr_err("Unable to register block device (%d)\n",
4497 		error);
4498 		return -EBUSY;
4499 	}
4500 	mtip_major = error;
4501 
4502 	dfs_parent = debugfs_create_dir("rssd", NULL);
4503 	if (IS_ERR_OR_NULL(dfs_parent)) {
4504 		pr_warn("Error creating debugfs parent\n");
4505 		dfs_parent = NULL;
4506 	}
4507 	if (dfs_parent) {
4508 		dfs_device_status = debugfs_create_file("device_status",
4509 					0444, dfs_parent, NULL,
4510 					&mtip_device_status_fops);
4511 		if (IS_ERR_OR_NULL(dfs_device_status)) {
4512 			pr_err("Error creating device_status node\n");
4513 			dfs_device_status = NULL;
4514 		}
4515 	}
4516 
4517 	/* Register our PCI operations. */
4518 	error = pci_register_driver(&mtip_pci_driver);
4519 	if (error) {
4520 		debugfs_remove(dfs_parent);
4521 		unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4522 	}
4523 
4524 	return error;
4525 }
4526 
4527 /*
4528  * Module de-initialization function.
4529  *
4530  * Called once when the module is unloaded. This function deallocates
4531  * the major block device number allocated by mtip_init() and
4532  * unregisters the PCI layer of the driver.
4533  *
4534  * Return value
4535  *      none
4536  */
4537 static void __exit mtip_exit(void)
4538 {
4539 	/* Release the allocated major block device number. */
4540 	unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4541 
4542 	/* Unregister the PCI driver. */
4543 	pci_unregister_driver(&mtip_pci_driver);
4544 
4545 	debugfs_remove_recursive(dfs_parent);
4546 }
4547 
4548 MODULE_AUTHOR("Micron Technology, Inc");
4549 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
4550 MODULE_LICENSE("GPL");
4551 MODULE_VERSION(MTIP_DRV_VERSION);
4552 
4553 module_init(mtip_init);
4554 module_exit(mtip_exit);
4555