Lines Matching +full:fis +full:- +full:index +full:- +full:block
1 // SPDX-License-Identifier: GPL-2.0-or-later
23 #include <linux/blk-mq.h>
25 #include <linux/dma-mapping.h>
37 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
98 * Global variable used to hold the major block device number
137 if (dd->sr) in mtip_check_surprise_removal()
141 pci_read_config_word(dd->pdev, 0x00, &vendor_id); in mtip_check_surprise_removal()
143 dd->sr = true; in mtip_check_surprise_removal()
144 if (dd->disk) in mtip_check_surprise_removal()
145 blk_mark_disk_dead(dd->disk); in mtip_check_surprise_removal()
155 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(dd->tags.tags[0], tag)); in mtip_cmd_from_tag()
165 * -1 The HBA Reset bit did not clear.
172 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
175 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
184 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
185 return -1; in mtip_hba_reset()
187 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
190 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
191 return -1; in mtip_hba_reset()
213 spin_lock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
215 port->s_active[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
217 port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_ncq_command()
218 spin_unlock(&port->cmd_issue_lock[group]); in mtip_issue_ncq_command()
222 * Enable/disable the reception of FIS
234 /* enable FIS reception */ in mtip_enable_fis()
235 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_fis()
237 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
239 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); in mtip_enable_fis()
242 readl(port->mmio + PORT_CMD); in mtip_enable_fis()
260 /* enable FIS reception */ in mtip_enable_engine()
261 tmp = readl(port->mmio + PORT_CMD); in mtip_enable_engine()
263 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
265 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); in mtip_enable_engine()
267 readl(port->mmio + PORT_CMD); in mtip_enable_engine()
272 * Enables the port DMA engine and FIS reception.
279 /* Enable FIS reception */ in mtip_start_port()
288 * and FIS reception.
298 writel(0, port->mmio + PORT_IRQ_MASK); in mtip_deinit_port()
303 /* Disable FIS reception */ in mtip_deinit_port()
311 * then initializes it by setting the command header and RX FIS addresses,
313 * re-enabling the default set of port interrupts.
325 /* Program the command list base and FIS base addresses */ in mtip_init_port()
326 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
327 writel((port->command_list_dma >> 16) >> 16, in mtip_init_port()
328 port->mmio + PORT_LST_ADDR_HI); in mtip_init_port()
329 writel((port->rxfis_dma >> 16) >> 16, in mtip_init_port()
330 port->mmio + PORT_FIS_ADDR_HI); in mtip_init_port()
331 set_bit(MTIP_PF_HOST_CAP_64, &port->flags); in mtip_init_port()
334 writel(port->command_list_dma & 0xFFFFFFFF, in mtip_init_port()
335 port->mmio + PORT_LST_ADDR); in mtip_init_port()
336 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); in mtip_init_port()
339 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); in mtip_init_port()
342 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
343 writel(0xFFFFFFFF, port->completed[i]); in mtip_init_port()
346 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); in mtip_init_port()
349 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
350 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
353 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); in mtip_init_port()
373 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) in mtip_restart_port()
377 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
384 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { in mtip_restart_port()
385 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
388 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
389 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
396 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
399 writel(readl(port->mmio + PORT_SCR_CTL) | in mtip_restart_port()
400 1, port->mmio + PORT_SCR_CTL); in mtip_restart_port()
401 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
408 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
412 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, in mtip_restart_port()
413 port->mmio + PORT_SCR_CTL); in mtip_restart_port()
414 readl(port->mmio + PORT_SCR_CTL); in mtip_restart_port()
418 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
422 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
425 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) in mtip_restart_port()
426 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
442 rv = -EFAULT; in mtip_device_reset()
445 mtip_init_port(dd->port); in mtip_device_reset()
446 mtip_start_port(dd->port); in mtip_device_reset()
449 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
450 dd->mmio + HOST_CTL); in mtip_device_reset()
466 for (group = SLOTBITS_IN_LONGS; group > 0; group--) in print_tags()
468 tagbits[group-1]); in print_tags()
469 dev_warn(&dd->pdev->dev, in print_tags()
482 cmd->status = status; in mtip_complete_command()
483 if (likely(!blk_should_fake_timeout(req->q))) in mtip_complete_command()
501 struct host_to_dev_fis *fis; in mtip_handle_tfe() local
508 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
510 port = dd->port; in mtip_handle_tfe()
512 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { in mtip_handle_tfe()
523 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
524 completed = readl(port->completed[group]); in mtip_handle_tfe()
526 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); in mtip_handle_tfe()
529 writel(completed, port->completed[group]); in mtip_handle_tfe()
555 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
556 dd->port->log_buf, in mtip_handle_tfe()
557 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
559 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
561 /* non-critical error, don't fail the load */ in mtip_handle_tfe()
563 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
565 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
567 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
572 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
574 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
579 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_handle_tfe()
580 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
591 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
597 fis = (struct host_to_dev_fis *)cmd->command; in mtip_handle_tfe()
599 /* Should re-issue? */ in mtip_handle_tfe()
601 fis->command == ATA_CMD_SET_FEATURES) in mtip_handle_tfe()
606 fis->command == ATA_CMD_FPDMA_WRITE)) { in mtip_handle_tfe()
607 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
609 fis->command == ATA_CMD_FPDMA_WRITE ? in mtip_handle_tfe()
623 if (reissue && (cmd->retries-- > 0)) { in mtip_handle_tfe()
627 /* Re-issue the command. */ in mtip_handle_tfe()
634 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
649 struct driver_data *dd = port->dd; in mtip_workq_sdbfx()
658 writel(completed, port->completed[group]); in mtip_workq_sdbfx()
675 /* If last, re-enable interrupts */ in mtip_workq_sdbfx()
676 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
677 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
685 struct mtip_port *port = dd->port; in mtip_process_legacy()
688 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) { in mtip_process_legacy()
690 int status = readl(port->cmd_issue[group]); in mtip_process_legacy()
703 dev_warn(&dd->pdev->dev, in mtip_process_errors()
705 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
709 dev_warn(&dd->pdev->dev, in mtip_process_errors()
711 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
715 dev_warn(&dd->pdev->dev, in mtip_process_errors()
722 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
723 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
730 struct mtip_port *port = dd->port; in mtip_handle_irq()
736 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
741 port_stat = readl(port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
746 writel(port_stat, port->mmio + PORT_IRQ_STAT); in mtip_handle_irq()
751 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
756 twork = &dd->work[i]; in mtip_handle_irq()
757 twork->completed = readl(port->completed[i]); in mtip_handle_irq()
758 if (twork->completed) in mtip_handle_irq()
762 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
765 twork = &dd->work[i]; in mtip_handle_irq()
766 if (twork->completed) in mtip_handle_irq()
768 twork->cpu_binding, in mtip_handle_irq()
769 dd->isr_workq, in mtip_handle_irq()
770 &twork->work); in mtip_handle_irq()
773 if (likely(dd->work[0].completed)) in mtip_handle_irq()
775 dd->work[0].completed); in mtip_handle_irq()
792 &dd->dd_flag)) in mtip_handle_irq()
804 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
828 writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]); in mtip_issue_non_ncq_command()
832 struct host_to_dev_fis *fis) in mtip_pause_ncq() argument
836 task_file_data = readl(port->mmio+PORT_TFDATA); in mtip_pause_ncq()
840 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { in mtip_pause_ncq()
841 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
843 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && in mtip_pause_ncq()
844 (fis->features == 0x03)) { in mtip_pause_ncq()
845 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
846 port->ic_pause_timer = jiffies; in mtip_pause_ncq()
848 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || in mtip_pause_ncq()
849 ((fis->command == 0xFC) && in mtip_pause_ncq()
850 (fis->features == 0x27 || fis->features == 0x72 || in mtip_pause_ncq()
851 fis->features == 0x62 || fis->features == 0x26))) { in mtip_pause_ncq()
852 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
853 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
856 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_pause_ncq()
872 active = readl(port->s_active[0]) & 0xFFFFFFFE; in mtip_commands_active()
873 for (n = 1; n < port->dd->slot_groups; n++) in mtip_commands_active()
874 active |= readl(port->s_active[n]); in mtip_commands_active()
887 * -EBUSY Commands still active
894 blk_mq_quiesce_queue(port->dd->queue); in mtip_quiesce_io()
898 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && in mtip_quiesce_io()
899 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_quiesce_io()
906 if (mtip_check_surprise_removal(port->dd)) in mtip_quiesce_io()
914 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
915 return active ? -EBUSY : 0; in mtip_quiesce_io()
917 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
918 return -EFAULT; in mtip_quiesce_io()
932 * @fis Pointer to the FIS that describes the command.
933 * @fis_len Length in WORDS of the FIS.
936 * @opts Command header options, excluding the FIS length
942 * -EFAULT The buffer address is not correctly aligned.
943 * -EBUSY Internal command or other IO in progress.
944 * -EAGAIN Time out waiting for command to complete.
947 struct host_to_dev_fis *fis, in mtip_exec_internal_command() argument
955 struct driver_data *dd = port->dd; in mtip_exec_internal_command()
967 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
968 return -EFAULT; in mtip_exec_internal_command()
972 return -EFAULT; in mtip_exec_internal_command()
974 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); in mtip_exec_internal_command()
977 return -EFAULT; in mtip_exec_internal_command()
980 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
982 if (fis->command == ATA_CMD_SEC_ERASE_PREP) in mtip_exec_internal_command()
983 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
985 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
987 if (fis->command != ATA_CMD_STANDBYNOW1) { in mtip_exec_internal_command()
990 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); in mtip_exec_internal_command()
992 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
993 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
994 return -EBUSY; in mtip_exec_internal_command()
1000 int_cmd->icmd = &icmd; in mtip_exec_internal_command()
1001 memcpy(int_cmd->command, fis, fis_len*4); in mtip_exec_internal_command()
1003 rq->timeout = timeout; in mtip_exec_internal_command()
1008 if (int_cmd->status) { in mtip_exec_internal_command()
1009 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", in mtip_exec_internal_command()
1010 fis->command, int_cmd->status); in mtip_exec_internal_command()
1011 rv = -EIO; in mtip_exec_internal_command()
1015 &dd->dd_flag)) { in mtip_exec_internal_command()
1016 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1018 fis->command); in mtip_exec_internal_command()
1019 rv = -ENXIO; in mtip_exec_internal_command()
1023 rv = -EAGAIN; in mtip_exec_internal_command()
1027 if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)]) in mtip_exec_internal_command()
1029 rv = -ENXIO; in mtip_exec_internal_command()
1030 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1032 rv = -EAGAIN; in mtip_exec_internal_command()
1038 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); in mtip_exec_internal_command()
1039 if (rv >= 0 && mtip_pause_ncq(port, fis)) { in mtip_exec_internal_command()
1043 wake_up_interruptible(&port->svc_wait); in mtip_exec_internal_command()
1049 * Byte-swap ATA ID strings.
1051 * ATA identify data contains strings in byte-swapped 16-bit words.
1053 * This function swaps bytes in-place.
1069 struct host_to_dev_fis *fis, in mtip_set_timeout() argument
1072 switch (fis->command) { in mtip_set_timeout()
1079 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1081 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1104 * into the identify data buffer (@e port->identify) in the
1107 * port->identify_valid is non-zero.
1115 * -EFAULT An error occurred while coping data to the user buffer.
1116 * -1 Command failed.
1121 struct host_to_dev_fis fis; in mtip_get_identify() local
1123 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1124 return -EFAULT; in mtip_get_identify()
1126 /* Build the FIS. */ in mtip_get_identify()
1127 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_get_identify()
1128 fis.type = 0x27; in mtip_get_identify()
1129 fis.opts = 1 << 7; in mtip_get_identify()
1130 fis.command = ATA_CMD_ID_ATA; in mtip_get_identify()
1133 port->identify_valid = 0; in mtip_get_identify()
1136 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); in mtip_get_identify()
1140 &fis, in mtip_get_identify()
1142 port->identify_dma, in mtip_get_identify()
1147 rv = -1; in mtip_get_identify()
1152 * Perform any necessary byte-swapping. Yes, the kernel does in fact in mtip_get_identify()
1153 * perform field-sensitive swapping on the string fields. in mtip_get_identify()
1157 ata_swap_string(port->identify + 27, 40); /* model string*/ in mtip_get_identify()
1158 ata_swap_string(port->identify + 23, 8); /* firmware string*/ in mtip_get_identify()
1159 ata_swap_string(port->identify + 10, 20); /* serial# string*/ in mtip_get_identify()
1164 port->identify[i] = le16_to_cpu(port->identify[i]); in mtip_get_identify()
1169 if (port->identify[128] & 0x4) in mtip_get_identify()
1170 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1172 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1175 port->identify_valid = 1; in mtip_get_identify()
1180 port->identify, in mtip_get_identify()
1182 rv = -EFAULT; in mtip_get_identify()
1198 * -1 An error occurred while executing the command.
1203 struct host_to_dev_fis fis; in mtip_standby_immediate() local
1207 /* Build the FIS. */ in mtip_standby_immediate()
1208 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_standby_immediate()
1209 fis.type = 0x27; in mtip_standby_immediate()
1210 fis.opts = 1 << 7; in mtip_standby_immediate()
1211 fis.command = ATA_CMD_STANDBYNOW1; in mtip_standby_immediate()
1213 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1217 &fis, in mtip_standby_immediate()
1224 jiffies_to_msecs(jiffies - start)); in mtip_standby_immediate()
1226 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1247 struct host_to_dev_fis fis; in mtip_read_log_page() local
1249 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_read_log_page()
1250 fis.type = 0x27; in mtip_read_log_page()
1251 fis.opts = 1 << 7; in mtip_read_log_page()
1252 fis.command = ATA_CMD_READ_LOG_EXT; in mtip_read_log_page()
1253 fis.sect_count = sectors & 0xFF; in mtip_read_log_page()
1254 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; in mtip_read_log_page()
1255 fis.lba_low = page; in mtip_read_log_page()
1256 fis.lba_mid = 0; in mtip_read_log_page()
1257 fis.device = ATA_DEVICE_OBS; in mtip_read_log_page()
1262 &fis, in mtip_read_log_page()
1283 struct host_to_dev_fis fis; in mtip_get_smart_data() local
1285 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in mtip_get_smart_data()
1286 fis.type = 0x27; in mtip_get_smart_data()
1287 fis.opts = 1 << 7; in mtip_get_smart_data()
1288 fis.command = ATA_CMD_SMART; in mtip_get_smart_data()
1289 fis.features = 0xD0; in mtip_get_smart_data()
1290 fis.sect_count = 1; in mtip_get_smart_data()
1291 fis.lba_mid = 0x4F; in mtip_get_smart_data()
1292 fis.lba_hi = 0xC2; in mtip_get_smart_data()
1293 fis.device = ATA_DEVICE_OBS; in mtip_get_smart_data()
1296 &fis, in mtip_get_smart_data()
1312 * -EINVAL NULL buffer passed or unsupported attribute @id.
1313 * -EPERM Identify data not valid, SMART not supported or not enabled
1322 return -EINVAL; in mtip_get_smart_attr()
1324 if (!port->identify_valid) { in mtip_get_smart_attr()
1325 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1326 return -EPERM; in mtip_get_smart_attr()
1328 if (!(port->identify[82] & 0x1)) { in mtip_get_smart_attr()
1329 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1330 return -EPERM; in mtip_get_smart_attr()
1332 if (!(port->identify[85] & 0x1)) { in mtip_get_smart_attr()
1333 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1334 return -EPERM; in mtip_get_smart_attr()
1337 memset(port->smart_buf, 0, ATA_SECT_SIZE); in mtip_get_smart_attr()
1338 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); in mtip_get_smart_attr()
1340 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1344 pattr = (struct smart_attr *)(port->smart_buf + 2); in mtip_get_smart_attr()
1346 if (pattr->attr_id == id) { in mtip_get_smart_attr()
1352 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1354 rv = -EINVAL; in mtip_get_smart_attr()
1372 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1374 raw0 = port->identify[100]; in mtip_hw_get_capacity()
1375 raw1 = port->identify[101]; in mtip_hw_get_capacity()
1376 raw2 = port->identify[102]; in mtip_hw_get_capacity()
1377 raw3 = port->identify[103]; in mtip_hw_get_capacity()
1380 return (bool) !!port->identify_valid; in mtip_hw_get_capacity()
1397 if (!port->identify_valid) in mtip_dump_identify()
1400 strscpy(cbuf, (char *)(port->identify + 10), 21); in mtip_dump_identify()
1401 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1404 strscpy(cbuf, (char *)(port->identify + 23), 9); in mtip_dump_identify()
1405 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1408 strscpy(cbuf, (char *)(port->identify + 27), 41); in mtip_dump_identify()
1409 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1411 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1412 port->identify[128], in mtip_dump_identify()
1413 port->identify[128] & 0x4 ? "(LOCKED)" : ""); in mtip_dump_identify()
1415 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1416 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1421 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1433 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1455 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; in fill_command_sg()
1457 for_each_sg(command->sg, sg, nents, n) { in fill_command_sg()
1460 dev_err(&dd->pdev->dev, in fill_command_sg()
1462 command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF); in fill_command_sg()
1463 command_sg->dba = cpu_to_le32(sg_dma_address(sg)); in fill_command_sg()
1464 command_sg->dba_upper = in fill_command_sg()
1474 * return value -1 An error occurred while executing the command.
1478 struct host_to_dev_fis fis; in exec_drive_task() local
1479 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_task()
1482 /* Build the FIS. */ in exec_drive_task()
1483 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_task()
1484 fis.type = 0x27; in exec_drive_task()
1485 fis.opts = 1 << 7; in exec_drive_task()
1486 fis.command = command[0]; in exec_drive_task()
1487 fis.features = command[1]; in exec_drive_task()
1488 fis.sect_count = command[2]; in exec_drive_task()
1489 fis.sector = command[3]; in exec_drive_task()
1490 fis.cyl_low = command[4]; in exec_drive_task()
1491 fis.cyl_hi = command[5]; in exec_drive_task()
1492 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ in exec_drive_task()
1494 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1508 &fis, in exec_drive_task()
1514 return -1; in exec_drive_task()
1517 command[0] = reply->command; /* Status*/ in exec_drive_task()
1518 command[1] = reply->features; /* Error*/ in exec_drive_task()
1519 command[4] = reply->cyl_low; in exec_drive_task()
1520 command[5] = reply->cyl_hi; in exec_drive_task()
1541 * return value -EFAULT An error occurred while copying the completion
1543 * return value -1 An error occurred while executing the command.
1548 struct host_to_dev_fis fis; in exec_drive_command() local
1557 return -EFAULT; in exec_drive_command()
1559 buf = dma_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1564 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1567 return -ENOMEM; in exec_drive_command()
1571 /* Build the FIS. */ in exec_drive_command()
1572 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_command()
1573 fis.type = 0x27; in exec_drive_command()
1574 fis.opts = 1 << 7; in exec_drive_command()
1575 fis.command = command[0]; in exec_drive_command()
1576 fis.features = command[2]; in exec_drive_command()
1577 fis.sect_count = command[3]; in exec_drive_command()
1578 if (fis.command == ATA_CMD_SMART) { in exec_drive_command()
1579 fis.sector = command[1]; in exec_drive_command()
1580 fis.cyl_low = 0x4F; in exec_drive_command()
1581 fis.cyl_hi = 0xC2; in exec_drive_command()
1584 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1587 reply = (port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_command()
1589 reply = (port->rxfis + RX_FIS_D2H_REG); in exec_drive_command()
1602 &fis, in exec_drive_command()
1609 rv = -EFAULT; in exec_drive_command()
1614 command[0] = reply->command; /* Status*/ in exec_drive_command()
1615 command[1] = reply->features; /* Error*/ in exec_drive_command()
1616 command[2] = reply->sect_count; in exec_drive_command()
1630 rv = -EFAULT; in exec_drive_command()
1636 dma_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
1697 struct host_to_dev_fis fis; in exec_drive_taskfile() local
1712 int intotal = outtotal + req_task->out_size; in exec_drive_taskfile()
1715 taskout = req_task->out_size; in exec_drive_taskfile()
1716 taskin = req_task->in_size; in exec_drive_taskfile()
1719 return -EINVAL; in exec_drive_taskfile()
1726 outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf, in exec_drive_taskfile()
1728 if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) { in exec_drive_taskfile()
1729 err = -ENOMEM; in exec_drive_taskfile()
1742 inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf, in exec_drive_taskfile()
1744 if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) { in exec_drive_taskfile()
1745 err = -ENOMEM; in exec_drive_taskfile()
1751 /* only supports PIO and non-data commands from this ioctl. */ in exec_drive_taskfile()
1752 switch (req_task->data_phase) { in exec_drive_taskfile()
1755 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1758 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1761 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
1764 err = -EINVAL; in exec_drive_taskfile()
1768 /* Build the FIS. */ in exec_drive_taskfile()
1769 memset(&fis, 0, sizeof(struct host_to_dev_fis)); in exec_drive_taskfile()
1771 fis.type = 0x27; in exec_drive_taskfile()
1772 fis.opts = 1 << 7; in exec_drive_taskfile()
1773 fis.command = req_task->io_ports[7]; in exec_drive_taskfile()
1774 fis.features = req_task->io_ports[1]; in exec_drive_taskfile()
1775 fis.sect_count = req_task->io_ports[2]; in exec_drive_taskfile()
1776 fis.lba_low = req_task->io_ports[3]; in exec_drive_taskfile()
1777 fis.lba_mid = req_task->io_ports[4]; in exec_drive_taskfile()
1778 fis.lba_hi = req_task->io_ports[5]; in exec_drive_taskfile()
1780 fis.device = req_task->io_ports[6] & ~0x10; in exec_drive_taskfile()
1782 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { in exec_drive_taskfile()
1783 req_task->in_flags.all = in exec_drive_taskfile()
1786 fis.lba_low_ex = req_task->hob_ports[3]; in exec_drive_taskfile()
1787 fis.lba_mid_ex = req_task->hob_ports[4]; in exec_drive_taskfile()
1788 fis.lba_hi_ex = req_task->hob_ports[5]; in exec_drive_taskfile()
1789 fis.features_ex = req_task->hob_ports[1]; in exec_drive_taskfile()
1790 fis.sect_cnt_ex = req_task->hob_ports[2]; in exec_drive_taskfile()
1793 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; in exec_drive_taskfile()
1796 force_single_sector = implicit_sector(fis.command, fis.features); in exec_drive_taskfile()
1798 if ((taskin || taskout) && (!fis.sect_count)) { in exec_drive_taskfile()
1800 fis.sect_count = nsect; in exec_drive_taskfile()
1803 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
1806 err = -EINVAL; in exec_drive_taskfile()
1817 fis.command, in exec_drive_taskfile()
1818 fis.features, in exec_drive_taskfile()
1819 fis.sect_count, in exec_drive_taskfile()
1820 fis.lba_low, in exec_drive_taskfile()
1821 fis.lba_mid, in exec_drive_taskfile()
1822 fis.lba_hi, in exec_drive_taskfile()
1823 fis.device); in exec_drive_taskfile()
1826 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && in exec_drive_taskfile()
1831 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
1837 transfer_size = ATA_SECT_SIZE * fis.sect_count; in exec_drive_taskfile()
1840 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
1841 &fis, in exec_drive_taskfile()
1847 err = -EIO; in exec_drive_taskfile()
1851 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
1853 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { in exec_drive_taskfile()
1854 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
1855 req_task->io_ports[7] = reply->control; in exec_drive_taskfile()
1857 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
1858 req_task->io_ports[7] = reply->command; in exec_drive_taskfile()
1863 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1866 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1872 req_task->io_ports[1] = reply->features; in exec_drive_taskfile()
1873 req_task->io_ports[2] = reply->sect_count; in exec_drive_taskfile()
1874 req_task->io_ports[3] = reply->lba_low; in exec_drive_taskfile()
1875 req_task->io_ports[4] = reply->lba_mid; in exec_drive_taskfile()
1876 req_task->io_ports[5] = reply->lba_hi; in exec_drive_taskfile()
1877 req_task->io_ports[6] = reply->device; in exec_drive_taskfile()
1879 if (req_task->out_flags.all & 1) { in exec_drive_taskfile()
1881 req_task->hob_ports[3] = reply->lba_low_ex; in exec_drive_taskfile()
1882 req_task->hob_ports[4] = reply->lba_mid_ex; in exec_drive_taskfile()
1883 req_task->hob_ports[5] = reply->lba_hi_ex; in exec_drive_taskfile()
1884 req_task->hob_ports[1] = reply->features_ex; in exec_drive_taskfile()
1885 req_task->hob_ports[2] = reply->sect_cnt_ex; in exec_drive_taskfile()
1892 req_task->io_ports[7], in exec_drive_taskfile()
1893 req_task->io_ports[1], in exec_drive_taskfile()
1894 req_task->io_ports[2], in exec_drive_taskfile()
1895 req_task->io_ports[3], in exec_drive_taskfile()
1896 req_task->io_ports[4], in exec_drive_taskfile()
1897 req_task->io_ports[5], in exec_drive_taskfile()
1898 req_task->io_ports[6]); in exec_drive_taskfile()
1902 err = -EFAULT; in exec_drive_taskfile()
1908 err = -EFAULT; in exec_drive_taskfile()
1914 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1917 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1926 * Handle IOCTL calls from the Block Layer.
1928 * This function is called by the Block Layer when it receives an IOCTL
1930 * this function returns -ENOTTY.
1933 * @cmd IOCTL command passed from the Block Layer.
1934 * @arg IOCTL argument passed from the Block Layer.
1938 * -ENOTTY The specified command is not supported.
1939 * -EFAULT An error occurred copying data to a user space buffer.
1940 * -EIO An error occurred while executing the command.
1948 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
1950 return -EFAULT; in mtip_hw_ioctl()
1961 return -EFAULT; in mtip_hw_ioctl()
1964 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
1967 return -EIO; in mtip_hw_ioctl()
1973 return -EFAULT; in mtip_hw_ioctl()
1985 return -EFAULT; in mtip_hw_ioctl()
1988 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
1989 return -EIO; in mtip_hw_ioctl()
1995 return -EFAULT; in mtip_hw_ioctl()
2005 return -EFAULT; in mtip_hw_ioctl()
2014 return -EFAULT; in mtip_hw_ioctl()
2020 return -EINVAL; in mtip_hw_ioctl()
2028 * This function is called by the block layer to issue an io
2050 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_hw_submit_io()
2051 struct host_to_dev_fis *fis; in mtip_hw_submit_io() local
2052 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2059 nents = blk_rq_map_sg(hctx->queue, rq, command->sg); in mtip_hw_submit_io()
2060 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2062 prefetch(&port->flags); in mtip_hw_submit_io()
2064 command->scatter_ents = nents; in mtip_hw_submit_io()
2070 command->retries = MTIP_MAX_RETRIES; in mtip_hw_submit_io()
2072 /* Fill out fis */ in mtip_hw_submit_io()
2073 fis = command->command; in mtip_hw_submit_io()
2074 fis->type = 0x27; in mtip_hw_submit_io()
2075 fis->opts = 1 << 7; in mtip_hw_submit_io()
2077 fis->command = ATA_CMD_FPDMA_READ; in mtip_hw_submit_io()
2079 fis->command = ATA_CMD_FPDMA_WRITE; in mtip_hw_submit_io()
2080 fis->lba_low = start & 0xFF; in mtip_hw_submit_io()
2081 fis->lba_mid = (start >> 8) & 0xFF; in mtip_hw_submit_io()
2082 fis->lba_hi = (start >> 16) & 0xFF; in mtip_hw_submit_io()
2083 fis->lba_low_ex = (start >> 24) & 0xFF; in mtip_hw_submit_io()
2084 fis->lba_mid_ex = (start >> 32) & 0xFF; in mtip_hw_submit_io()
2085 fis->lba_hi_ex = (start >> 40) & 0xFF; in mtip_hw_submit_io()
2086 fis->device = 1 << 6; in mtip_hw_submit_io()
2087 fis->features = nsect & 0xFF; in mtip_hw_submit_io()
2088 fis->features_ex = (nsect >> 8) & 0xFF; in mtip_hw_submit_io()
2089 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); in mtip_hw_submit_io()
2090 fis->sect_cnt_ex = 0; in mtip_hw_submit_io()
2091 fis->control = 0; in mtip_hw_submit_io()
2092 fis->res2 = 0; in mtip_hw_submit_io()
2093 fis->res3 = 0; in mtip_hw_submit_io()
2096 if (unlikely(command->unaligned)) in mtip_hw_submit_io()
2097 fis->device |= 1 << 7; in mtip_hw_submit_io()
2100 hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF); in mtip_hw_submit_io()
2101 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_hw_submit_io()
2102 hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16); in mtip_hw_submit_io()
2103 hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH); in mtip_hw_submit_io()
2104 hdr->byte_count = 0; in mtip_hw_submit_io()
2106 command->direction = dma_dir; in mtip_hw_submit_io()
2112 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { in mtip_hw_submit_io()
2113 set_bit(rq->tag, port->cmds_to_issue); in mtip_hw_submit_io()
2114 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_hw_submit_io()
2119 mtip_issue_ncq_command(port, rq->tag); in mtip_hw_submit_io()
2136 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status()
2139 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2141 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2168 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers()
2179 return -ENOMEM; in mtip_hw_read_registers()
2183 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2185 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2190 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2192 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2197 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2199 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2203 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2205 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2210 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2213 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2215 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2223 rv = -EFAULT; in mtip_hw_read_registers()
2232 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags()
2242 return -ENOMEM; in mtip_hw_read_flags()
2244 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", in mtip_hw_read_flags()
2245 dd->port->flags); in mtip_hw_read_flags()
2246 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", in mtip_hw_read_flags()
2247 dd->dd_flag); in mtip_hw_read_flags()
2252 rv = -EFAULT; in mtip_hw_read_flags()
2272 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2273 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); in mtip_hw_debugfs_init()
2274 debugfs_create_file("registers", 0444, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2280 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
2294 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
2300 dd->mmio + HOST_HSORG); in hba_setup()
2305 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
2324 * HBA base + 0xFC [15:0] - vendor-specific hardware interface in mtip_detect_product()
2327 * [ 3] asic-style interface in mtip_detect_product()
2328 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). in mtip_detect_product()
2330 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
2332 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
2333 dd->slot_groups = 1; in mtip_detect_product()
2336 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
2339 dev_info(&dd->pdev->dev, in mtip_detect_product()
2340 "ASIC-FPGA design, HS rev 0x%x, " in mtip_detect_product()
2347 dev_warn(&dd->pdev->dev, in mtip_detect_product()
2352 dd->slot_groups = slotgroups; in mtip_detect_product()
2356 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
2366 * -EFAULT FTL rebuild error/timeout/interruption
2372 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2380 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
2381 return -EFAULT; in mtip_ftl_rebuild_poll()
2383 return -EFAULT; in mtip_ftl_rebuild_poll()
2385 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2386 return -EFAULT; in mtip_ftl_rebuild_poll()
2388 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2393 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2395 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2399 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2401 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2408 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2410 jiffies_to_msecs(jiffies - start) / 1000); in mtip_ftl_rebuild_poll()
2411 return -EFAULT; in mtip_ftl_rebuild_poll()
2417 struct driver_data *dd = rq->q->queuedata; in mtip_softirq_done_fn()
2420 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, in mtip_softirq_done_fn()
2421 cmd->direction); in mtip_softirq_done_fn()
2423 if (unlikely(cmd->unaligned)) in mtip_softirq_done_fn()
2424 atomic_inc(&dd->port->cmd_slot_unal); in mtip_softirq_done_fn()
2426 blk_mq_end_request(rq, cmd->status); in mtip_softirq_done_fn()
2434 dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag); in mtip_abort_cmd()
2436 clear_bit(req->tag, dd->port->cmds_to_issue); in mtip_abort_cmd()
2437 cmd->status = BLK_STS_IOERR; in mtip_abort_cmd()
2446 set_bit(req->tag, dd->port->cmds_to_issue); in mtip_queue_cmd()
2464 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
2465 struct mtip_port *port = dd->port; in mtip_service_thread()
2469 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2471 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2477 wait_event_interruptible(port->svc_wait, (port->flags) && in mtip_service_thread()
2478 (port->flags & MTIP_PF_SVC_THD_WORK)); in mtip_service_thread()
2481 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) in mtip_service_thread()
2485 &dd->dd_flag))) in mtip_service_thread()
2488 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2492 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2494 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); in mtip_service_thread()
2497 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) in mtip_service_thread()
2500 if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { in mtip_service_thread()
2505 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_service_thread()
2508 if (atomic_read(&dd->irq_workers_active) != 0) in mtip_service_thread()
2509 dev_warn(&dd->pdev->dev, in mtip_service_thread()
2512 blk_mq_quiesce_queue(dd->queue); in mtip_service_thread()
2514 blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd); in mtip_service_thread()
2516 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); in mtip_service_thread()
2519 blk_mq_tagset_busy_iter(&dd->tags, in mtip_service_thread()
2522 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); in mtip_service_thread()
2524 blk_mq_unquiesce_queue(dd->queue); in mtip_service_thread()
2527 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { in mtip_service_thread()
2533 slot = find_next_bit(port->cmds_to_issue, in mtip_service_thread()
2552 clear_bit(slot, port->cmds_to_issue); in mtip_service_thread()
2555 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); in mtip_service_thread()
2558 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { in mtip_service_thread()
2560 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); in mtip_service_thread()
2578 struct mtip_port *port = dd->port; in mtip_dma_free()
2580 if (port->block1) in mtip_dma_free()
2581 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_free()
2582 port->block1, port->block1_dma); in mtip_dma_free()
2584 if (port->command_list) { in mtip_dma_free()
2585 dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_free()
2586 port->command_list, port->command_list_dma); in mtip_dma_free()
2596 * -ENOMEM Not enough free DMA region space to initialize driver
2600 struct mtip_port *port = dd->port; in mtip_dma_alloc()
2602 /* Allocate dma memory for RX Fis, Identify, and Sector Buffer */ in mtip_dma_alloc()
2603 port->block1 = in mtip_dma_alloc()
2604 dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2605 &port->block1_dma, GFP_KERNEL); in mtip_dma_alloc()
2606 if (!port->block1) in mtip_dma_alloc()
2607 return -ENOMEM; in mtip_dma_alloc()
2610 port->command_list = in mtip_dma_alloc()
2611 dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_alloc()
2612 &port->command_list_dma, GFP_KERNEL); in mtip_dma_alloc()
2613 if (!port->command_list) { in mtip_dma_alloc()
2614 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2615 port->block1, port->block1_dma); in mtip_dma_alloc()
2616 port->block1 = NULL; in mtip_dma_alloc()
2617 port->block1_dma = 0; in mtip_dma_alloc()
2618 return -ENOMEM; in mtip_dma_alloc()
2622 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
2623 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; in mtip_dma_alloc()
2624 port->identify = port->block1 + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
2625 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; in mtip_dma_alloc()
2626 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
2627 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; in mtip_dma_alloc()
2628 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
2629 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; in mtip_dma_alloc()
2640 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
2641 return -EFAULT; in mtip_hw_get_identify()
2643 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
2645 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
2648 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
2651 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
2652 dd->port->log_buf, in mtip_hw_get_identify()
2653 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
2655 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2657 /* non-critical error, don't fail the load */ in mtip_hw_get_identify()
2659 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
2661 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2663 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2666 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2668 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2671 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2673 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2679 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
2680 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2683 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2704 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; in mtip_hw_init()
2707 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
2708 rv = -EIO; in mtip_hw_init()
2714 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
2715 dd->numa_node); in mtip_hw_init()
2716 if (!dd->port) in mtip_hw_init()
2717 return -ENOMEM; in mtip_hw_init()
2721 dd->work[i].port = dd->port; in mtip_hw_init()
2725 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
2727 dd->unal_qdepth = 0; in mtip_hw_init()
2729 atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
2733 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
2736 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
2737 dd->port->dd = dd; in mtip_hw_init()
2745 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
2746 dd->port->s_active[i] = in mtip_hw_init()
2747 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
2748 dd->port->cmd_issue[i] = in mtip_hw_init()
2749 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
2750 dd->port->completed[i] = in mtip_hw_init()
2751 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
2756 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
2761 timetaken = jiffies - timetaken; in mtip_hw_init()
2762 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2765 rv = -ENODEV; in mtip_hw_init()
2768 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
2769 timetaken = jiffies - timetaken; in mtip_hw_init()
2770 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2773 rv = -EFAULT; in mtip_hw_init()
2778 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
2780 dev_err(&dd->pdev->dev, in mtip_hw_init()
2782 rv = -EIO; in mtip_hw_init()
2787 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
2788 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
2791 mtip_init_port(dd->port); in mtip_hw_init()
2792 mtip_start_port(dd->port); in mtip_hw_init()
2795 rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED, in mtip_hw_init()
2796 dev_driver_string(&dd->pdev->dev), dd); in mtip_hw_init()
2798 dev_err(&dd->pdev->dev, in mtip_hw_init()
2799 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
2802 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
2805 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
2806 dd->mmio + HOST_CTL); in mtip_hw_init()
2808 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
2810 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
2811 rv = -EFAULT; in mtip_hw_init()
2819 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
2820 dd->mmio + HOST_CTL); in mtip_hw_init()
2823 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
2824 free_irq(dd->pdev->irq, dd); in mtip_hw_init()
2827 mtip_deinit_port(dd->port); in mtip_hw_init()
2832 kfree(dd->port); in mtip_hw_init()
2841 if (dd->sr || !dd->port) in mtip_standby_drive()
2842 return -ENODEV; in mtip_standby_drive()
2847 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
2848 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && in mtip_standby_drive()
2849 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { in mtip_standby_drive()
2850 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
2852 dev_warn(&dd->pdev->dev, in mtip_standby_drive()
2868 if (!dd->sr) { in mtip_hw_exit()
2869 /* de-initialize the port. */ in mtip_hw_exit()
2870 mtip_deinit_port(dd->port); in mtip_hw_exit()
2873 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
2874 dd->mmio + HOST_CTL); in mtip_hw_exit()
2878 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
2879 free_irq(dd->pdev->irq, dd); in mtip_hw_exit()
2886 kfree(dd->port); in mtip_hw_exit()
2887 dd->port = NULL; in mtip_hw_exit()
2895 * This function is called by the Block Layer just before the
2917 * This function is called by the Block Layer just before the
2924 * -EFAULT Suspend was not successful
2933 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
2934 "Failed standby-immediate command\n"); in mtip_hw_suspend()
2935 return -EFAULT; in mtip_hw_suspend()
2939 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
2940 dd->mmio + HOST_CTL); in mtip_hw_suspend()
2941 mtip_deinit_port(dd->port); in mtip_hw_suspend()
2949 * This function is called by the Block Layer as the
2956 * -EFAULT Resume was not successful
2965 dev_err(&dd->pdev->dev, in mtip_hw_resume()
2967 return -EFAULT; in mtip_hw_resume()
2971 * Enable the port, DMA engine, and FIS reception specific in mtip_hw_resume()
2974 mtip_init_port(dd->port); in mtip_hw_resume()
2975 mtip_start_port(dd->port); in mtip_hw_resume()
2978 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
2979 dd->mmio + HOST_CTL); in mtip_hw_resume()
2989 int index, in rssd_disk_name_format() argument
2993 const int base = 'z' - 'a' + 1; in rssd_disk_name_format()
2999 p = end - 1; in rssd_disk_name_format()
3004 return -EINVAL; in rssd_disk_name_format()
3005 *--p = 'a' + (index % unit); in rssd_disk_name_format()
3006 index = (index / unit) - 1; in rssd_disk_name_format()
3007 } while (index >= 0); in rssd_disk_name_format()
3009 memmove(begin, p, end - p); in rssd_disk_name_format()
3016 * Block layer IOCTL handler.
3025 * -ENOTTY IOCTL not supported or invalid driver data
3033 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl()
3036 return -EACCES; in mtip_block_ioctl()
3039 return -ENOTTY; in mtip_block_ioctl()
3041 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3042 return -ENOTTY; in mtip_block_ioctl()
3046 return -ENOTTY; in mtip_block_ioctl()
3054 * Block layer compat IOCTL handler.
3063 * -ENOTTY IOCTL not supported or invalid driver data
3071 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl()
3074 return -EACCES; in mtip_block_compat_ioctl()
3077 return -ENOTTY; in mtip_block_compat_ioctl()
3079 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3080 return -ENOTTY; in mtip_block_compat_ioctl()
3084 return -ENOTTY; in mtip_block_compat_ioctl()
3097 compat_tasksize - (2 * sizeof(compat_long_t)))) in mtip_block_compat_ioctl()
3098 return -EFAULT; in mtip_block_compat_ioctl()
3100 if (get_user(req_task.out_size, &compat_req_task->out_size)) in mtip_block_compat_ioctl()
3101 return -EFAULT; in mtip_block_compat_ioctl()
3103 if (get_user(req_task.in_size, &compat_req_task->in_size)) in mtip_block_compat_ioctl()
3104 return -EFAULT; in mtip_block_compat_ioctl()
3112 compat_tasksize - in mtip_block_compat_ioctl()
3114 return -EFAULT; in mtip_block_compat_ioctl()
3116 if (put_user(req_task.out_size, &compat_req_task->out_size)) in mtip_block_compat_ioctl()
3117 return -EFAULT; in mtip_block_compat_ioctl()
3119 if (put_user(req_task.in_size, &compat_req_task->in_size)) in mtip_block_compat_ioctl()
3120 return -EFAULT; in mtip_block_compat_ioctl()
3138 * that each partition is also 4KB aligned. Non-aligned partitions adversely
3146 * -ENOTTY An error occurred while reading the drive capacity.
3151 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo()
3155 return -ENOTTY; in mtip_block_getgeo()
3158 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3160 return -ENOTTY; in mtip_block_getgeo()
3163 geo->heads = 224; in mtip_block_getgeo()
3164 geo->sectors = 56; in mtip_block_getgeo()
3165 sector_div(capacity, (geo->heads * geo->sectors)); in mtip_block_getgeo()
3166 geo->cylinders = capacity; in mtip_block_getgeo()
3172 struct driver_data *dd = disk->private_data; in mtip_block_free_disk()
3174 ida_free(&rssd_index_ida, dd->index); in mtip_block_free_disk()
3179 * Block device operation function.
3181 * This structure contains pointers to the functions required by the block
3196 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { in is_se_active()
3197 if (dd->port->ic_pause_timer) { in is_se_active()
3198 unsigned long to = dd->port->ic_pause_timer + in is_se_active()
3202 &dd->port->flags); in is_se_active()
3203 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); in is_se_active()
3204 dd->port->ic_pause_timer = 0; in is_se_active()
3205 wake_up_interruptible(&dd->port->svc_wait); in is_se_active()
3216 if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO))) in is_stopped()
3219 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in is_stopped()
3221 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in is_stopped()
3223 if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) && in is_stopped()
3226 if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) in is_stopped()
3228 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) in is_stopped()
3237 struct driver_data *dd = hctx->queue->queuedata; in mtip_check_unal_depth()
3240 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) in mtip_check_unal_depth()
3249 cmd->unaligned = 1; in mtip_check_unal_depth()
3252 if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0) in mtip_check_unal_depth()
3261 struct driver_data *dd = hctx->queue->queuedata; in mtip_issue_reserved_cmd()
3263 struct mtip_int_cmd *icmd = cmd->icmd; in mtip_issue_reserved_cmd()
3265 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_issue_reserved_cmd()
3268 if (mtip_commands_active(dd->port)) in mtip_issue_reserved_cmd()
3271 hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); in mtip_issue_reserved_cmd()
3272 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_issue_reserved_cmd()
3273 hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16); in mtip_issue_reserved_cmd()
3275 hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len); in mtip_issue_reserved_cmd()
3276 if (icmd->buf_len) { in mtip_issue_reserved_cmd()
3277 command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ; in mtip_issue_reserved_cmd()
3279 command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF); in mtip_issue_reserved_cmd()
3280 command_sg->dba = cpu_to_le32(icmd->buffer & 0xFFFFFFFF); in mtip_issue_reserved_cmd()
3281 command_sg->dba_upper = in mtip_issue_reserved_cmd()
3282 cpu_to_le32((icmd->buffer >> 16) >> 16); in mtip_issue_reserved_cmd()
3284 hdr->opts |= cpu_to_le32((1 << 16)); in mtip_issue_reserved_cmd()
3288 hdr->byte_count = 0; in mtip_issue_reserved_cmd()
3291 mtip_issue_non_ncq_command(dd->port, rq->tag); in mtip_issue_reserved_cmd()
3298 struct driver_data *dd = hctx->queue->queuedata; in mtip_queue_rq()
3299 struct request *rq = bd->rq; in mtip_queue_rq()
3320 struct driver_data *dd = set->driver_data; in mtip_free_cmd()
3323 if (!cmd->command) in mtip_free_cmd()
3326 dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command, in mtip_free_cmd()
3327 cmd->command_dma); in mtip_free_cmd()
3333 struct driver_data *dd = set->driver_data; in mtip_init_cmd()
3336 cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_init_cmd()
3337 &cmd->command_dma, GFP_KERNEL); in mtip_init_cmd()
3338 if (!cmd->command) in mtip_init_cmd()
3339 return -ENOMEM; in mtip_init_cmd()
3341 sg_init_table(cmd->sg, MTIP_MAX_SG); in mtip_init_cmd()
3347 struct driver_data *dd = req->q->queuedata; in mtip_cmd_timeout()
3352 cmd->status = BLK_STS_TIMEOUT; in mtip_cmd_timeout()
3357 if (test_bit(req->tag, dd->port->cmds_to_issue)) in mtip_cmd_timeout()
3360 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) in mtip_cmd_timeout()
3363 wake_up_interruptible(&dd->port->svc_wait); in mtip_cmd_timeout()
3377 * Block layer initialization function.
3397 unsigned int index = 0; in mtip_block_initialize() local
3399 if (dd->disk) in mtip_block_initialize()
3403 rv = -EINVAL; in mtip_block_initialize()
3407 memset(&dd->tags, 0, sizeof(dd->tags)); in mtip_block_initialize()
3408 dd->tags.ops = &mtip_mq_ops; in mtip_block_initialize()
3409 dd->tags.nr_hw_queues = 1; in mtip_block_initialize()
3410 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; in mtip_block_initialize()
3411 dd->tags.reserved_tags = 1; in mtip_block_initialize()
3412 dd->tags.cmd_size = sizeof(struct mtip_cmd); in mtip_block_initialize()
3413 dd->tags.numa_node = dd->numa_node; in mtip_block_initialize()
3414 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; in mtip_block_initialize()
3415 dd->tags.driver_data = dd; in mtip_block_initialize()
3416 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; in mtip_block_initialize()
3418 rv = blk_mq_alloc_tag_set(&dd->tags); in mtip_block_initialize()
3420 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3425 dd->disk = blk_mq_alloc_disk(&dd->tags, &lim, dd); in mtip_block_initialize()
3426 if (IS_ERR(dd->disk)) { in mtip_block_initialize()
3427 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3429 rv = -ENOMEM; in mtip_block_initialize()
3432 dd->queue = dd->disk->queue; in mtip_block_initialize()
3437 index = rv; in mtip_block_initialize()
3440 index, in mtip_block_initialize()
3441 dd->disk->disk_name, in mtip_block_initialize()
3446 dd->disk->major = dd->major; in mtip_block_initialize()
3447 dd->disk->first_minor = index * MTIP_MAX_MINORS; in mtip_block_initialize()
3448 dd->disk->minors = MTIP_MAX_MINORS; in mtip_block_initialize()
3449 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
3450 dd->disk->private_data = dd; in mtip_block_initialize()
3451 dd->index = index; in mtip_block_initialize()
3459 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3461 rv = -EINVAL; in mtip_block_initialize()
3466 * if rebuild pending, start the service thread, and delay the block in mtip_block_initialize()
3473 dma_set_max_seg_size(&dd->pdev->dev, 0x400000); in mtip_block_initialize()
3477 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
3479 rv = -EIO; in mtip_block_initialize()
3482 set_capacity(dd->disk, capacity); in mtip_block_initialize()
3484 /* Enable the block device and add it to /dev */ in mtip_block_initialize()
3485 rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); in mtip_block_initialize()
3489 if (dd->mtip_svc_handler) { in mtip_block_initialize()
3490 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
3495 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
3496 dd, dd->numa_node, in mtip_block_initialize()
3497 "mtip_svc_thd_%02d", index); in mtip_block_initialize()
3499 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
3500 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
3501 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
3502 rv = -EFAULT; in mtip_block_initialize()
3505 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
3513 del_gendisk(dd->disk); in mtip_block_initialize()
3518 ida_free(&rssd_index_ida, index); in mtip_block_initialize()
3520 put_disk(dd->disk); in mtip_block_initialize()
3522 blk_mq_free_tag_set(&dd->tags); in mtip_block_initialize()
3524 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
3545 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
3546 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
3548 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_block_shutdown()
3549 del_gendisk(dd->disk); in mtip_block_shutdown()
3551 blk_mq_free_tag_set(&dd->tags); in mtip_block_shutdown()
3552 put_disk(dd->disk); in mtip_block_shutdown()
3558 dev_info(&dd->pdev->dev, in mtip_block_suspend()
3559 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
3566 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
3567 dd->disk->disk_name); in mtip_block_resume()
3574 cpu_use[cpu]--; in drop_cpu()
3630 dev_info(&dd->pdev->dev, in mtip_disable_link_opts()
3631 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", in mtip_disable_link_opts()
3632 pdev->vendor, pdev->device); in mtip_disable_link_opts()
3647 if (pdev->bus && pdev->bus->self) { in mtip_fix_ero_nosnoop()
3648 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && in mtip_fix_ero_nosnoop()
3649 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { in mtip_fix_ero_nosnoop()
3650 mtip_disable_link_opts(dd, pdev->bus->self); in mtip_fix_ero_nosnoop()
3653 struct pci_dev *parent_dev = pdev->bus->self; in mtip_fix_ero_nosnoop()
3654 if (parent_dev->bus && in mtip_fix_ero_nosnoop()
3655 parent_dev->bus->parent && in mtip_fix_ero_nosnoop()
3656 parent_dev->bus->parent->self && in mtip_fix_ero_nosnoop()
3657 parent_dev->bus->parent->self->vendor == in mtip_fix_ero_nosnoop()
3659 (parent_dev->bus->parent->self->device & in mtip_fix_ero_nosnoop()
3662 parent_dev->bus->parent->self); in mtip_fix_ero_nosnoop()
3672 * PCI device and then calls the block layer initialization function.
3688 my_node = pcibus_to_node(pdev->bus); in mtip_pci_probe()
3693 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); in mtip_pci_probe()
3696 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", in mtip_pci_probe()
3697 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), in mtip_pci_probe()
3702 return -ENOMEM; in mtip_pci_probe()
3709 dev_err(&pdev->dev, "Unable to enable device\n"); in mtip_pci_probe()
3716 dev_err(&pdev->dev, "Unable to map regions\n"); in mtip_pci_probe()
3720 rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in mtip_pci_probe()
3722 dev_warn(&pdev->dev, "64-bit DMA enable failed\n"); in mtip_pci_probe()
3727 dd->major = mtip_major; in mtip_pci_probe()
3728 dd->instance = instance; in mtip_pci_probe()
3729 dd->pdev = pdev; in mtip_pci_probe()
3730 dd->numa_node = my_node; in mtip_pci_probe()
3732 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
3733 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
3735 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
3736 if (!dd->isr_workq) { in mtip_pci_probe()
3737 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
3738 rv = -ENOMEM; in mtip_pci_probe()
3744 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
3748 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); in mtip_pci_probe()
3752 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", in mtip_pci_probe()
3753 dd->numa_node, in mtip_pci_probe()
3755 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
3758 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); in mtip_pci_probe()
3760 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3761 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", in mtip_pci_probe()
3762 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
3765 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
3766 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3767 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3768 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
3769 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3770 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3771 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3772 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3778 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
3779 snprintf(&cpu_list[j], 256 - j, "%d ", i); in mtip_pci_probe()
3784 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); in mtip_pci_probe()
3787 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
3788 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
3789 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
3790 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
3791 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
3792 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
3793 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
3794 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
3799 dev_warn(&pdev->dev, in mtip_pci_probe()
3806 /* Initialize the block layer. */ in mtip_pci_probe()
3809 dev_err(&pdev->dev, in mtip_pci_probe()
3810 "Unable to initialize block layer\n"); in mtip_pci_probe()
3820 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
3830 if (dd->isr_workq) { in mtip_pci_probe()
3831 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
3832 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
3833 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
3834 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
3860 synchronize_irq(dd->pdev->irq); in mtip_pci_remove()
3866 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_pci_remove()
3869 if (atomic_read(&dd->irq_workers_active) != 0) { in mtip_pci_remove()
3870 dev_warn(&dd->pdev->dev, in mtip_pci_remove()
3874 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
3876 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_pci_remove()
3877 del_gendisk(dd->disk); in mtip_pci_remove()
3881 if (dd->mtip_svc_handler) { in mtip_pci_remove()
3882 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_pci_remove()
3883 wake_up_interruptible(&dd->port->svc_wait); in mtip_pci_remove()
3884 kthread_stop(dd->mtip_svc_handler); in mtip_pci_remove()
3887 if (!dd->sr) { in mtip_pci_remove()
3892 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) in mtip_pci_remove()
3896 dev_info(&dd->pdev->dev, "device %s surprise removal\n", in mtip_pci_remove()
3897 dd->disk->disk_name); in mtip_pci_remove()
3899 blk_mq_free_tag_set(&dd->tags); in mtip_pci_remove()
3901 /* De-initialize the protocol layer. */ in mtip_pci_remove()
3904 if (dd->isr_workq) { in mtip_pci_remove()
3905 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
3906 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
3907 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
3908 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
3916 put_disk(dd->disk); in mtip_pci_remove()
3931 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
3961 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
4009 * block device number to the Cyclone devices and registers the PCI layer
4021 /* Allocate a major block device number to use with this driver. */ in mtip_init()
4024 pr_err("Unable to register block device (%d)\n", in mtip_init()
4026 return -EBUSY; in mtip_init()
4043 * Module de-initialization function.
4046 * the major block device number allocated by mtip_init() and
4054 /* Release the allocated major block device number. */ in mtip_exit()
4064 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");