1 /* 2 * Driver for the Micron P320 SSD 3 * Copyright (C) 2011 Micron Technology, Inc. 4 * 5 * Portions of this code were derived from works subjected to the 6 * following copyright: 7 * Copyright (C) 2009 Integrated Device Technology, Inc. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 */ 20 21 #include <linux/pci.h> 22 #include <linux/interrupt.h> 23 #include <linux/ata.h> 24 #include <linux/delay.h> 25 #include <linux/hdreg.h> 26 #include <linux/uaccess.h> 27 #include <linux/random.h> 28 #include <linux/smp.h> 29 #include <linux/compat.h> 30 #include <linux/fs.h> 31 #include <linux/module.h> 32 #include <linux/genhd.h> 33 #include <linux/blkdev.h> 34 #include <linux/blk-mq.h> 35 #include <linux/bio.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/idr.h> 38 #include <linux/kthread.h> 39 #include <../drivers/ata/ahci.h> 40 #include <linux/export.h> 41 #include <linux/debugfs.h> 42 #include <linux/prefetch.h> 43 #include "mtip32xx.h" 44 45 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 46 47 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */ 48 #define AHCI_RX_FIS_SZ 0x100 49 #define AHCI_RX_FIS_OFFSET 0x0 50 #define AHCI_IDFY_SZ ATA_SECT_SIZE 51 #define AHCI_IDFY_OFFSET 0x400 52 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 53 #define AHCI_SECTBUF_OFFSET 0x800 54 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 55 #define AHCI_SMARTBUF_OFFSET 0xC00 56 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */ 57 #define BLOCK_DMA_ALLOC_SZ 4096 58 59 /* DMA region containing command table (should be 8192 bytes) */ 60 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 61 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 62 #define AHCI_CMD_TBL_OFFSET 0x0 63 64 /* DMA region per command (contains header and SGL) */ 65 #define AHCI_CMD_TBL_HDR_SZ 0x80 66 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 67 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 68 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 69 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 70 71 72 #define HOST_CAP_NZDMA (1 << 19) 73 #define HOST_HSORG 0xFC 74 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 75 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 76 #define HSORG_HWREV 0xFF00 77 #define HSORG_STYLE 0x8 78 #define HSORG_SLOTGROUPS 0x7 79 80 #define PORT_COMMAND_ISSUE 0x38 81 #define PORT_SDBV 0x7C 82 83 #define PORT_OFFSET 0x100 84 #define PORT_MEM_SIZE 0x80 85 86 #define PORT_IRQ_ERR \ 87 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 88 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 89 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 90 PORT_IRQ_OVERFLOW) 91 #define PORT_IRQ_LEGACY \ 92 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 93 #define PORT_IRQ_HANDLED \ 94 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 95 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 96 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 97 #define DEF_PORT_IRQ \ 98 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 99 100 /* product numbers */ 101 #define MTIP_PRODUCT_UNKNOWN 0x00 102 #define MTIP_PRODUCT_ASICFPGA 0x11 103 104 /* Device instance number, incremented each time a device is probed. */ 105 static int instance; 106 107 static struct list_head online_list; 108 static struct list_head removing_list; 109 static spinlock_t dev_lock; 110 111 /* 112 * Global variable used to hold the major block device number 113 * allocated in mtip_init(). 114 */ 115 static int mtip_major; 116 static struct dentry *dfs_parent; 117 static struct dentry *dfs_device_status; 118 119 static u32 cpu_use[NR_CPUS]; 120 121 static DEFINE_SPINLOCK(rssd_index_lock); 122 static DEFINE_IDA(rssd_index_ida); 123 124 static int mtip_block_initialize(struct driver_data *dd); 125 126 #ifdef CONFIG_COMPAT 127 struct mtip_compat_ide_task_request_s { 128 __u8 io_ports[8]; 129 __u8 hob_ports[8]; 130 ide_reg_valid_t out_flags; 131 ide_reg_valid_t in_flags; 132 int data_phase; 133 int req_cmd; 134 compat_ulong_t out_size; 135 compat_ulong_t in_size; 136 }; 137 #endif 138 139 /* 140 * This function check_for_surprise_removal is called 141 * while card is removed from the system and it will 142 * read the vendor id from the configration space 143 * 144 * @pdev Pointer to the pci_dev structure. 145 * 146 * return value 147 * true if device removed, else false 148 */ 149 static bool mtip_check_surprise_removal(struct pci_dev *pdev) 150 { 151 u16 vendor_id = 0; 152 struct driver_data *dd = pci_get_drvdata(pdev); 153 154 if (dd->sr) 155 return true; 156 157 /* Read the vendorID from the configuration space */ 158 pci_read_config_word(pdev, 0x00, &vendor_id); 159 if (vendor_id == 0xFFFF) { 160 dd->sr = true; 161 if (dd->queue) 162 blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue); 163 else 164 dev_warn(&dd->pdev->dev, 165 "%s: dd->queue is NULL\n", __func__); 166 return true; /* device removed */ 167 } 168 169 return false; /* device present */ 170 } 171 172 /* we have to use runtime tag to setup command header */ 173 static void mtip_init_cmd_header(struct request *rq) 174 { 175 struct driver_data *dd = rq->q->queuedata; 176 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 177 178 /* Point the command headers at the command tables. */ 179 cmd->command_header = dd->port->command_list + 180 (sizeof(struct mtip_cmd_hdr) * rq->tag); 181 cmd->command_header_dma = dd->port->command_list_dma + 182 (sizeof(struct mtip_cmd_hdr) * rq->tag); 183 184 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) 185 cmd->command_header->ctbau = __force_bit2int cpu_to_le32((cmd->command_dma >> 16) >> 16); 186 187 cmd->command_header->ctba = __force_bit2int cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); 188 } 189 190 static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) 191 { 192 struct request *rq; 193 194 if (mtip_check_surprise_removal(dd->pdev)) 195 return NULL; 196 197 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); 198 if (IS_ERR(rq)) 199 return NULL; 200 201 /* Internal cmd isn't submitted via .queue_rq */ 202 mtip_init_cmd_header(rq); 203 204 return blk_mq_rq_to_pdu(rq); 205 } 206 207 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, 208 unsigned int tag) 209 { 210 struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0]; 211 212 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag)); 213 } 214 215 /* 216 * Reset the HBA (without sleeping) 217 * 218 * @dd Pointer to the driver data structure. 219 * 220 * return value 221 * 0 The reset was successful. 222 * -1 The HBA Reset bit did not clear. 223 */ 224 static int mtip_hba_reset(struct driver_data *dd) 225 { 226 unsigned long timeout; 227 228 /* Set the reset bit */ 229 writel(HOST_RESET, dd->mmio + HOST_CTL); 230 231 /* Flush */ 232 readl(dd->mmio + HOST_CTL); 233 234 /* 235 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec 236 * is 1 sec but in LUN failure conditions, up to 10 secs are required 237 */ 238 timeout = jiffies + msecs_to_jiffies(10000); 239 do { 240 mdelay(10); 241 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) 242 return -1; 243 244 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) 245 && time_before(jiffies, timeout)); 246 247 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) 248 return -1; 249 250 return 0; 251 } 252 253 /* 254 * Issue a command to the hardware. 255 * 256 * Set the appropriate bit in the s_active and Command Issue hardware 257 * registers, causing hardware command processing to begin. 258 * 259 * @port Pointer to the port structure. 260 * @tag The tag of the command to be issued. 261 * 262 * return value 263 * None 264 */ 265 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) 266 { 267 int group = tag >> 5; 268 269 /* guard SACT and CI registers */ 270 spin_lock(&port->cmd_issue_lock[group]); 271 writel((1 << MTIP_TAG_BIT(tag)), 272 port->s_active[MTIP_TAG_INDEX(tag)]); 273 writel((1 << MTIP_TAG_BIT(tag)), 274 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 275 spin_unlock(&port->cmd_issue_lock[group]); 276 } 277 278 /* 279 * Enable/disable the reception of FIS 280 * 281 * @port Pointer to the port data structure 282 * @enable 1 to enable, 0 to disable 283 * 284 * return value 285 * Previous state: 1 enabled, 0 disabled 286 */ 287 static int mtip_enable_fis(struct mtip_port *port, int enable) 288 { 289 u32 tmp; 290 291 /* enable FIS reception */ 292 tmp = readl(port->mmio + PORT_CMD); 293 if (enable) 294 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 295 else 296 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 297 298 /* Flush */ 299 readl(port->mmio + PORT_CMD); 300 301 return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); 302 } 303 304 /* 305 * Enable/disable the DMA engine 306 * 307 * @port Pointer to the port data structure 308 * @enable 1 to enable, 0 to disable 309 * 310 * return value 311 * Previous state: 1 enabled, 0 disabled. 312 */ 313 static int mtip_enable_engine(struct mtip_port *port, int enable) 314 { 315 u32 tmp; 316 317 /* enable FIS reception */ 318 tmp = readl(port->mmio + PORT_CMD); 319 if (enable) 320 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); 321 else 322 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); 323 324 readl(port->mmio + PORT_CMD); 325 return (((tmp & PORT_CMD_START) == PORT_CMD_START)); 326 } 327 328 /* 329 * Enables the port DMA engine and FIS reception. 330 * 331 * return value 332 * None 333 */ 334 static inline void mtip_start_port(struct mtip_port *port) 335 { 336 /* Enable FIS reception */ 337 mtip_enable_fis(port, 1); 338 339 /* Enable the DMA engine */ 340 mtip_enable_engine(port, 1); 341 } 342 343 /* 344 * Deinitialize a port by disabling port interrupts, the DMA engine, 345 * and FIS reception. 346 * 347 * @port Pointer to the port structure 348 * 349 * return value 350 * None 351 */ 352 static inline void mtip_deinit_port(struct mtip_port *port) 353 { 354 /* Disable interrupts on this port */ 355 writel(0, port->mmio + PORT_IRQ_MASK); 356 357 /* Disable the DMA engine */ 358 mtip_enable_engine(port, 0); 359 360 /* Disable FIS reception */ 361 mtip_enable_fis(port, 0); 362 } 363 364 /* 365 * Initialize a port. 366 * 367 * This function deinitializes the port by calling mtip_deinit_port() and 368 * then initializes it by setting the command header and RX FIS addresses, 369 * clearing the SError register and any pending port interrupts before 370 * re-enabling the default set of port interrupts. 371 * 372 * @port Pointer to the port structure. 373 * 374 * return value 375 * None 376 */ 377 static void mtip_init_port(struct mtip_port *port) 378 { 379 int i; 380 mtip_deinit_port(port); 381 382 /* Program the command list base and FIS base addresses */ 383 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { 384 writel((port->command_list_dma >> 16) >> 16, 385 port->mmio + PORT_LST_ADDR_HI); 386 writel((port->rxfis_dma >> 16) >> 16, 387 port->mmio + PORT_FIS_ADDR_HI); 388 set_bit(MTIP_PF_HOST_CAP_64, &port->flags); 389 } 390 391 writel(port->command_list_dma & 0xFFFFFFFF, 392 port->mmio + PORT_LST_ADDR); 393 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); 394 395 /* Clear SError */ 396 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); 397 398 /* reset the completed registers.*/ 399 for (i = 0; i < port->dd->slot_groups; i++) 400 writel(0xFFFFFFFF, port->completed[i]); 401 402 /* Clear any pending interrupts for this port */ 403 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); 404 405 /* Clear any pending interrupts on the HBA. */ 406 writel(readl(port->dd->mmio + HOST_IRQ_STAT), 407 port->dd->mmio + HOST_IRQ_STAT); 408 409 /* Enable port interrupts */ 410 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); 411 } 412 413 /* 414 * Restart a port 415 * 416 * @port Pointer to the port data structure. 417 * 418 * return value 419 * None 420 */ 421 static void mtip_restart_port(struct mtip_port *port) 422 { 423 unsigned long timeout; 424 425 /* Disable the DMA engine */ 426 mtip_enable_engine(port, 0); 427 428 /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ 429 timeout = jiffies + msecs_to_jiffies(500); 430 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) 431 && time_before(jiffies, timeout)) 432 ; 433 434 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 435 return; 436 437 /* 438 * Chip quirk: escalate to hba reset if 439 * PxCMD.CR not clear after 500 ms 440 */ 441 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { 442 dev_warn(&port->dd->pdev->dev, 443 "PxCMD.CR not clear, escalating reset\n"); 444 445 if (mtip_hba_reset(port->dd)) 446 dev_err(&port->dd->pdev->dev, 447 "HBA reset escalation failed.\n"); 448 449 /* 30 ms delay before com reset to quiesce chip */ 450 mdelay(30); 451 } 452 453 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); 454 455 /* Set PxSCTL.DET */ 456 writel(readl(port->mmio + PORT_SCR_CTL) | 457 1, port->mmio + PORT_SCR_CTL); 458 readl(port->mmio + PORT_SCR_CTL); 459 460 /* Wait 1 ms to quiesce chip function */ 461 timeout = jiffies + msecs_to_jiffies(1); 462 while (time_before(jiffies, timeout)) 463 ; 464 465 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 466 return; 467 468 /* Clear PxSCTL.DET */ 469 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, 470 port->mmio + PORT_SCR_CTL); 471 readl(port->mmio + PORT_SCR_CTL); 472 473 /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ 474 timeout = jiffies + msecs_to_jiffies(500); 475 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 476 && time_before(jiffies, timeout)) 477 ; 478 479 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 480 return; 481 482 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 483 dev_warn(&port->dd->pdev->dev, 484 "COM reset failed\n"); 485 486 mtip_init_port(port); 487 mtip_start_port(port); 488 489 } 490 491 static int mtip_device_reset(struct driver_data *dd) 492 { 493 int rv = 0; 494 495 if (mtip_check_surprise_removal(dd->pdev)) 496 return 0; 497 498 if (mtip_hba_reset(dd) < 0) 499 rv = -EFAULT; 500 501 mdelay(1); 502 mtip_init_port(dd->port); 503 mtip_start_port(dd->port); 504 505 /* Enable interrupts on the HBA. */ 506 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 507 dd->mmio + HOST_CTL); 508 return rv; 509 } 510 511 /* 512 * Helper function for tag logging 513 */ 514 static void print_tags(struct driver_data *dd, 515 char *msg, 516 unsigned long *tagbits, 517 int cnt) 518 { 519 unsigned char tagmap[128]; 520 int group, tagmap_len = 0; 521 522 memset(tagmap, 0, sizeof(tagmap)); 523 for (group = SLOTBITS_IN_LONGS; group > 0; group--) 524 tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ", 525 tagbits[group-1]); 526 dev_warn(&dd->pdev->dev, 527 "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap); 528 } 529 530 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 531 dma_addr_t buffer_dma, unsigned int sectors); 532 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 533 struct smart_attr *attrib); 534 535 static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status) 536 { 537 struct request *req = blk_mq_rq_from_pdu(cmd); 538 539 cmd->status = status; 540 blk_mq_complete_request(req); 541 } 542 543 /* 544 * Handle an error. 545 * 546 * @dd Pointer to the DRIVER_DATA structure. 547 * 548 * return value 549 * None 550 */ 551 static void mtip_handle_tfe(struct driver_data *dd) 552 { 553 int group, tag, bit, reissue, rv; 554 struct mtip_port *port; 555 struct mtip_cmd *cmd; 556 u32 completed; 557 struct host_to_dev_fis *fis; 558 unsigned long tagaccum[SLOTBITS_IN_LONGS]; 559 unsigned int cmd_cnt = 0; 560 unsigned char *buf; 561 char *fail_reason = NULL; 562 int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0; 563 564 dev_warn(&dd->pdev->dev, "Taskfile error\n"); 565 566 port = dd->port; 567 568 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { 569 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 570 dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); 571 mtip_complete_command(cmd, BLK_STS_IOERR); 572 return; 573 } 574 575 /* clear the tag accumulator */ 576 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 577 578 /* Loop through all the groups */ 579 for (group = 0; group < dd->slot_groups; group++) { 580 completed = readl(port->completed[group]); 581 582 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); 583 584 /* clear completed status register in the hardware.*/ 585 writel(completed, port->completed[group]); 586 587 /* Process successfully completed commands */ 588 for (bit = 0; bit < 32 && completed; bit++) { 589 if (!(completed & (1<<bit))) 590 continue; 591 tag = (group << 5) + bit; 592 593 /* Skip the internal command slot */ 594 if (tag == MTIP_TAG_INTERNAL) 595 continue; 596 597 cmd = mtip_cmd_from_tag(dd, tag); 598 mtip_complete_command(cmd, 0); 599 set_bit(tag, tagaccum); 600 cmd_cnt++; 601 } 602 } 603 604 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); 605 606 /* Restart the port */ 607 mdelay(20); 608 mtip_restart_port(port); 609 610 /* Trying to determine the cause of the error */ 611 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 612 dd->port->log_buf, 613 dd->port->log_buf_dma, 1); 614 if (rv) { 615 dev_warn(&dd->pdev->dev, 616 "Error in READ LOG EXT (10h) command\n"); 617 /* non-critical error, don't fail the load */ 618 } else { 619 buf = (unsigned char *)dd->port->log_buf; 620 if (buf[259] & 0x1) { 621 dev_info(&dd->pdev->dev, 622 "Write protect bit is set.\n"); 623 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 624 fail_all_ncq_write = 1; 625 fail_reason = "write protect"; 626 } 627 if (buf[288] == 0xF7) { 628 dev_info(&dd->pdev->dev, 629 "Exceeded Tmax, drive in thermal shutdown.\n"); 630 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 631 fail_all_ncq_cmds = 1; 632 fail_reason = "thermal shutdown"; 633 } 634 if (buf[288] == 0xBF) { 635 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); 636 dev_info(&dd->pdev->dev, 637 "Drive indicates rebuild has failed. Secure erase required.\n"); 638 fail_all_ncq_cmds = 1; 639 fail_reason = "rebuild failed"; 640 } 641 } 642 643 /* clear the tag accumulator */ 644 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 645 646 /* Loop through all the groups */ 647 for (group = 0; group < dd->slot_groups; group++) { 648 for (bit = 0; bit < 32; bit++) { 649 reissue = 1; 650 tag = (group << 5) + bit; 651 cmd = mtip_cmd_from_tag(dd, tag); 652 653 fis = (struct host_to_dev_fis *)cmd->command; 654 655 /* Should re-issue? */ 656 if (tag == MTIP_TAG_INTERNAL || 657 fis->command == ATA_CMD_SET_FEATURES) 658 reissue = 0; 659 else { 660 if (fail_all_ncq_cmds || 661 (fail_all_ncq_write && 662 fis->command == ATA_CMD_FPDMA_WRITE)) { 663 dev_warn(&dd->pdev->dev, 664 " Fail: %s w/tag %d [%s].\n", 665 fis->command == ATA_CMD_FPDMA_WRITE ? 666 "write" : "read", 667 tag, 668 fail_reason != NULL ? 669 fail_reason : "unknown"); 670 mtip_complete_command(cmd, BLK_STS_MEDIUM); 671 continue; 672 } 673 } 674 675 /* 676 * First check if this command has 677 * exceeded its retries. 678 */ 679 if (reissue && (cmd->retries-- > 0)) { 680 681 set_bit(tag, tagaccum); 682 683 /* Re-issue the command. */ 684 mtip_issue_ncq_command(port, tag); 685 686 continue; 687 } 688 689 /* Retire a command that will not be reissued */ 690 dev_warn(&port->dd->pdev->dev, 691 "retiring tag %d\n", tag); 692 693 mtip_complete_command(cmd, BLK_STS_IOERR); 694 } 695 } 696 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); 697 } 698 699 /* 700 * Handle a set device bits interrupt 701 */ 702 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, 703 u32 completed) 704 { 705 struct driver_data *dd = port->dd; 706 int tag, bit; 707 struct mtip_cmd *command; 708 709 if (!completed) { 710 WARN_ON_ONCE(!completed); 711 return; 712 } 713 /* clear completed status register in the hardware.*/ 714 writel(completed, port->completed[group]); 715 716 /* Process completed commands. */ 717 for (bit = 0; (bit < 32) && completed; bit++) { 718 if (completed & 0x01) { 719 tag = (group << 5) | bit; 720 721 /* skip internal command slot. */ 722 if (unlikely(tag == MTIP_TAG_INTERNAL)) 723 continue; 724 725 command = mtip_cmd_from_tag(dd, tag); 726 mtip_complete_command(command, 0); 727 } 728 completed >>= 1; 729 } 730 731 /* If last, re-enable interrupts */ 732 if (atomic_dec_return(&dd->irq_workers_active) == 0) 733 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); 734 } 735 736 /* 737 * Process legacy pio and d2h interrupts 738 */ 739 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) 740 { 741 struct mtip_port *port = dd->port; 742 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 743 744 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) { 745 int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL); 746 int status = readl(port->cmd_issue[group]); 747 748 if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL)))) 749 mtip_complete_command(cmd, 0); 750 } 751 } 752 753 /* 754 * Demux and handle errors 755 */ 756 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) 757 { 758 if (unlikely(port_stat & PORT_IRQ_CONNECT)) { 759 dev_warn(&dd->pdev->dev, 760 "Clearing PxSERR.DIAG.x\n"); 761 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); 762 } 763 764 if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { 765 dev_warn(&dd->pdev->dev, 766 "Clearing PxSERR.DIAG.n\n"); 767 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); 768 } 769 770 if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { 771 dev_warn(&dd->pdev->dev, 772 "Port stat errors %x unhandled\n", 773 (port_stat & ~PORT_IRQ_HANDLED)); 774 if (mtip_check_surprise_removal(dd->pdev)) 775 return; 776 } 777 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) { 778 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); 779 wake_up_interruptible(&dd->port->svc_wait); 780 } 781 } 782 783 static inline irqreturn_t mtip_handle_irq(struct driver_data *data) 784 { 785 struct driver_data *dd = (struct driver_data *) data; 786 struct mtip_port *port = dd->port; 787 u32 hba_stat, port_stat; 788 int rv = IRQ_NONE; 789 int do_irq_enable = 1, i, workers; 790 struct mtip_work *twork; 791 792 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 793 if (hba_stat) { 794 rv = IRQ_HANDLED; 795 796 /* Acknowledge the interrupt status on the port.*/ 797 port_stat = readl(port->mmio + PORT_IRQ_STAT); 798 if (unlikely(port_stat == 0xFFFFFFFF)) { 799 mtip_check_surprise_removal(dd->pdev); 800 return IRQ_HANDLED; 801 } 802 writel(port_stat, port->mmio + PORT_IRQ_STAT); 803 804 /* Demux port status */ 805 if (likely(port_stat & PORT_IRQ_SDB_FIS)) { 806 do_irq_enable = 0; 807 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); 808 809 /* Start at 1: group zero is always local? */ 810 for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS; 811 i++) { 812 twork = &dd->work[i]; 813 twork->completed = readl(port->completed[i]); 814 if (twork->completed) 815 workers++; 816 } 817 818 atomic_set(&dd->irq_workers_active, workers); 819 if (workers) { 820 for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) { 821 twork = &dd->work[i]; 822 if (twork->completed) 823 queue_work_on( 824 twork->cpu_binding, 825 dd->isr_workq, 826 &twork->work); 827 } 828 829 if (likely(dd->work[0].completed)) 830 mtip_workq_sdbfx(port, 0, 831 dd->work[0].completed); 832 833 } else { 834 /* 835 * Chip quirk: SDB interrupt but nothing 836 * to complete 837 */ 838 do_irq_enable = 1; 839 } 840 } 841 842 if (unlikely(port_stat & PORT_IRQ_ERR)) { 843 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 844 /* don't proceed further */ 845 return IRQ_HANDLED; 846 } 847 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 848 &dd->dd_flag)) 849 return rv; 850 851 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); 852 } 853 854 if (unlikely(port_stat & PORT_IRQ_LEGACY)) 855 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); 856 } 857 858 /* acknowledge interrupt */ 859 if (unlikely(do_irq_enable)) 860 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); 861 862 return rv; 863 } 864 865 /* 866 * HBA interrupt subroutine. 867 * 868 * @irq IRQ number. 869 * @instance Pointer to the driver data structure. 870 * 871 * return value 872 * IRQ_HANDLED A HBA interrupt was pending and handled. 873 * IRQ_NONE This interrupt was not for the HBA. 874 */ 875 static irqreturn_t mtip_irq_handler(int irq, void *instance) 876 { 877 struct driver_data *dd = instance; 878 879 return mtip_handle_irq(dd); 880 } 881 882 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) 883 { 884 writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]); 885 } 886 887 static bool mtip_pause_ncq(struct mtip_port *port, 888 struct host_to_dev_fis *fis) 889 { 890 unsigned long task_file_data; 891 892 task_file_data = readl(port->mmio+PORT_TFDATA); 893 if ((task_file_data & 1)) 894 return false; 895 896 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { 897 port->ic_pause_timer = jiffies; 898 return true; 899 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && 900 (fis->features == 0x03)) { 901 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 902 port->ic_pause_timer = jiffies; 903 return true; 904 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || 905 ((fis->command == 0xFC) && 906 (fis->features == 0x27 || fis->features == 0x72 || 907 fis->features == 0x62 || fis->features == 0x26))) { 908 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 909 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); 910 /* Com reset after secure erase or lowlevel format */ 911 mtip_restart_port(port); 912 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 913 return false; 914 } 915 916 return false; 917 } 918 919 static bool mtip_commands_active(struct mtip_port *port) 920 { 921 unsigned int active; 922 unsigned int n; 923 924 /* 925 * Ignore s_active bit 0 of array element 0. 926 * This bit will always be set 927 */ 928 active = readl(port->s_active[0]) & 0xFFFFFFFE; 929 for (n = 1; n < port->dd->slot_groups; n++) 930 active |= readl(port->s_active[n]); 931 932 return active != 0; 933 } 934 935 /* 936 * Wait for port to quiesce 937 * 938 * @port Pointer to port data structure 939 * @timeout Max duration to wait (ms) 940 * 941 * return value 942 * 0 Success 943 * -EBUSY Commands still active 944 */ 945 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 946 { 947 unsigned long to; 948 bool active = true; 949 950 blk_mq_quiesce_queue(port->dd->queue); 951 952 to = jiffies + msecs_to_jiffies(timeout); 953 do { 954 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && 955 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 956 msleep(20); 957 continue; /* svc thd is actively issuing commands */ 958 } 959 960 msleep(100); 961 962 if (mtip_check_surprise_removal(port->dd->pdev)) 963 goto err_fault; 964 965 active = mtip_commands_active(port); 966 if (!active) 967 break; 968 } while (time_before(jiffies, to)); 969 970 blk_mq_unquiesce_queue(port->dd->queue); 971 return active ? -EBUSY : 0; 972 err_fault: 973 blk_mq_unquiesce_queue(port->dd->queue); 974 return -EFAULT; 975 } 976 977 struct mtip_int_cmd { 978 int fis_len; 979 dma_addr_t buffer; 980 int buf_len; 981 u32 opts; 982 }; 983 984 /* 985 * Execute an internal command and wait for the completion. 986 * 987 * @port Pointer to the port data structure. 988 * @fis Pointer to the FIS that describes the command. 989 * @fis_len Length in WORDS of the FIS. 990 * @buffer DMA accessible for command data. 991 * @buf_len Length, in bytes, of the data buffer. 992 * @opts Command header options, excluding the FIS length 993 * and the number of PRD entries. 994 * @timeout Time in ms to wait for the command to complete. 995 * 996 * return value 997 * 0 Command completed successfully. 998 * -EFAULT The buffer address is not correctly aligned. 999 * -EBUSY Internal command or other IO in progress. 1000 * -EAGAIN Time out waiting for command to complete. 1001 */ 1002 static int mtip_exec_internal_command(struct mtip_port *port, 1003 struct host_to_dev_fis *fis, 1004 int fis_len, 1005 dma_addr_t buffer, 1006 int buf_len, 1007 u32 opts, 1008 unsigned long timeout) 1009 { 1010 struct mtip_cmd *int_cmd; 1011 struct driver_data *dd = port->dd; 1012 struct request *rq; 1013 struct mtip_int_cmd icmd = { 1014 .fis_len = fis_len, 1015 .buffer = buffer, 1016 .buf_len = buf_len, 1017 .opts = opts 1018 }; 1019 int rv = 0; 1020 1021 /* Make sure the buffer is 8 byte aligned. This is asic specific. */ 1022 if (buffer & 0x00000007) { 1023 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); 1024 return -EFAULT; 1025 } 1026 1027 int_cmd = mtip_get_int_command(dd); 1028 if (!int_cmd) { 1029 dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n"); 1030 return -EFAULT; 1031 } 1032 rq = blk_mq_rq_from_pdu(int_cmd); 1033 rq->special = &icmd; 1034 1035 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1036 1037 if (fis->command == ATA_CMD_SEC_ERASE_PREP) 1038 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1039 1040 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1041 1042 if (fis->command != ATA_CMD_STANDBYNOW1) { 1043 /* wait for io to complete if non atomic */ 1044 if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) { 1045 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); 1046 blk_mq_free_request(rq); 1047 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1048 wake_up_interruptible(&port->svc_wait); 1049 return -EBUSY; 1050 } 1051 } 1052 1053 /* Copy the command to the command table */ 1054 memcpy(int_cmd->command, fis, fis_len*4); 1055 1056 rq->timeout = timeout; 1057 1058 /* insert request and run queue */ 1059 blk_execute_rq(rq->q, NULL, rq, true); 1060 1061 if (int_cmd->status) { 1062 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", 1063 fis->command, int_cmd->status); 1064 rv = -EIO; 1065 1066 if (mtip_check_surprise_removal(dd->pdev) || 1067 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1068 &dd->dd_flag)) { 1069 dev_err(&dd->pdev->dev, 1070 "Internal command [%02X] wait returned due to SR\n", 1071 fis->command); 1072 rv = -ENXIO; 1073 goto exec_ic_exit; 1074 } 1075 mtip_device_reset(dd); /* recover from timeout issue */ 1076 rv = -EAGAIN; 1077 goto exec_ic_exit; 1078 } 1079 1080 if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)]) 1081 & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) { 1082 rv = -ENXIO; 1083 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 1084 mtip_device_reset(dd); 1085 rv = -EAGAIN; 1086 } 1087 } 1088 exec_ic_exit: 1089 /* Clear the allocated and active bits for the internal command. */ 1090 blk_mq_free_request(rq); 1091 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1092 if (rv >= 0 && mtip_pause_ncq(port, fis)) { 1093 /* NCQ paused */ 1094 return rv; 1095 } 1096 wake_up_interruptible(&port->svc_wait); 1097 1098 return rv; 1099 } 1100 1101 /* 1102 * Byte-swap ATA ID strings. 1103 * 1104 * ATA identify data contains strings in byte-swapped 16-bit words. 1105 * They must be swapped (on all architectures) to be usable as C strings. 1106 * This function swaps bytes in-place. 1107 * 1108 * @buf The buffer location of the string 1109 * @len The number of bytes to swap 1110 * 1111 * return value 1112 * None 1113 */ 1114 static inline void ata_swap_string(u16 *buf, unsigned int len) 1115 { 1116 int i; 1117 for (i = 0; i < (len/2); i++) 1118 be16_to_cpus(&buf[i]); 1119 } 1120 1121 static void mtip_set_timeout(struct driver_data *dd, 1122 struct host_to_dev_fis *fis, 1123 unsigned int *timeout, u8 erasemode) 1124 { 1125 switch (fis->command) { 1126 case ATA_CMD_DOWNLOAD_MICRO: 1127 *timeout = 120000; /* 2 minutes */ 1128 break; 1129 case ATA_CMD_SEC_ERASE_UNIT: 1130 case 0xFC: 1131 if (erasemode) 1132 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); 1133 else 1134 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); 1135 break; 1136 case ATA_CMD_STANDBYNOW1: 1137 *timeout = 120000; /* 2 minutes */ 1138 break; 1139 case 0xF7: 1140 case 0xFA: 1141 *timeout = 60000; /* 60 seconds */ 1142 break; 1143 case ATA_CMD_SMART: 1144 *timeout = 15000; /* 15 seconds */ 1145 break; 1146 default: 1147 *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS; 1148 break; 1149 } 1150 } 1151 1152 /* 1153 * Request the device identity information. 1154 * 1155 * If a user space buffer is not specified, i.e. is NULL, the 1156 * identify information is still read from the drive and placed 1157 * into the identify data buffer (@e port->identify) in the 1158 * port data structure. 1159 * When the identify buffer contains valid identify information @e 1160 * port->identify_valid is non-zero. 1161 * 1162 * @port Pointer to the port structure. 1163 * @user_buffer A user space buffer where the identify data should be 1164 * copied. 1165 * 1166 * return value 1167 * 0 Command completed successfully. 1168 * -EFAULT An error occurred while coping data to the user buffer. 1169 * -1 Command failed. 1170 */ 1171 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) 1172 { 1173 int rv = 0; 1174 struct host_to_dev_fis fis; 1175 1176 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1177 return -EFAULT; 1178 1179 /* Build the FIS. */ 1180 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1181 fis.type = 0x27; 1182 fis.opts = 1 << 7; 1183 fis.command = ATA_CMD_ID_ATA; 1184 1185 /* Set the identify information as invalid. */ 1186 port->identify_valid = 0; 1187 1188 /* Clear the identify information. */ 1189 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); 1190 1191 /* Execute the command. */ 1192 if (mtip_exec_internal_command(port, 1193 &fis, 1194 5, 1195 port->identify_dma, 1196 sizeof(u16) * ATA_ID_WORDS, 1197 0, 1198 MTIP_INT_CMD_TIMEOUT_MS) 1199 < 0) { 1200 rv = -1; 1201 goto out; 1202 } 1203 1204 /* 1205 * Perform any necessary byte-swapping. Yes, the kernel does in fact 1206 * perform field-sensitive swapping on the string fields. 1207 * See the kernel use of ata_id_string() for proof of this. 1208 */ 1209 #ifdef __LITTLE_ENDIAN 1210 ata_swap_string(port->identify + 27, 40); /* model string*/ 1211 ata_swap_string(port->identify + 23, 8); /* firmware string*/ 1212 ata_swap_string(port->identify + 10, 20); /* serial# string*/ 1213 #else 1214 { 1215 int i; 1216 for (i = 0; i < ATA_ID_WORDS; i++) 1217 port->identify[i] = le16_to_cpu(port->identify[i]); 1218 } 1219 #endif 1220 1221 /* Check security locked state */ 1222 if (port->identify[128] & 0x4) 1223 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1224 else 1225 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1226 1227 #ifdef MTIP_TRIM /* Disabling TRIM support temporarily */ 1228 /* Demux ID.DRAT & ID.RZAT to determine trim support */ 1229 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) 1230 port->dd->trim_supp = true; 1231 else 1232 #endif 1233 port->dd->trim_supp = false; 1234 1235 /* Set the identify buffer as valid. */ 1236 port->identify_valid = 1; 1237 1238 if (user_buffer) { 1239 if (copy_to_user( 1240 user_buffer, 1241 port->identify, 1242 ATA_ID_WORDS * sizeof(u16))) { 1243 rv = -EFAULT; 1244 goto out; 1245 } 1246 } 1247 1248 out: 1249 return rv; 1250 } 1251 1252 /* 1253 * Issue a standby immediate command to the device. 1254 * 1255 * @port Pointer to the port structure. 1256 * 1257 * return value 1258 * 0 Command was executed successfully. 1259 * -1 An error occurred while executing the command. 1260 */ 1261 static int mtip_standby_immediate(struct mtip_port *port) 1262 { 1263 int rv; 1264 struct host_to_dev_fis fis; 1265 unsigned long start; 1266 unsigned int timeout; 1267 1268 /* Build the FIS. */ 1269 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1270 fis.type = 0x27; 1271 fis.opts = 1 << 7; 1272 fis.command = ATA_CMD_STANDBYNOW1; 1273 1274 mtip_set_timeout(port->dd, &fis, &timeout, 0); 1275 1276 start = jiffies; 1277 rv = mtip_exec_internal_command(port, 1278 &fis, 1279 5, 1280 0, 1281 0, 1282 0, 1283 timeout); 1284 dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n", 1285 jiffies_to_msecs(jiffies - start)); 1286 if (rv) 1287 dev_warn(&port->dd->pdev->dev, 1288 "STANDBY IMMEDIATE command failed.\n"); 1289 1290 return rv; 1291 } 1292 1293 /* 1294 * Issue a READ LOG EXT command to the device. 1295 * 1296 * @port pointer to the port structure. 1297 * @page page number to fetch 1298 * @buffer pointer to buffer 1299 * @buffer_dma dma address corresponding to @buffer 1300 * @sectors page length to fetch, in sectors 1301 * 1302 * return value 1303 * @rv return value from mtip_exec_internal_command() 1304 */ 1305 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 1306 dma_addr_t buffer_dma, unsigned int sectors) 1307 { 1308 struct host_to_dev_fis fis; 1309 1310 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1311 fis.type = 0x27; 1312 fis.opts = 1 << 7; 1313 fis.command = ATA_CMD_READ_LOG_EXT; 1314 fis.sect_count = sectors & 0xFF; 1315 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; 1316 fis.lba_low = page; 1317 fis.lba_mid = 0; 1318 fis.device = ATA_DEVICE_OBS; 1319 1320 memset(buffer, 0, sectors * ATA_SECT_SIZE); 1321 1322 return mtip_exec_internal_command(port, 1323 &fis, 1324 5, 1325 buffer_dma, 1326 sectors * ATA_SECT_SIZE, 1327 0, 1328 MTIP_INT_CMD_TIMEOUT_MS); 1329 } 1330 1331 /* 1332 * Issue a SMART READ DATA command to the device. 1333 * 1334 * @port pointer to the port structure. 1335 * @buffer pointer to buffer 1336 * @buffer_dma dma address corresponding to @buffer 1337 * 1338 * return value 1339 * @rv return value from mtip_exec_internal_command() 1340 */ 1341 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, 1342 dma_addr_t buffer_dma) 1343 { 1344 struct host_to_dev_fis fis; 1345 1346 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1347 fis.type = 0x27; 1348 fis.opts = 1 << 7; 1349 fis.command = ATA_CMD_SMART; 1350 fis.features = 0xD0; 1351 fis.sect_count = 1; 1352 fis.lba_mid = 0x4F; 1353 fis.lba_hi = 0xC2; 1354 fis.device = ATA_DEVICE_OBS; 1355 1356 return mtip_exec_internal_command(port, 1357 &fis, 1358 5, 1359 buffer_dma, 1360 ATA_SECT_SIZE, 1361 0, 1362 15000); 1363 } 1364 1365 /* 1366 * Get the value of a smart attribute 1367 * 1368 * @port pointer to the port structure 1369 * @id attribute number 1370 * @attrib pointer to return attrib information corresponding to @id 1371 * 1372 * return value 1373 * -EINVAL NULL buffer passed or unsupported attribute @id. 1374 * -EPERM Identify data not valid, SMART not supported or not enabled 1375 */ 1376 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 1377 struct smart_attr *attrib) 1378 { 1379 int rv, i; 1380 struct smart_attr *pattr; 1381 1382 if (!attrib) 1383 return -EINVAL; 1384 1385 if (!port->identify_valid) { 1386 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); 1387 return -EPERM; 1388 } 1389 if (!(port->identify[82] & 0x1)) { 1390 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); 1391 return -EPERM; 1392 } 1393 if (!(port->identify[85] & 0x1)) { 1394 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); 1395 return -EPERM; 1396 } 1397 1398 memset(port->smart_buf, 0, ATA_SECT_SIZE); 1399 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); 1400 if (rv) { 1401 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); 1402 return rv; 1403 } 1404 1405 pattr = (struct smart_attr *)(port->smart_buf + 2); 1406 for (i = 0; i < 29; i++, pattr++) 1407 if (pattr->attr_id == id) { 1408 memcpy(attrib, pattr, sizeof(struct smart_attr)); 1409 break; 1410 } 1411 1412 if (i == 29) { 1413 dev_warn(&port->dd->pdev->dev, 1414 "Query for invalid SMART attribute ID\n"); 1415 rv = -EINVAL; 1416 } 1417 1418 return rv; 1419 } 1420 1421 /* 1422 * Trim unused sectors 1423 * 1424 * @dd pointer to driver_data structure 1425 * @lba starting lba 1426 * @len # of 512b sectors to trim 1427 * 1428 * return value 1429 * -ENOMEM Out of dma memory 1430 * -EINVAL Invalid parameters passed in, trim not supported 1431 * -EIO Error submitting trim request to hw 1432 */ 1433 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, 1434 unsigned int len) 1435 { 1436 int i, rv = 0; 1437 u64 tlba, tlen, sect_left; 1438 struct mtip_trim_entry *buf; 1439 dma_addr_t dma_addr; 1440 struct host_to_dev_fis fis; 1441 1442 if (!len || dd->trim_supp == false) 1443 return -EINVAL; 1444 1445 /* Trim request too big */ 1446 WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES)); 1447 1448 /* Trim request not aligned on 4k boundary */ 1449 WARN_ON(len % 8 != 0); 1450 1451 /* Warn if vu_trim structure is too big */ 1452 WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE); 1453 1454 /* Allocate a DMA buffer for the trim structure */ 1455 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, 1456 GFP_KERNEL); 1457 if (!buf) 1458 return -ENOMEM; 1459 memset(buf, 0, ATA_SECT_SIZE); 1460 1461 for (i = 0, sect_left = len, tlba = lba; 1462 i < MTIP_MAX_TRIM_ENTRIES && sect_left; 1463 i++) { 1464 tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ? 1465 MTIP_MAX_TRIM_ENTRY_LEN : 1466 sect_left); 1467 buf[i].lba = __force_bit2int cpu_to_le32(tlba); 1468 buf[i].range = __force_bit2int cpu_to_le16(tlen); 1469 tlba += tlen; 1470 sect_left -= tlen; 1471 } 1472 WARN_ON(sect_left != 0); 1473 1474 /* Build the fis */ 1475 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1476 fis.type = 0x27; 1477 fis.opts = 1 << 7; 1478 fis.command = 0xfb; 1479 fis.features = 0x60; 1480 fis.sect_count = 1; 1481 fis.device = ATA_DEVICE_OBS; 1482 1483 if (mtip_exec_internal_command(dd->port, 1484 &fis, 1485 5, 1486 dma_addr, 1487 ATA_SECT_SIZE, 1488 0, 1489 MTIP_TRIM_TIMEOUT_MS) < 0) 1490 rv = -EIO; 1491 1492 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); 1493 return rv; 1494 } 1495 1496 /* 1497 * Get the drive capacity. 1498 * 1499 * @dd Pointer to the device data structure. 1500 * @sectors Pointer to the variable that will receive the sector count. 1501 * 1502 * return value 1503 * 1 Capacity was returned successfully. 1504 * 0 The identify information is invalid. 1505 */ 1506 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) 1507 { 1508 struct mtip_port *port = dd->port; 1509 u64 total, raw0, raw1, raw2, raw3; 1510 raw0 = port->identify[100]; 1511 raw1 = port->identify[101]; 1512 raw2 = port->identify[102]; 1513 raw3 = port->identify[103]; 1514 total = raw0 | raw1<<16 | raw2<<32 | raw3<<48; 1515 *sectors = total; 1516 return (bool) !!port->identify_valid; 1517 } 1518 1519 /* 1520 * Display the identify command data. 1521 * 1522 * @port Pointer to the port data structure. 1523 * 1524 * return value 1525 * None 1526 */ 1527 static void mtip_dump_identify(struct mtip_port *port) 1528 { 1529 sector_t sectors; 1530 unsigned short revid; 1531 char cbuf[42]; 1532 1533 if (!port->identify_valid) 1534 return; 1535 1536 strlcpy(cbuf, (char *)(port->identify+10), 21); 1537 dev_info(&port->dd->pdev->dev, 1538 "Serial No.: %s\n", cbuf); 1539 1540 strlcpy(cbuf, (char *)(port->identify+23), 9); 1541 dev_info(&port->dd->pdev->dev, 1542 "Firmware Ver.: %s\n", cbuf); 1543 1544 strlcpy(cbuf, (char *)(port->identify+27), 41); 1545 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); 1546 1547 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", 1548 port->identify[128], 1549 port->identify[128] & 0x4 ? "(LOCKED)" : ""); 1550 1551 if (mtip_hw_get_capacity(port->dd, §ors)) 1552 dev_info(&port->dd->pdev->dev, 1553 "Capacity: %llu sectors (%llu MB)\n", 1554 (u64)sectors, 1555 ((u64)sectors) * ATA_SECT_SIZE >> 20); 1556 1557 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); 1558 switch (revid & 0xFF) { 1559 case 0x1: 1560 strlcpy(cbuf, "A0", 3); 1561 break; 1562 case 0x3: 1563 strlcpy(cbuf, "A2", 3); 1564 break; 1565 default: 1566 strlcpy(cbuf, "?", 2); 1567 break; 1568 } 1569 dev_info(&port->dd->pdev->dev, 1570 "Card Type: %s\n", cbuf); 1571 } 1572 1573 /* 1574 * Map the commands scatter list into the command table. 1575 * 1576 * @command Pointer to the command. 1577 * @nents Number of scatter list entries. 1578 * 1579 * return value 1580 * None 1581 */ 1582 static inline void fill_command_sg(struct driver_data *dd, 1583 struct mtip_cmd *command, 1584 int nents) 1585 { 1586 int n; 1587 unsigned int dma_len; 1588 struct mtip_cmd_sg *command_sg; 1589 struct scatterlist *sg = command->sg; 1590 1591 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; 1592 1593 for (n = 0; n < nents; n++) { 1594 dma_len = sg_dma_len(sg); 1595 if (dma_len > 0x400000) 1596 dev_err(&dd->pdev->dev, 1597 "DMA segment length truncated\n"); 1598 command_sg->info = __force_bit2int 1599 cpu_to_le32((dma_len-1) & 0x3FFFFF); 1600 command_sg->dba = __force_bit2int 1601 cpu_to_le32(sg_dma_address(sg)); 1602 command_sg->dba_upper = __force_bit2int 1603 cpu_to_le32((sg_dma_address(sg) >> 16) >> 16); 1604 command_sg++; 1605 sg++; 1606 } 1607 } 1608 1609 /* 1610 * @brief Execute a drive command. 1611 * 1612 * return value 0 The command completed successfully. 1613 * return value -1 An error occurred while executing the command. 1614 */ 1615 static int exec_drive_task(struct mtip_port *port, u8 *command) 1616 { 1617 struct host_to_dev_fis fis; 1618 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); 1619 unsigned int to; 1620 1621 /* Build the FIS. */ 1622 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1623 fis.type = 0x27; 1624 fis.opts = 1 << 7; 1625 fis.command = command[0]; 1626 fis.features = command[1]; 1627 fis.sect_count = command[2]; 1628 fis.sector = command[3]; 1629 fis.cyl_low = command[4]; 1630 fis.cyl_hi = command[5]; 1631 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ 1632 1633 mtip_set_timeout(port->dd, &fis, &to, 0); 1634 1635 dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n", 1636 __func__, 1637 command[0], 1638 command[1], 1639 command[2], 1640 command[3], 1641 command[4], 1642 command[5], 1643 command[6]); 1644 1645 /* Execute the command. */ 1646 if (mtip_exec_internal_command(port, 1647 &fis, 1648 5, 1649 0, 1650 0, 1651 0, 1652 to) < 0) { 1653 return -1; 1654 } 1655 1656 command[0] = reply->command; /* Status*/ 1657 command[1] = reply->features; /* Error*/ 1658 command[4] = reply->cyl_low; 1659 command[5] = reply->cyl_hi; 1660 1661 dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n", 1662 __func__, 1663 command[0], 1664 command[1], 1665 command[4], 1666 command[5]); 1667 1668 return 0; 1669 } 1670 1671 /* 1672 * @brief Execute a drive command. 1673 * 1674 * @param port Pointer to the port data structure. 1675 * @param command Pointer to the user specified command parameters. 1676 * @param user_buffer Pointer to the user space buffer where read sector 1677 * data should be copied. 1678 * 1679 * return value 0 The command completed successfully. 1680 * return value -EFAULT An error occurred while copying the completion 1681 * data to the user space buffer. 1682 * return value -1 An error occurred while executing the command. 1683 */ 1684 static int exec_drive_command(struct mtip_port *port, u8 *command, 1685 void __user *user_buffer) 1686 { 1687 struct host_to_dev_fis fis; 1688 struct host_to_dev_fis *reply; 1689 u8 *buf = NULL; 1690 dma_addr_t dma_addr = 0; 1691 int rv = 0, xfer_sz = command[3]; 1692 unsigned int to; 1693 1694 if (xfer_sz) { 1695 if (!user_buffer) 1696 return -EFAULT; 1697 1698 buf = dmam_alloc_coherent(&port->dd->pdev->dev, 1699 ATA_SECT_SIZE * xfer_sz, 1700 &dma_addr, 1701 GFP_KERNEL); 1702 if (!buf) { 1703 dev_err(&port->dd->pdev->dev, 1704 "Memory allocation failed (%d bytes)\n", 1705 ATA_SECT_SIZE * xfer_sz); 1706 return -ENOMEM; 1707 } 1708 memset(buf, 0, ATA_SECT_SIZE * xfer_sz); 1709 } 1710 1711 /* Build the FIS. */ 1712 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1713 fis.type = 0x27; 1714 fis.opts = 1 << 7; 1715 fis.command = command[0]; 1716 fis.features = command[2]; 1717 fis.sect_count = command[3]; 1718 if (fis.command == ATA_CMD_SMART) { 1719 fis.sector = command[1]; 1720 fis.cyl_low = 0x4F; 1721 fis.cyl_hi = 0xC2; 1722 } 1723 1724 mtip_set_timeout(port->dd, &fis, &to, 0); 1725 1726 if (xfer_sz) 1727 reply = (port->rxfis + RX_FIS_PIO_SETUP); 1728 else 1729 reply = (port->rxfis + RX_FIS_D2H_REG); 1730 1731 dbg_printk(MTIP_DRV_NAME 1732 " %s: User Command: cmd %x, sect %x, " 1733 "feat %x, sectcnt %x\n", 1734 __func__, 1735 command[0], 1736 command[1], 1737 command[2], 1738 command[3]); 1739 1740 /* Execute the command. */ 1741 if (mtip_exec_internal_command(port, 1742 &fis, 1743 5, 1744 (xfer_sz ? dma_addr : 0), 1745 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0), 1746 0, 1747 to) 1748 < 0) { 1749 rv = -EFAULT; 1750 goto exit_drive_command; 1751 } 1752 1753 /* Collect the completion status. */ 1754 command[0] = reply->command; /* Status*/ 1755 command[1] = reply->features; /* Error*/ 1756 command[2] = reply->sect_count; 1757 1758 dbg_printk(MTIP_DRV_NAME 1759 " %s: Completion Status: stat %x, " 1760 "err %x, nsect %x\n", 1761 __func__, 1762 command[0], 1763 command[1], 1764 command[2]); 1765 1766 if (xfer_sz) { 1767 if (copy_to_user(user_buffer, 1768 buf, 1769 ATA_SECT_SIZE * command[3])) { 1770 rv = -EFAULT; 1771 goto exit_drive_command; 1772 } 1773 } 1774 exit_drive_command: 1775 if (buf) 1776 dmam_free_coherent(&port->dd->pdev->dev, 1777 ATA_SECT_SIZE * xfer_sz, buf, dma_addr); 1778 return rv; 1779 } 1780 1781 /* 1782 * Indicates whether a command has a single sector payload. 1783 * 1784 * @command passed to the device to perform the certain event. 1785 * @features passed to the device to perform the certain event. 1786 * 1787 * return value 1788 * 1 command is one that always has a single sector payload, 1789 * regardless of the value in the Sector Count field. 1790 * 0 otherwise 1791 * 1792 */ 1793 static unsigned int implicit_sector(unsigned char command, 1794 unsigned char features) 1795 { 1796 unsigned int rv = 0; 1797 1798 /* list of commands that have an implicit sector count of 1 */ 1799 switch (command) { 1800 case ATA_CMD_SEC_SET_PASS: 1801 case ATA_CMD_SEC_UNLOCK: 1802 case ATA_CMD_SEC_ERASE_PREP: 1803 case ATA_CMD_SEC_ERASE_UNIT: 1804 case ATA_CMD_SEC_FREEZE_LOCK: 1805 case ATA_CMD_SEC_DISABLE_PASS: 1806 case ATA_CMD_PMP_READ: 1807 case ATA_CMD_PMP_WRITE: 1808 rv = 1; 1809 break; 1810 case ATA_CMD_SET_MAX: 1811 if (features == ATA_SET_MAX_UNLOCK) 1812 rv = 1; 1813 break; 1814 case ATA_CMD_SMART: 1815 if ((features == ATA_SMART_READ_VALUES) || 1816 (features == ATA_SMART_READ_THRESHOLDS)) 1817 rv = 1; 1818 break; 1819 case ATA_CMD_CONF_OVERLAY: 1820 if ((features == ATA_DCO_IDENTIFY) || 1821 (features == ATA_DCO_SET)) 1822 rv = 1; 1823 break; 1824 } 1825 return rv; 1826 } 1827 1828 /* 1829 * Executes a taskfile 1830 * See ide_taskfile_ioctl() for derivation 1831 */ 1832 static int exec_drive_taskfile(struct driver_data *dd, 1833 void __user *buf, 1834 ide_task_request_t *req_task, 1835 int outtotal) 1836 { 1837 struct host_to_dev_fis fis; 1838 struct host_to_dev_fis *reply; 1839 u8 *outbuf = NULL; 1840 u8 *inbuf = NULL; 1841 dma_addr_t outbuf_dma = 0; 1842 dma_addr_t inbuf_dma = 0; 1843 dma_addr_t dma_buffer = 0; 1844 int err = 0; 1845 unsigned int taskin = 0; 1846 unsigned int taskout = 0; 1847 u8 nsect = 0; 1848 unsigned int timeout; 1849 unsigned int force_single_sector; 1850 unsigned int transfer_size; 1851 unsigned long task_file_data; 1852 int intotal = outtotal + req_task->out_size; 1853 int erasemode = 0; 1854 1855 taskout = req_task->out_size; 1856 taskin = req_task->in_size; 1857 /* 130560 = 512 * 0xFF*/ 1858 if (taskin > 130560 || taskout > 130560) 1859 return -EINVAL; 1860 1861 if (taskout) { 1862 outbuf = memdup_user(buf + outtotal, taskout); 1863 if (IS_ERR(outbuf)) 1864 return PTR_ERR(outbuf); 1865 1866 outbuf_dma = pci_map_single(dd->pdev, 1867 outbuf, 1868 taskout, 1869 DMA_TO_DEVICE); 1870 if (pci_dma_mapping_error(dd->pdev, outbuf_dma)) { 1871 err = -ENOMEM; 1872 goto abort; 1873 } 1874 dma_buffer = outbuf_dma; 1875 } 1876 1877 if (taskin) { 1878 inbuf = memdup_user(buf + intotal, taskin); 1879 if (IS_ERR(inbuf)) { 1880 err = PTR_ERR(inbuf); 1881 inbuf = NULL; 1882 goto abort; 1883 } 1884 inbuf_dma = pci_map_single(dd->pdev, 1885 inbuf, 1886 taskin, DMA_FROM_DEVICE); 1887 if (pci_dma_mapping_error(dd->pdev, inbuf_dma)) { 1888 err = -ENOMEM; 1889 goto abort; 1890 } 1891 dma_buffer = inbuf_dma; 1892 } 1893 1894 /* only supports PIO and non-data commands from this ioctl. */ 1895 switch (req_task->data_phase) { 1896 case TASKFILE_OUT: 1897 nsect = taskout / ATA_SECT_SIZE; 1898 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 1899 break; 1900 case TASKFILE_IN: 1901 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 1902 break; 1903 case TASKFILE_NO_DATA: 1904 reply = (dd->port->rxfis + RX_FIS_D2H_REG); 1905 break; 1906 default: 1907 err = -EINVAL; 1908 goto abort; 1909 } 1910 1911 /* Build the FIS. */ 1912 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1913 1914 fis.type = 0x27; 1915 fis.opts = 1 << 7; 1916 fis.command = req_task->io_ports[7]; 1917 fis.features = req_task->io_ports[1]; 1918 fis.sect_count = req_task->io_ports[2]; 1919 fis.lba_low = req_task->io_ports[3]; 1920 fis.lba_mid = req_task->io_ports[4]; 1921 fis.lba_hi = req_task->io_ports[5]; 1922 /* Clear the dev bit*/ 1923 fis.device = req_task->io_ports[6] & ~0x10; 1924 1925 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { 1926 req_task->in_flags.all = 1927 IDE_TASKFILE_STD_IN_FLAGS | 1928 (IDE_HOB_STD_IN_FLAGS << 8); 1929 fis.lba_low_ex = req_task->hob_ports[3]; 1930 fis.lba_mid_ex = req_task->hob_ports[4]; 1931 fis.lba_hi_ex = req_task->hob_ports[5]; 1932 fis.features_ex = req_task->hob_ports[1]; 1933 fis.sect_cnt_ex = req_task->hob_ports[2]; 1934 1935 } else { 1936 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; 1937 } 1938 1939 force_single_sector = implicit_sector(fis.command, fis.features); 1940 1941 if ((taskin || taskout) && (!fis.sect_count)) { 1942 if (nsect) 1943 fis.sect_count = nsect; 1944 else { 1945 if (!force_single_sector) { 1946 dev_warn(&dd->pdev->dev, 1947 "data movement but " 1948 "sect_count is 0\n"); 1949 err = -EINVAL; 1950 goto abort; 1951 } 1952 } 1953 } 1954 1955 dbg_printk(MTIP_DRV_NAME 1956 " %s: cmd %x, feat %x, nsect %x," 1957 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x," 1958 " head/dev %x\n", 1959 __func__, 1960 fis.command, 1961 fis.features, 1962 fis.sect_count, 1963 fis.lba_low, 1964 fis.lba_mid, 1965 fis.lba_hi, 1966 fis.device); 1967 1968 /* check for erase mode support during secure erase.*/ 1969 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && 1970 (outbuf[0] & MTIP_SEC_ERASE_MODE)) { 1971 erasemode = 1; 1972 } 1973 1974 mtip_set_timeout(dd, &fis, &timeout, erasemode); 1975 1976 /* Determine the correct transfer size.*/ 1977 if (force_single_sector) 1978 transfer_size = ATA_SECT_SIZE; 1979 else 1980 transfer_size = ATA_SECT_SIZE * fis.sect_count; 1981 1982 /* Execute the command.*/ 1983 if (mtip_exec_internal_command(dd->port, 1984 &fis, 1985 5, 1986 dma_buffer, 1987 transfer_size, 1988 0, 1989 timeout) < 0) { 1990 err = -EIO; 1991 goto abort; 1992 } 1993 1994 task_file_data = readl(dd->port->mmio+PORT_TFDATA); 1995 1996 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { 1997 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; 1998 req_task->io_ports[7] = reply->control; 1999 } else { 2000 reply = dd->port->rxfis + RX_FIS_D2H_REG; 2001 req_task->io_ports[7] = reply->command; 2002 } 2003 2004 /* reclaim the DMA buffers.*/ 2005 if (inbuf_dma) 2006 pci_unmap_single(dd->pdev, inbuf_dma, 2007 taskin, DMA_FROM_DEVICE); 2008 if (outbuf_dma) 2009 pci_unmap_single(dd->pdev, outbuf_dma, 2010 taskout, DMA_TO_DEVICE); 2011 inbuf_dma = 0; 2012 outbuf_dma = 0; 2013 2014 /* return the ATA registers to the caller.*/ 2015 req_task->io_ports[1] = reply->features; 2016 req_task->io_ports[2] = reply->sect_count; 2017 req_task->io_ports[3] = reply->lba_low; 2018 req_task->io_ports[4] = reply->lba_mid; 2019 req_task->io_ports[5] = reply->lba_hi; 2020 req_task->io_ports[6] = reply->device; 2021 2022 if (req_task->out_flags.all & 1) { 2023 2024 req_task->hob_ports[3] = reply->lba_low_ex; 2025 req_task->hob_ports[4] = reply->lba_mid_ex; 2026 req_task->hob_ports[5] = reply->lba_hi_ex; 2027 req_task->hob_ports[1] = reply->features_ex; 2028 req_task->hob_ports[2] = reply->sect_cnt_ex; 2029 } 2030 dbg_printk(MTIP_DRV_NAME 2031 " %s: Completion: stat %x," 2032 "err %x, sect_cnt %x, lbalo %x," 2033 "lbamid %x, lbahi %x, dev %x\n", 2034 __func__, 2035 req_task->io_ports[7], 2036 req_task->io_ports[1], 2037 req_task->io_ports[2], 2038 req_task->io_ports[3], 2039 req_task->io_ports[4], 2040 req_task->io_ports[5], 2041 req_task->io_ports[6]); 2042 2043 if (taskout) { 2044 if (copy_to_user(buf + outtotal, outbuf, taskout)) { 2045 err = -EFAULT; 2046 goto abort; 2047 } 2048 } 2049 if (taskin) { 2050 if (copy_to_user(buf + intotal, inbuf, taskin)) { 2051 err = -EFAULT; 2052 goto abort; 2053 } 2054 } 2055 abort: 2056 if (inbuf_dma) 2057 pci_unmap_single(dd->pdev, inbuf_dma, 2058 taskin, DMA_FROM_DEVICE); 2059 if (outbuf_dma) 2060 pci_unmap_single(dd->pdev, outbuf_dma, 2061 taskout, DMA_TO_DEVICE); 2062 kfree(outbuf); 2063 kfree(inbuf); 2064 2065 return err; 2066 } 2067 2068 /* 2069 * Handle IOCTL calls from the Block Layer. 2070 * 2071 * This function is called by the Block Layer when it receives an IOCTL 2072 * command that it does not understand. If the IOCTL command is not supported 2073 * this function returns -ENOTTY. 2074 * 2075 * @dd Pointer to the driver data structure. 2076 * @cmd IOCTL command passed from the Block Layer. 2077 * @arg IOCTL argument passed from the Block Layer. 2078 * 2079 * return value 2080 * 0 The IOCTL completed successfully. 2081 * -ENOTTY The specified command is not supported. 2082 * -EFAULT An error occurred copying data to a user space buffer. 2083 * -EIO An error occurred while executing the command. 2084 */ 2085 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, 2086 unsigned long arg) 2087 { 2088 switch (cmd) { 2089 case HDIO_GET_IDENTITY: 2090 { 2091 if (copy_to_user((void __user *)arg, dd->port->identify, 2092 sizeof(u16) * ATA_ID_WORDS)) 2093 return -EFAULT; 2094 break; 2095 } 2096 case HDIO_DRIVE_CMD: 2097 { 2098 u8 drive_command[4]; 2099 2100 /* Copy the user command info to our buffer. */ 2101 if (copy_from_user(drive_command, 2102 (void __user *) arg, 2103 sizeof(drive_command))) 2104 return -EFAULT; 2105 2106 /* Execute the drive command. */ 2107 if (exec_drive_command(dd->port, 2108 drive_command, 2109 (void __user *) (arg+4))) 2110 return -EIO; 2111 2112 /* Copy the status back to the users buffer. */ 2113 if (copy_to_user((void __user *) arg, 2114 drive_command, 2115 sizeof(drive_command))) 2116 return -EFAULT; 2117 2118 break; 2119 } 2120 case HDIO_DRIVE_TASK: 2121 { 2122 u8 drive_command[7]; 2123 2124 /* Copy the user command info to our buffer. */ 2125 if (copy_from_user(drive_command, 2126 (void __user *) arg, 2127 sizeof(drive_command))) 2128 return -EFAULT; 2129 2130 /* Execute the drive command. */ 2131 if (exec_drive_task(dd->port, drive_command)) 2132 return -EIO; 2133 2134 /* Copy the status back to the users buffer. */ 2135 if (copy_to_user((void __user *) arg, 2136 drive_command, 2137 sizeof(drive_command))) 2138 return -EFAULT; 2139 2140 break; 2141 } 2142 case HDIO_DRIVE_TASKFILE: { 2143 ide_task_request_t req_task; 2144 int ret, outtotal; 2145 2146 if (copy_from_user(&req_task, (void __user *) arg, 2147 sizeof(req_task))) 2148 return -EFAULT; 2149 2150 outtotal = sizeof(req_task); 2151 2152 ret = exec_drive_taskfile(dd, (void __user *) arg, 2153 &req_task, outtotal); 2154 2155 if (copy_to_user((void __user *) arg, &req_task, 2156 sizeof(req_task))) 2157 return -EFAULT; 2158 2159 return ret; 2160 } 2161 2162 default: 2163 return -EINVAL; 2164 } 2165 return 0; 2166 } 2167 2168 /* 2169 * Submit an IO to the hw 2170 * 2171 * This function is called by the block layer to issue an io 2172 * to the device. Upon completion, the callback function will 2173 * be called with the data parameter passed as the callback data. 2174 * 2175 * @dd Pointer to the driver data structure. 2176 * @start First sector to read. 2177 * @nsect Number of sectors to read. 2178 * @nents Number of entries in scatter list for the read command. 2179 * @tag The tag of this read command. 2180 * @callback Pointer to the function that should be called 2181 * when the read completes. 2182 * @data Callback data passed to the callback function 2183 * when the read completes. 2184 * @dir Direction (read or write) 2185 * 2186 * return value 2187 * None 2188 */ 2189 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2190 struct mtip_cmd *command, int nents, 2191 struct blk_mq_hw_ctx *hctx) 2192 { 2193 struct host_to_dev_fis *fis; 2194 struct mtip_port *port = dd->port; 2195 int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2196 u64 start = blk_rq_pos(rq); 2197 unsigned int nsect = blk_rq_sectors(rq); 2198 2199 /* Map the scatter list for DMA access */ 2200 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2201 2202 prefetch(&port->flags); 2203 2204 command->scatter_ents = nents; 2205 2206 /* 2207 * The number of retries for this command before it is 2208 * reported as a failure to the upper layers. 2209 */ 2210 command->retries = MTIP_MAX_RETRIES; 2211 2212 /* Fill out fis */ 2213 fis = command->command; 2214 fis->type = 0x27; 2215 fis->opts = 1 << 7; 2216 if (dma_dir == DMA_FROM_DEVICE) 2217 fis->command = ATA_CMD_FPDMA_READ; 2218 else 2219 fis->command = ATA_CMD_FPDMA_WRITE; 2220 fis->lba_low = start & 0xFF; 2221 fis->lba_mid = (start >> 8) & 0xFF; 2222 fis->lba_hi = (start >> 16) & 0xFF; 2223 fis->lba_low_ex = (start >> 24) & 0xFF; 2224 fis->lba_mid_ex = (start >> 32) & 0xFF; 2225 fis->lba_hi_ex = (start >> 40) & 0xFF; 2226 fis->device = 1 << 6; 2227 fis->features = nsect & 0xFF; 2228 fis->features_ex = (nsect >> 8) & 0xFF; 2229 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); 2230 fis->sect_cnt_ex = 0; 2231 fis->control = 0; 2232 fis->res2 = 0; 2233 fis->res3 = 0; 2234 fill_command_sg(dd, command, nents); 2235 2236 if (unlikely(command->unaligned)) 2237 fis->device |= 1 << 7; 2238 2239 /* Populate the command header */ 2240 command->command_header->opts = 2241 __force_bit2int cpu_to_le32( 2242 (nents << 16) | 5 | AHCI_CMD_PREFETCH); 2243 command->command_header->byte_count = 0; 2244 2245 command->direction = dma_dir; 2246 2247 /* 2248 * To prevent this command from being issued 2249 * if an internal command is in progress or error handling is active. 2250 */ 2251 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { 2252 set_bit(rq->tag, port->cmds_to_issue); 2253 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2254 return; 2255 } 2256 2257 /* Issue the command to the hardware */ 2258 mtip_issue_ncq_command(port, rq->tag); 2259 } 2260 2261 /* 2262 * Sysfs status dump. 2263 * 2264 * @dev Pointer to the device structure, passed by the kernrel. 2265 * @attr Pointer to the device_attribute structure passed by the kernel. 2266 * @buf Pointer to the char buffer that will receive the stats info. 2267 * 2268 * return value 2269 * The size, in bytes, of the data copied into buf. 2270 */ 2271 static ssize_t mtip_hw_show_status(struct device *dev, 2272 struct device_attribute *attr, 2273 char *buf) 2274 { 2275 struct driver_data *dd = dev_to_disk(dev)->private_data; 2276 int size = 0; 2277 2278 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) 2279 size += sprintf(buf, "%s", "thermal_shutdown\n"); 2280 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) 2281 size += sprintf(buf, "%s", "write_protect\n"); 2282 else 2283 size += sprintf(buf, "%s", "online\n"); 2284 2285 return size; 2286 } 2287 2288 static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL); 2289 2290 /* debugsfs entries */ 2291 2292 static ssize_t show_device_status(struct device_driver *drv, char *buf) 2293 { 2294 int size = 0; 2295 struct driver_data *dd, *tmp; 2296 unsigned long flags; 2297 char id_buf[42]; 2298 u16 status = 0; 2299 2300 spin_lock_irqsave(&dev_lock, flags); 2301 size += sprintf(&buf[size], "Devices Present:\n"); 2302 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { 2303 if (dd->pdev) { 2304 if (dd->port && 2305 dd->port->identify && 2306 dd->port->identify_valid) { 2307 strlcpy(id_buf, 2308 (char *) (dd->port->identify + 10), 21); 2309 status = *(dd->port->identify + 141); 2310 } else { 2311 memset(id_buf, 0, 42); 2312 status = 0; 2313 } 2314 2315 if (dd->port && 2316 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2317 size += sprintf(&buf[size], 2318 " device %s %s (ftl rebuild %d %%)\n", 2319 dev_name(&dd->pdev->dev), 2320 id_buf, 2321 status); 2322 } else { 2323 size += sprintf(&buf[size], 2324 " device %s %s\n", 2325 dev_name(&dd->pdev->dev), 2326 id_buf); 2327 } 2328 } 2329 } 2330 2331 size += sprintf(&buf[size], "Devices Being Removed:\n"); 2332 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { 2333 if (dd->pdev) { 2334 if (dd->port && 2335 dd->port->identify && 2336 dd->port->identify_valid) { 2337 strlcpy(id_buf, 2338 (char *) (dd->port->identify+10), 21); 2339 status = *(dd->port->identify + 141); 2340 } else { 2341 memset(id_buf, 0, 42); 2342 status = 0; 2343 } 2344 2345 if (dd->port && 2346 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2347 size += sprintf(&buf[size], 2348 " device %s %s (ftl rebuild %d %%)\n", 2349 dev_name(&dd->pdev->dev), 2350 id_buf, 2351 status); 2352 } else { 2353 size += sprintf(&buf[size], 2354 " device %s %s\n", 2355 dev_name(&dd->pdev->dev), 2356 id_buf); 2357 } 2358 } 2359 } 2360 spin_unlock_irqrestore(&dev_lock, flags); 2361 2362 return size; 2363 } 2364 2365 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf, 2366 size_t len, loff_t *offset) 2367 { 2368 struct driver_data *dd = (struct driver_data *)f->private_data; 2369 int size = *offset; 2370 char *buf; 2371 int rv = 0; 2372 2373 if (!len || *offset) 2374 return 0; 2375 2376 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2377 if (!buf) { 2378 dev_err(&dd->pdev->dev, 2379 "Memory allocation: status buffer\n"); 2380 return -ENOMEM; 2381 } 2382 2383 size += show_device_status(NULL, buf); 2384 2385 *offset = size <= len ? size : len; 2386 size = copy_to_user(ubuf, buf, *offset); 2387 if (size) 2388 rv = -EFAULT; 2389 2390 kfree(buf); 2391 return rv ? rv : *offset; 2392 } 2393 2394 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, 2395 size_t len, loff_t *offset) 2396 { 2397 struct driver_data *dd = (struct driver_data *)f->private_data; 2398 char *buf; 2399 u32 group_allocated; 2400 int size = *offset; 2401 int n, rv = 0; 2402 2403 if (!len || size) 2404 return 0; 2405 2406 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2407 if (!buf) { 2408 dev_err(&dd->pdev->dev, 2409 "Memory allocation: register buffer\n"); 2410 return -ENOMEM; 2411 } 2412 2413 size += sprintf(&buf[size], "H/ S ACTive : [ 0x"); 2414 2415 for (n = dd->slot_groups-1; n >= 0; n--) 2416 size += sprintf(&buf[size], "%08X ", 2417 readl(dd->port->s_active[n])); 2418 2419 size += sprintf(&buf[size], "]\n"); 2420 size += sprintf(&buf[size], "H/ Command Issue : [ 0x"); 2421 2422 for (n = dd->slot_groups-1; n >= 0; n--) 2423 size += sprintf(&buf[size], "%08X ", 2424 readl(dd->port->cmd_issue[n])); 2425 2426 size += sprintf(&buf[size], "]\n"); 2427 size += sprintf(&buf[size], "H/ Completed : [ 0x"); 2428 2429 for (n = dd->slot_groups-1; n >= 0; n--) 2430 size += sprintf(&buf[size], "%08X ", 2431 readl(dd->port->completed[n])); 2432 2433 size += sprintf(&buf[size], "]\n"); 2434 size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n", 2435 readl(dd->port->mmio + PORT_IRQ_STAT)); 2436 size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n", 2437 readl(dd->mmio + HOST_IRQ_STAT)); 2438 size += sprintf(&buf[size], "\n"); 2439 2440 size += sprintf(&buf[size], "L/ Commands in Q : [ 0x"); 2441 2442 for (n = dd->slot_groups-1; n >= 0; n--) { 2443 if (sizeof(long) > sizeof(u32)) 2444 group_allocated = 2445 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); 2446 else 2447 group_allocated = dd->port->cmds_to_issue[n]; 2448 size += sprintf(&buf[size], "%08X ", group_allocated); 2449 } 2450 size += sprintf(&buf[size], "]\n"); 2451 2452 *offset = size <= len ? size : len; 2453 size = copy_to_user(ubuf, buf, *offset); 2454 if (size) 2455 rv = -EFAULT; 2456 2457 kfree(buf); 2458 return rv ? rv : *offset; 2459 } 2460 2461 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, 2462 size_t len, loff_t *offset) 2463 { 2464 struct driver_data *dd = (struct driver_data *)f->private_data; 2465 char *buf; 2466 int size = *offset; 2467 int rv = 0; 2468 2469 if (!len || size) 2470 return 0; 2471 2472 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2473 if (!buf) { 2474 dev_err(&dd->pdev->dev, 2475 "Memory allocation: flag buffer\n"); 2476 return -ENOMEM; 2477 } 2478 2479 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", 2480 dd->port->flags); 2481 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", 2482 dd->dd_flag); 2483 2484 *offset = size <= len ? size : len; 2485 size = copy_to_user(ubuf, buf, *offset); 2486 if (size) 2487 rv = -EFAULT; 2488 2489 kfree(buf); 2490 return rv ? rv : *offset; 2491 } 2492 2493 static const struct file_operations mtip_device_status_fops = { 2494 .owner = THIS_MODULE, 2495 .open = simple_open, 2496 .read = mtip_hw_read_device_status, 2497 .llseek = no_llseek, 2498 }; 2499 2500 static const struct file_operations mtip_regs_fops = { 2501 .owner = THIS_MODULE, 2502 .open = simple_open, 2503 .read = mtip_hw_read_registers, 2504 .llseek = no_llseek, 2505 }; 2506 2507 static const struct file_operations mtip_flags_fops = { 2508 .owner = THIS_MODULE, 2509 .open = simple_open, 2510 .read = mtip_hw_read_flags, 2511 .llseek = no_llseek, 2512 }; 2513 2514 /* 2515 * Create the sysfs related attributes. 2516 * 2517 * @dd Pointer to the driver data structure. 2518 * @kobj Pointer to the kobj for the block device. 2519 * 2520 * return value 2521 * 0 Operation completed successfully. 2522 * -EINVAL Invalid parameter. 2523 */ 2524 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) 2525 { 2526 if (!kobj || !dd) 2527 return -EINVAL; 2528 2529 if (sysfs_create_file(kobj, &dev_attr_status.attr)) 2530 dev_warn(&dd->pdev->dev, 2531 "Error creating 'status' sysfs entry\n"); 2532 return 0; 2533 } 2534 2535 /* 2536 * Remove the sysfs related attributes. 2537 * 2538 * @dd Pointer to the driver data structure. 2539 * @kobj Pointer to the kobj for the block device. 2540 * 2541 * return value 2542 * 0 Operation completed successfully. 2543 * -EINVAL Invalid parameter. 2544 */ 2545 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) 2546 { 2547 if (!kobj || !dd) 2548 return -EINVAL; 2549 2550 sysfs_remove_file(kobj, &dev_attr_status.attr); 2551 2552 return 0; 2553 } 2554 2555 static int mtip_hw_debugfs_init(struct driver_data *dd) 2556 { 2557 if (!dfs_parent) 2558 return -1; 2559 2560 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); 2561 if (IS_ERR_OR_NULL(dd->dfs_node)) { 2562 dev_warn(&dd->pdev->dev, 2563 "Error creating node %s under debugfs\n", 2564 dd->disk->disk_name); 2565 dd->dfs_node = NULL; 2566 return -1; 2567 } 2568 2569 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); 2570 debugfs_create_file("registers", 0444, dd->dfs_node, dd, 2571 &mtip_regs_fops); 2572 2573 return 0; 2574 } 2575 2576 static void mtip_hw_debugfs_exit(struct driver_data *dd) 2577 { 2578 debugfs_remove_recursive(dd->dfs_node); 2579 } 2580 2581 /* 2582 * Perform any init/resume time hardware setup 2583 * 2584 * @dd Pointer to the driver data structure. 2585 * 2586 * return value 2587 * None 2588 */ 2589 static inline void hba_setup(struct driver_data *dd) 2590 { 2591 u32 hwdata; 2592 hwdata = readl(dd->mmio + HOST_HSORG); 2593 2594 /* interrupt bug workaround: use only 1 IS bit.*/ 2595 writel(hwdata | 2596 HSORG_DISABLE_SLOTGRP_INTR | 2597 HSORG_DISABLE_SLOTGRP_PXIS, 2598 dd->mmio + HOST_HSORG); 2599 } 2600 2601 static int mtip_device_unaligned_constrained(struct driver_data *dd) 2602 { 2603 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); 2604 } 2605 2606 /* 2607 * Detect the details of the product, and store anything needed 2608 * into the driver data structure. This includes product type and 2609 * version and number of slot groups. 2610 * 2611 * @dd Pointer to the driver data structure. 2612 * 2613 * return value 2614 * None 2615 */ 2616 static void mtip_detect_product(struct driver_data *dd) 2617 { 2618 u32 hwdata; 2619 unsigned int rev, slotgroups; 2620 2621 /* 2622 * HBA base + 0xFC [15:0] - vendor-specific hardware interface 2623 * info register: 2624 * [15:8] hardware/software interface rev# 2625 * [ 3] asic-style interface 2626 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). 2627 */ 2628 hwdata = readl(dd->mmio + HOST_HSORG); 2629 2630 dd->product_type = MTIP_PRODUCT_UNKNOWN; 2631 dd->slot_groups = 1; 2632 2633 if (hwdata & 0x8) { 2634 dd->product_type = MTIP_PRODUCT_ASICFPGA; 2635 rev = (hwdata & HSORG_HWREV) >> 8; 2636 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1; 2637 dev_info(&dd->pdev->dev, 2638 "ASIC-FPGA design, HS rev 0x%x, " 2639 "%i slot groups [%i slots]\n", 2640 rev, 2641 slotgroups, 2642 slotgroups * 32); 2643 2644 if (slotgroups > MTIP_MAX_SLOT_GROUPS) { 2645 dev_warn(&dd->pdev->dev, 2646 "Warning: driver only supports " 2647 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS); 2648 slotgroups = MTIP_MAX_SLOT_GROUPS; 2649 } 2650 dd->slot_groups = slotgroups; 2651 return; 2652 } 2653 2654 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); 2655 } 2656 2657 /* 2658 * Blocking wait for FTL rebuild to complete 2659 * 2660 * @dd Pointer to the DRIVER_DATA structure. 2661 * 2662 * return value 2663 * 0 FTL rebuild completed successfully 2664 * -EFAULT FTL rebuild error/timeout/interruption 2665 */ 2666 static int mtip_ftl_rebuild_poll(struct driver_data *dd) 2667 { 2668 unsigned long timeout, cnt = 0, start; 2669 2670 dev_warn(&dd->pdev->dev, 2671 "FTL rebuild in progress. Polling for completion.\n"); 2672 2673 start = jiffies; 2674 timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS); 2675 2676 do { 2677 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2678 &dd->dd_flag))) 2679 return -EFAULT; 2680 if (mtip_check_surprise_removal(dd->pdev)) 2681 return -EFAULT; 2682 2683 if (mtip_get_identify(dd->port, NULL) < 0) 2684 return -EFAULT; 2685 2686 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2687 MTIP_FTL_REBUILD_MAGIC) { 2688 ssleep(1); 2689 /* Print message every 3 minutes */ 2690 if (cnt++ >= 180) { 2691 dev_warn(&dd->pdev->dev, 2692 "FTL rebuild in progress (%d secs).\n", 2693 jiffies_to_msecs(jiffies - start) / 1000); 2694 cnt = 0; 2695 } 2696 } else { 2697 dev_warn(&dd->pdev->dev, 2698 "FTL rebuild complete (%d secs).\n", 2699 jiffies_to_msecs(jiffies - start) / 1000); 2700 mtip_block_initialize(dd); 2701 return 0; 2702 } 2703 } while (time_before(jiffies, timeout)); 2704 2705 /* Check for timeout */ 2706 dev_err(&dd->pdev->dev, 2707 "Timed out waiting for FTL rebuild to complete (%d secs).\n", 2708 jiffies_to_msecs(jiffies - start) / 1000); 2709 return -EFAULT; 2710 } 2711 2712 static void mtip_softirq_done_fn(struct request *rq) 2713 { 2714 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 2715 struct driver_data *dd = rq->q->queuedata; 2716 2717 /* Unmap the DMA scatter list entries */ 2718 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, 2719 cmd->direction); 2720 2721 if (unlikely(cmd->unaligned)) 2722 up(&dd->port->cmd_slot_unal); 2723 2724 blk_mq_end_request(rq, cmd->status); 2725 } 2726 2727 static void mtip_abort_cmd(struct request *req, void *data, bool reserved) 2728 { 2729 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); 2730 struct driver_data *dd = data; 2731 2732 dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag); 2733 2734 clear_bit(req->tag, dd->port->cmds_to_issue); 2735 cmd->status = BLK_STS_IOERR; 2736 mtip_softirq_done_fn(req); 2737 } 2738 2739 static void mtip_queue_cmd(struct request *req, void *data, bool reserved) 2740 { 2741 struct driver_data *dd = data; 2742 2743 set_bit(req->tag, dd->port->cmds_to_issue); 2744 blk_abort_request(req); 2745 } 2746 2747 /* 2748 * service thread to issue queued commands 2749 * 2750 * @data Pointer to the driver data structure. 2751 * 2752 * return value 2753 * 0 2754 */ 2755 2756 static int mtip_service_thread(void *data) 2757 { 2758 struct driver_data *dd = (struct driver_data *)data; 2759 unsigned long slot, slot_start, slot_wrap, to; 2760 unsigned int num_cmd_slots = dd->slot_groups * 32; 2761 struct mtip_port *port = dd->port; 2762 2763 while (1) { 2764 if (kthread_should_stop() || 2765 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2766 goto st_out; 2767 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2768 2769 /* 2770 * the condition is to check neither an internal command is 2771 * is in progress nor error handling is active 2772 */ 2773 wait_event_interruptible(port->svc_wait, (port->flags) && 2774 (port->flags & MTIP_PF_SVC_THD_WORK)); 2775 2776 if (kthread_should_stop() || 2777 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2778 goto st_out; 2779 2780 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2781 &dd->dd_flag))) 2782 goto st_out; 2783 2784 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2785 2786 restart_eh: 2787 /* Demux bits: start with error handling */ 2788 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { 2789 mtip_handle_tfe(dd); 2790 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 2791 } 2792 2793 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) 2794 goto restart_eh; 2795 2796 if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { 2797 to = jiffies + msecs_to_jiffies(5000); 2798 2799 do { 2800 mdelay(100); 2801 } while (atomic_read(&dd->irq_workers_active) != 0 && 2802 time_before(jiffies, to)); 2803 2804 if (atomic_read(&dd->irq_workers_active) != 0) 2805 dev_warn(&dd->pdev->dev, 2806 "Completion workers still active!"); 2807 2808 blk_mq_quiesce_queue(dd->queue); 2809 2810 spin_lock(dd->queue->queue_lock); 2811 blk_mq_tagset_busy_iter(&dd->tags, 2812 mtip_queue_cmd, dd); 2813 spin_unlock(dd->queue->queue_lock); 2814 2815 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); 2816 2817 if (mtip_device_reset(dd)) 2818 blk_mq_tagset_busy_iter(&dd->tags, 2819 mtip_abort_cmd, dd); 2820 2821 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); 2822 2823 blk_mq_unquiesce_queue(dd->queue); 2824 } 2825 2826 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 2827 slot = 1; 2828 /* used to restrict the loop to one iteration */ 2829 slot_start = num_cmd_slots; 2830 slot_wrap = 0; 2831 while (1) { 2832 slot = find_next_bit(port->cmds_to_issue, 2833 num_cmd_slots, slot); 2834 if (slot_wrap == 1) { 2835 if ((slot_start >= slot) || 2836 (slot >= num_cmd_slots)) 2837 break; 2838 } 2839 if (unlikely(slot_start == num_cmd_slots)) 2840 slot_start = slot; 2841 2842 if (unlikely(slot == num_cmd_slots)) { 2843 slot = 1; 2844 slot_wrap = 1; 2845 continue; 2846 } 2847 2848 /* Issue the command to the hardware */ 2849 mtip_issue_ncq_command(port, slot); 2850 2851 clear_bit(slot, port->cmds_to_issue); 2852 } 2853 2854 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2855 } 2856 2857 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { 2858 if (mtip_ftl_rebuild_poll(dd) == 0) 2859 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); 2860 } 2861 } 2862 2863 st_out: 2864 return 0; 2865 } 2866 2867 /* 2868 * DMA region teardown 2869 * 2870 * @dd Pointer to driver_data structure 2871 * 2872 * return value 2873 * None 2874 */ 2875 static void mtip_dma_free(struct driver_data *dd) 2876 { 2877 struct mtip_port *port = dd->port; 2878 2879 if (port->block1) 2880 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2881 port->block1, port->block1_dma); 2882 2883 if (port->command_list) { 2884 dmam_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 2885 port->command_list, port->command_list_dma); 2886 } 2887 } 2888 2889 /* 2890 * DMA region setup 2891 * 2892 * @dd Pointer to driver_data structure 2893 * 2894 * return value 2895 * -ENOMEM Not enough free DMA region space to initialize driver 2896 */ 2897 static int mtip_dma_alloc(struct driver_data *dd) 2898 { 2899 struct mtip_port *port = dd->port; 2900 2901 /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */ 2902 port->block1 = 2903 dmam_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2904 &port->block1_dma, GFP_KERNEL); 2905 if (!port->block1) 2906 return -ENOMEM; 2907 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); 2908 2909 /* Allocate dma memory for command list */ 2910 port->command_list = 2911 dmam_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 2912 &port->command_list_dma, GFP_KERNEL); 2913 if (!port->command_list) { 2914 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2915 port->block1, port->block1_dma); 2916 port->block1 = NULL; 2917 port->block1_dma = 0; 2918 return -ENOMEM; 2919 } 2920 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); 2921 2922 /* Setup all pointers into first DMA region */ 2923 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; 2924 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; 2925 port->identify = port->block1 + AHCI_IDFY_OFFSET; 2926 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; 2927 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; 2928 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; 2929 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; 2930 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; 2931 2932 return 0; 2933 } 2934 2935 static int mtip_hw_get_identify(struct driver_data *dd) 2936 { 2937 struct smart_attr attr242; 2938 unsigned char *buf; 2939 int rv; 2940 2941 if (mtip_get_identify(dd->port, NULL) < 0) 2942 return -EFAULT; 2943 2944 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2945 MTIP_FTL_REBUILD_MAGIC) { 2946 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); 2947 return MTIP_FTL_REBUILD_MAGIC; 2948 } 2949 mtip_dump_identify(dd->port); 2950 2951 /* check write protect, over temp and rebuild statuses */ 2952 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 2953 dd->port->log_buf, 2954 dd->port->log_buf_dma, 1); 2955 if (rv) { 2956 dev_warn(&dd->pdev->dev, 2957 "Error in READ LOG EXT (10h) command\n"); 2958 /* non-critical error, don't fail the load */ 2959 } else { 2960 buf = (unsigned char *)dd->port->log_buf; 2961 if (buf[259] & 0x1) { 2962 dev_info(&dd->pdev->dev, 2963 "Write protect bit is set.\n"); 2964 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 2965 } 2966 if (buf[288] == 0xF7) { 2967 dev_info(&dd->pdev->dev, 2968 "Exceeded Tmax, drive in thermal shutdown.\n"); 2969 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 2970 } 2971 if (buf[288] == 0xBF) { 2972 dev_info(&dd->pdev->dev, 2973 "Drive indicates rebuild has failed.\n"); 2974 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); 2975 } 2976 } 2977 2978 /* get write protect progess */ 2979 memset(&attr242, 0, sizeof(struct smart_attr)); 2980 if (mtip_get_smart_attr(dd->port, 242, &attr242)) 2981 dev_warn(&dd->pdev->dev, 2982 "Unable to check write protect progress\n"); 2983 else 2984 dev_info(&dd->pdev->dev, 2985 "Write protect progress: %u%% (%u blocks)\n", 2986 attr242.cur, le32_to_cpu(attr242.data)); 2987 2988 return rv; 2989 } 2990 2991 /* 2992 * Called once for each card. 2993 * 2994 * @dd Pointer to the driver data structure. 2995 * 2996 * return value 2997 * 0 on success, else an error code. 2998 */ 2999 static int mtip_hw_init(struct driver_data *dd) 3000 { 3001 int i; 3002 int rv; 3003 unsigned long timeout, timetaken; 3004 3005 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; 3006 3007 mtip_detect_product(dd); 3008 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { 3009 rv = -EIO; 3010 goto out1; 3011 } 3012 3013 hba_setup(dd); 3014 3015 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, 3016 dd->numa_node); 3017 if (!dd->port) { 3018 dev_err(&dd->pdev->dev, 3019 "Memory allocation: port structure\n"); 3020 return -ENOMEM; 3021 } 3022 3023 /* Continue workqueue setup */ 3024 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3025 dd->work[i].port = dd->port; 3026 3027 /* Enable unaligned IO constraints for some devices */ 3028 if (mtip_device_unaligned_constrained(dd)) 3029 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; 3030 else 3031 dd->unal_qdepth = 0; 3032 3033 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); 3034 3035 /* Spinlock to prevent concurrent issue */ 3036 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3037 spin_lock_init(&dd->port->cmd_issue_lock[i]); 3038 3039 /* Set the port mmio base address. */ 3040 dd->port->mmio = dd->mmio + PORT_OFFSET; 3041 dd->port->dd = dd; 3042 3043 /* DMA allocations */ 3044 rv = mtip_dma_alloc(dd); 3045 if (rv < 0) 3046 goto out1; 3047 3048 /* Setup the pointers to the extended s_active and CI registers. */ 3049 for (i = 0; i < dd->slot_groups; i++) { 3050 dd->port->s_active[i] = 3051 dd->port->mmio + i*0x80 + PORT_SCR_ACT; 3052 dd->port->cmd_issue[i] = 3053 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; 3054 dd->port->completed[i] = 3055 dd->port->mmio + i*0x80 + PORT_SDBV; 3056 } 3057 3058 timetaken = jiffies; 3059 timeout = jiffies + msecs_to_jiffies(30000); 3060 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && 3061 time_before(jiffies, timeout)) { 3062 mdelay(100); 3063 } 3064 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 3065 timetaken = jiffies - timetaken; 3066 dev_warn(&dd->pdev->dev, 3067 "Surprise removal detected at %u ms\n", 3068 jiffies_to_msecs(timetaken)); 3069 rv = -ENODEV; 3070 goto out2 ; 3071 } 3072 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 3073 timetaken = jiffies - timetaken; 3074 dev_warn(&dd->pdev->dev, 3075 "Removal detected at %u ms\n", 3076 jiffies_to_msecs(timetaken)); 3077 rv = -EFAULT; 3078 goto out2; 3079 } 3080 3081 /* Conditionally reset the HBA. */ 3082 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { 3083 if (mtip_hba_reset(dd) < 0) { 3084 dev_err(&dd->pdev->dev, 3085 "Card did not reset within timeout\n"); 3086 rv = -EIO; 3087 goto out2; 3088 } 3089 } else { 3090 /* Clear any pending interrupts on the HBA */ 3091 writel(readl(dd->mmio + HOST_IRQ_STAT), 3092 dd->mmio + HOST_IRQ_STAT); 3093 } 3094 3095 mtip_init_port(dd->port); 3096 mtip_start_port(dd->port); 3097 3098 /* Setup the ISR and enable interrupts. */ 3099 rv = devm_request_irq(&dd->pdev->dev, 3100 dd->pdev->irq, 3101 mtip_irq_handler, 3102 IRQF_SHARED, 3103 dev_driver_string(&dd->pdev->dev), 3104 dd); 3105 3106 if (rv) { 3107 dev_err(&dd->pdev->dev, 3108 "Unable to allocate IRQ %d\n", dd->pdev->irq); 3109 goto out2; 3110 } 3111 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); 3112 3113 /* Enable interrupts on the HBA. */ 3114 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3115 dd->mmio + HOST_CTL); 3116 3117 init_waitqueue_head(&dd->port->svc_wait); 3118 3119 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 3120 rv = -EFAULT; 3121 goto out3; 3122 } 3123 3124 return rv; 3125 3126 out3: 3127 /* Disable interrupts on the HBA. */ 3128 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3129 dd->mmio + HOST_CTL); 3130 3131 /* Release the IRQ. */ 3132 irq_set_affinity_hint(dd->pdev->irq, NULL); 3133 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3134 3135 out2: 3136 mtip_deinit_port(dd->port); 3137 mtip_dma_free(dd); 3138 3139 out1: 3140 /* Free the memory allocated for the for structure. */ 3141 kfree(dd->port); 3142 3143 return rv; 3144 } 3145 3146 static int mtip_standby_drive(struct driver_data *dd) 3147 { 3148 int rv = 0; 3149 3150 if (dd->sr || !dd->port) 3151 return -ENODEV; 3152 /* 3153 * Send standby immediate (E0h) to the drive so that it 3154 * saves its state. 3155 */ 3156 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && 3157 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && 3158 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { 3159 rv = mtip_standby_immediate(dd->port); 3160 if (rv) 3161 dev_warn(&dd->pdev->dev, 3162 "STANDBY IMMEDIATE failed\n"); 3163 } 3164 return rv; 3165 } 3166 3167 /* 3168 * Called to deinitialize an interface. 3169 * 3170 * @dd Pointer to the driver data structure. 3171 * 3172 * return value 3173 * 0 3174 */ 3175 static int mtip_hw_exit(struct driver_data *dd) 3176 { 3177 if (!dd->sr) { 3178 /* de-initialize the port. */ 3179 mtip_deinit_port(dd->port); 3180 3181 /* Disable interrupts on the HBA. */ 3182 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3183 dd->mmio + HOST_CTL); 3184 } 3185 3186 /* Release the IRQ. */ 3187 irq_set_affinity_hint(dd->pdev->irq, NULL); 3188 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3189 msleep(1000); 3190 3191 /* Free dma regions */ 3192 mtip_dma_free(dd); 3193 3194 /* Free the memory allocated for the for structure. */ 3195 kfree(dd->port); 3196 dd->port = NULL; 3197 3198 return 0; 3199 } 3200 3201 /* 3202 * Issue a Standby Immediate command to the device. 3203 * 3204 * This function is called by the Block Layer just before the 3205 * system powers off during a shutdown. 3206 * 3207 * @dd Pointer to the driver data structure. 3208 * 3209 * return value 3210 * 0 3211 */ 3212 static int mtip_hw_shutdown(struct driver_data *dd) 3213 { 3214 /* 3215 * Send standby immediate (E0h) to the drive so that it 3216 * saves its state. 3217 */ 3218 mtip_standby_drive(dd); 3219 3220 return 0; 3221 } 3222 3223 /* 3224 * Suspend function 3225 * 3226 * This function is called by the Block Layer just before the 3227 * system hibernates. 3228 * 3229 * @dd Pointer to the driver data structure. 3230 * 3231 * return value 3232 * 0 Suspend was successful 3233 * -EFAULT Suspend was not successful 3234 */ 3235 static int mtip_hw_suspend(struct driver_data *dd) 3236 { 3237 /* 3238 * Send standby immediate (E0h) to the drive 3239 * so that it saves its state. 3240 */ 3241 if (mtip_standby_drive(dd) != 0) { 3242 dev_err(&dd->pdev->dev, 3243 "Failed standby-immediate command\n"); 3244 return -EFAULT; 3245 } 3246 3247 /* Disable interrupts on the HBA.*/ 3248 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3249 dd->mmio + HOST_CTL); 3250 mtip_deinit_port(dd->port); 3251 3252 return 0; 3253 } 3254 3255 /* 3256 * Resume function 3257 * 3258 * This function is called by the Block Layer as the 3259 * system resumes. 3260 * 3261 * @dd Pointer to the driver data structure. 3262 * 3263 * return value 3264 * 0 Resume was successful 3265 * -EFAULT Resume was not successful 3266 */ 3267 static int mtip_hw_resume(struct driver_data *dd) 3268 { 3269 /* Perform any needed hardware setup steps */ 3270 hba_setup(dd); 3271 3272 /* Reset the HBA */ 3273 if (mtip_hba_reset(dd) != 0) { 3274 dev_err(&dd->pdev->dev, 3275 "Unable to reset the HBA\n"); 3276 return -EFAULT; 3277 } 3278 3279 /* 3280 * Enable the port, DMA engine, and FIS reception specific 3281 * h/w in controller. 3282 */ 3283 mtip_init_port(dd->port); 3284 mtip_start_port(dd->port); 3285 3286 /* Enable interrupts on the HBA.*/ 3287 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3288 dd->mmio + HOST_CTL); 3289 3290 return 0; 3291 } 3292 3293 /* 3294 * Helper function for reusing disk name 3295 * upon hot insertion. 3296 */ 3297 static int rssd_disk_name_format(char *prefix, 3298 int index, 3299 char *buf, 3300 int buflen) 3301 { 3302 const int base = 'z' - 'a' + 1; 3303 char *begin = buf + strlen(prefix); 3304 char *end = buf + buflen; 3305 char *p; 3306 int unit; 3307 3308 p = end - 1; 3309 *p = '\0'; 3310 unit = base; 3311 do { 3312 if (p == begin) 3313 return -EINVAL; 3314 *--p = 'a' + (index % unit); 3315 index = (index / unit) - 1; 3316 } while (index >= 0); 3317 3318 memmove(begin, p, end - p); 3319 memcpy(buf, prefix, strlen(prefix)); 3320 3321 return 0; 3322 } 3323 3324 /* 3325 * Block layer IOCTL handler. 3326 * 3327 * @dev Pointer to the block_device structure. 3328 * @mode ignored 3329 * @cmd IOCTL command passed from the user application. 3330 * @arg Argument passed from the user application. 3331 * 3332 * return value 3333 * 0 IOCTL completed successfully. 3334 * -ENOTTY IOCTL not supported or invalid driver data 3335 * structure pointer. 3336 */ 3337 static int mtip_block_ioctl(struct block_device *dev, 3338 fmode_t mode, 3339 unsigned cmd, 3340 unsigned long arg) 3341 { 3342 struct driver_data *dd = dev->bd_disk->private_data; 3343 3344 if (!capable(CAP_SYS_ADMIN)) 3345 return -EACCES; 3346 3347 if (!dd) 3348 return -ENOTTY; 3349 3350 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3351 return -ENOTTY; 3352 3353 switch (cmd) { 3354 case BLKFLSBUF: 3355 return -ENOTTY; 3356 default: 3357 return mtip_hw_ioctl(dd, cmd, arg); 3358 } 3359 } 3360 3361 #ifdef CONFIG_COMPAT 3362 /* 3363 * Block layer compat IOCTL handler. 3364 * 3365 * @dev Pointer to the block_device structure. 3366 * @mode ignored 3367 * @cmd IOCTL command passed from the user application. 3368 * @arg Argument passed from the user application. 3369 * 3370 * return value 3371 * 0 IOCTL completed successfully. 3372 * -ENOTTY IOCTL not supported or invalid driver data 3373 * structure pointer. 3374 */ 3375 static int mtip_block_compat_ioctl(struct block_device *dev, 3376 fmode_t mode, 3377 unsigned cmd, 3378 unsigned long arg) 3379 { 3380 struct driver_data *dd = dev->bd_disk->private_data; 3381 3382 if (!capable(CAP_SYS_ADMIN)) 3383 return -EACCES; 3384 3385 if (!dd) 3386 return -ENOTTY; 3387 3388 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3389 return -ENOTTY; 3390 3391 switch (cmd) { 3392 case BLKFLSBUF: 3393 return -ENOTTY; 3394 case HDIO_DRIVE_TASKFILE: { 3395 struct mtip_compat_ide_task_request_s __user *compat_req_task; 3396 ide_task_request_t req_task; 3397 int compat_tasksize, outtotal, ret; 3398 3399 compat_tasksize = 3400 sizeof(struct mtip_compat_ide_task_request_s); 3401 3402 compat_req_task = 3403 (struct mtip_compat_ide_task_request_s __user *) arg; 3404 3405 if (copy_from_user(&req_task, (void __user *) arg, 3406 compat_tasksize - (2 * sizeof(compat_long_t)))) 3407 return -EFAULT; 3408 3409 if (get_user(req_task.out_size, &compat_req_task->out_size)) 3410 return -EFAULT; 3411 3412 if (get_user(req_task.in_size, &compat_req_task->in_size)) 3413 return -EFAULT; 3414 3415 outtotal = sizeof(struct mtip_compat_ide_task_request_s); 3416 3417 ret = exec_drive_taskfile(dd, (void __user *) arg, 3418 &req_task, outtotal); 3419 3420 if (copy_to_user((void __user *) arg, &req_task, 3421 compat_tasksize - 3422 (2 * sizeof(compat_long_t)))) 3423 return -EFAULT; 3424 3425 if (put_user(req_task.out_size, &compat_req_task->out_size)) 3426 return -EFAULT; 3427 3428 if (put_user(req_task.in_size, &compat_req_task->in_size)) 3429 return -EFAULT; 3430 3431 return ret; 3432 } 3433 default: 3434 return mtip_hw_ioctl(dd, cmd, arg); 3435 } 3436 } 3437 #endif 3438 3439 /* 3440 * Obtain the geometry of the device. 3441 * 3442 * You may think that this function is obsolete, but some applications, 3443 * fdisk for example still used CHS values. This function describes the 3444 * device as having 224 heads and 56 sectors per cylinder. These values are 3445 * chosen so that each cylinder is aligned on a 4KB boundary. Since a 3446 * partition is described in terms of a start and end cylinder this means 3447 * that each partition is also 4KB aligned. Non-aligned partitions adversely 3448 * affects performance. 3449 * 3450 * @dev Pointer to the block_device strucutre. 3451 * @geo Pointer to a hd_geometry structure. 3452 * 3453 * return value 3454 * 0 Operation completed successfully. 3455 * -ENOTTY An error occurred while reading the drive capacity. 3456 */ 3457 static int mtip_block_getgeo(struct block_device *dev, 3458 struct hd_geometry *geo) 3459 { 3460 struct driver_data *dd = dev->bd_disk->private_data; 3461 sector_t capacity; 3462 3463 if (!dd) 3464 return -ENOTTY; 3465 3466 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3467 dev_warn(&dd->pdev->dev, 3468 "Could not get drive capacity.\n"); 3469 return -ENOTTY; 3470 } 3471 3472 geo->heads = 224; 3473 geo->sectors = 56; 3474 sector_div(capacity, (geo->heads * geo->sectors)); 3475 geo->cylinders = capacity; 3476 return 0; 3477 } 3478 3479 static int mtip_block_open(struct block_device *dev, fmode_t mode) 3480 { 3481 struct driver_data *dd; 3482 3483 if (dev && dev->bd_disk) { 3484 dd = (struct driver_data *) dev->bd_disk->private_data; 3485 3486 if (dd) { 3487 if (test_bit(MTIP_DDF_REMOVAL_BIT, 3488 &dd->dd_flag)) { 3489 return -ENODEV; 3490 } 3491 return 0; 3492 } 3493 } 3494 return -ENODEV; 3495 } 3496 3497 static void mtip_block_release(struct gendisk *disk, fmode_t mode) 3498 { 3499 } 3500 3501 /* 3502 * Block device operation function. 3503 * 3504 * This structure contains pointers to the functions required by the block 3505 * layer. 3506 */ 3507 static const struct block_device_operations mtip_block_ops = { 3508 .open = mtip_block_open, 3509 .release = mtip_block_release, 3510 .ioctl = mtip_block_ioctl, 3511 #ifdef CONFIG_COMPAT 3512 .compat_ioctl = mtip_block_compat_ioctl, 3513 #endif 3514 .getgeo = mtip_block_getgeo, 3515 .owner = THIS_MODULE 3516 }; 3517 3518 static inline bool is_se_active(struct driver_data *dd) 3519 { 3520 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { 3521 if (dd->port->ic_pause_timer) { 3522 unsigned long to = dd->port->ic_pause_timer + 3523 msecs_to_jiffies(1000); 3524 if (time_after(jiffies, to)) { 3525 clear_bit(MTIP_PF_SE_ACTIVE_BIT, 3526 &dd->port->flags); 3527 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); 3528 dd->port->ic_pause_timer = 0; 3529 wake_up_interruptible(&dd->port->svc_wait); 3530 return false; 3531 } 3532 } 3533 return true; 3534 } 3535 return false; 3536 } 3537 3538 /* 3539 * Block layer make request function. 3540 * 3541 * This function is called by the kernel to process a BIO for 3542 * the P320 device. 3543 * 3544 * @queue Pointer to the request queue. Unused other than to obtain 3545 * the driver data structure. 3546 * @rq Pointer to the request. 3547 * 3548 */ 3549 static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) 3550 { 3551 struct driver_data *dd = hctx->queue->queuedata; 3552 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3553 unsigned int nents; 3554 3555 if (is_se_active(dd)) 3556 return -ENODATA; 3557 3558 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { 3559 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3560 &dd->dd_flag))) { 3561 return -ENXIO; 3562 } 3563 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { 3564 return -ENODATA; 3565 } 3566 if (unlikely(test_bit(MTIP_DDF_WRITE_PROTECT_BIT, 3567 &dd->dd_flag) && 3568 rq_data_dir(rq))) { 3569 return -ENODATA; 3570 } 3571 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag) || 3572 test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))) 3573 return -ENODATA; 3574 } 3575 3576 if (req_op(rq) == REQ_OP_DISCARD) { 3577 int err; 3578 3579 err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); 3580 blk_mq_end_request(rq, err ? BLK_STS_IOERR : BLK_STS_OK); 3581 return 0; 3582 } 3583 3584 /* Create the scatter list for this request. */ 3585 nents = blk_rq_map_sg(hctx->queue, rq, cmd->sg); 3586 3587 /* Issue the read/write. */ 3588 mtip_hw_submit_io(dd, rq, cmd, nents, hctx); 3589 return 0; 3590 } 3591 3592 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, 3593 struct request *rq) 3594 { 3595 struct driver_data *dd = hctx->queue->queuedata; 3596 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3597 3598 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) 3599 return false; 3600 3601 /* 3602 * If unaligned depth must be limited on this controller, mark it 3603 * as unaligned if the IO isn't on a 4k boundary (start of length). 3604 */ 3605 if (blk_rq_sectors(rq) <= 64) { 3606 if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7)) 3607 cmd->unaligned = 1; 3608 } 3609 3610 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) 3611 return true; 3612 3613 return false; 3614 } 3615 3616 static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx, 3617 struct request *rq) 3618 { 3619 struct driver_data *dd = hctx->queue->queuedata; 3620 struct mtip_int_cmd *icmd = rq->special; 3621 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3622 struct mtip_cmd_sg *command_sg; 3623 3624 if (mtip_commands_active(dd->port)) 3625 return BLK_STS_RESOURCE; 3626 3627 /* Populate the SG list */ 3628 cmd->command_header->opts = 3629 __force_bit2int cpu_to_le32(icmd->opts | icmd->fis_len); 3630 if (icmd->buf_len) { 3631 command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ; 3632 3633 command_sg->info = 3634 __force_bit2int cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF); 3635 command_sg->dba = 3636 __force_bit2int cpu_to_le32(icmd->buffer & 0xFFFFFFFF); 3637 command_sg->dba_upper = 3638 __force_bit2int cpu_to_le32((icmd->buffer >> 16) >> 16); 3639 3640 cmd->command_header->opts |= 3641 __force_bit2int cpu_to_le32((1 << 16)); 3642 } 3643 3644 /* Populate the command header */ 3645 cmd->command_header->byte_count = 0; 3646 3647 blk_mq_start_request(rq); 3648 mtip_issue_non_ncq_command(dd->port, rq->tag); 3649 return 0; 3650 } 3651 3652 static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx, 3653 const struct blk_mq_queue_data *bd) 3654 { 3655 struct request *rq = bd->rq; 3656 int ret; 3657 3658 mtip_init_cmd_header(rq); 3659 3660 if (blk_rq_is_passthrough(rq)) 3661 return mtip_issue_reserved_cmd(hctx, rq); 3662 3663 if (unlikely(mtip_check_unal_depth(hctx, rq))) 3664 return BLK_STS_RESOURCE; 3665 3666 blk_mq_start_request(rq); 3667 3668 ret = mtip_submit_request(hctx, rq); 3669 if (likely(!ret)) 3670 return BLK_STS_OK; 3671 return BLK_STS_IOERR; 3672 } 3673 3674 static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq, 3675 unsigned int hctx_idx) 3676 { 3677 struct driver_data *dd = set->driver_data; 3678 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3679 3680 if (!cmd->command) 3681 return; 3682 3683 dmam_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3684 cmd->command, cmd->command_dma); 3685 } 3686 3687 static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, 3688 unsigned int hctx_idx, unsigned int numa_node) 3689 { 3690 struct driver_data *dd = set->driver_data; 3691 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3692 3693 cmd->command = dmam_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3694 &cmd->command_dma, GFP_KERNEL); 3695 if (!cmd->command) 3696 return -ENOMEM; 3697 3698 memset(cmd->command, 0, CMD_DMA_ALLOC_SZ); 3699 3700 sg_init_table(cmd->sg, MTIP_MAX_SG); 3701 return 0; 3702 } 3703 3704 static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req, 3705 bool reserved) 3706 { 3707 struct driver_data *dd = req->q->queuedata; 3708 3709 if (reserved) { 3710 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); 3711 3712 cmd->status = BLK_STS_TIMEOUT; 3713 blk_mq_complete_request(req); 3714 return BLK_EH_DONE; 3715 } 3716 3717 if (test_bit(req->tag, dd->port->cmds_to_issue)) 3718 goto exit_handler; 3719 3720 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) 3721 goto exit_handler; 3722 3723 wake_up_interruptible(&dd->port->svc_wait); 3724 exit_handler: 3725 return BLK_EH_RESET_TIMER; 3726 } 3727 3728 static const struct blk_mq_ops mtip_mq_ops = { 3729 .queue_rq = mtip_queue_rq, 3730 .init_request = mtip_init_cmd, 3731 .exit_request = mtip_free_cmd, 3732 .complete = mtip_softirq_done_fn, 3733 .timeout = mtip_cmd_timeout, 3734 }; 3735 3736 /* 3737 * Block layer initialization function. 3738 * 3739 * This function is called once by the PCI layer for each P320 3740 * device that is connected to the system. 3741 * 3742 * @dd Pointer to the driver data structure. 3743 * 3744 * return value 3745 * 0 on success else an error code. 3746 */ 3747 static int mtip_block_initialize(struct driver_data *dd) 3748 { 3749 int rv = 0, wait_for_rebuild = 0; 3750 sector_t capacity; 3751 unsigned int index = 0; 3752 struct kobject *kobj; 3753 3754 if (dd->disk) 3755 goto skip_create_disk; /* hw init done, before rebuild */ 3756 3757 if (mtip_hw_init(dd)) { 3758 rv = -EINVAL; 3759 goto protocol_init_error; 3760 } 3761 3762 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); 3763 if (dd->disk == NULL) { 3764 dev_err(&dd->pdev->dev, 3765 "Unable to allocate gendisk structure\n"); 3766 rv = -EINVAL; 3767 goto alloc_disk_error; 3768 } 3769 3770 /* Generate the disk name, implemented same as in sd.c */ 3771 do { 3772 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) { 3773 rv = -ENOMEM; 3774 goto ida_get_error; 3775 } 3776 3777 spin_lock(&rssd_index_lock); 3778 rv = ida_get_new(&rssd_index_ida, &index); 3779 spin_unlock(&rssd_index_lock); 3780 } while (rv == -EAGAIN); 3781 3782 if (rv) 3783 goto ida_get_error; 3784 3785 rv = rssd_disk_name_format("rssd", 3786 index, 3787 dd->disk->disk_name, 3788 DISK_NAME_LEN); 3789 if (rv) 3790 goto disk_index_error; 3791 3792 dd->disk->major = dd->major; 3793 dd->disk->first_minor = index * MTIP_MAX_MINORS; 3794 dd->disk->minors = MTIP_MAX_MINORS; 3795 dd->disk->fops = &mtip_block_ops; 3796 dd->disk->private_data = dd; 3797 dd->index = index; 3798 3799 mtip_hw_debugfs_init(dd); 3800 3801 memset(&dd->tags, 0, sizeof(dd->tags)); 3802 dd->tags.ops = &mtip_mq_ops; 3803 dd->tags.nr_hw_queues = 1; 3804 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; 3805 dd->tags.reserved_tags = 1; 3806 dd->tags.cmd_size = sizeof(struct mtip_cmd); 3807 dd->tags.numa_node = dd->numa_node; 3808 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; 3809 dd->tags.driver_data = dd; 3810 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; 3811 3812 rv = blk_mq_alloc_tag_set(&dd->tags); 3813 if (rv) { 3814 dev_err(&dd->pdev->dev, 3815 "Unable to allocate request queue\n"); 3816 goto block_queue_alloc_tag_error; 3817 } 3818 3819 /* Allocate the request queue. */ 3820 dd->queue = blk_mq_init_queue(&dd->tags); 3821 if (IS_ERR(dd->queue)) { 3822 dev_err(&dd->pdev->dev, 3823 "Unable to allocate request queue\n"); 3824 rv = -ENOMEM; 3825 goto block_queue_alloc_init_error; 3826 } 3827 3828 dd->disk->queue = dd->queue; 3829 dd->queue->queuedata = dd; 3830 3831 skip_create_disk: 3832 /* Initialize the protocol layer. */ 3833 wait_for_rebuild = mtip_hw_get_identify(dd); 3834 if (wait_for_rebuild < 0) { 3835 dev_err(&dd->pdev->dev, 3836 "Protocol layer initialization failed\n"); 3837 rv = -EINVAL; 3838 goto init_hw_cmds_error; 3839 } 3840 3841 /* 3842 * if rebuild pending, start the service thread, and delay the block 3843 * queue creation and device_add_disk() 3844 */ 3845 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3846 goto start_service_thread; 3847 3848 /* Set device limits. */ 3849 blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue); 3850 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue); 3851 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); 3852 blk_queue_physical_block_size(dd->queue, 4096); 3853 blk_queue_max_hw_sectors(dd->queue, 0xffff); 3854 blk_queue_max_segment_size(dd->queue, 0x400000); 3855 blk_queue_io_min(dd->queue, 4096); 3856 3857 /* Signal trim support */ 3858 if (dd->trim_supp == true) { 3859 blk_queue_flag_set(QUEUE_FLAG_DISCARD, dd->queue); 3860 dd->queue->limits.discard_granularity = 4096; 3861 blk_queue_max_discard_sectors(dd->queue, 3862 MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES); 3863 } 3864 3865 /* Set the capacity of the device in 512 byte sectors. */ 3866 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3867 dev_warn(&dd->pdev->dev, 3868 "Could not read drive capacity\n"); 3869 rv = -EIO; 3870 goto read_capacity_error; 3871 } 3872 set_capacity(dd->disk, capacity); 3873 3874 /* Enable the block device and add it to /dev */ 3875 device_add_disk(&dd->pdev->dev, dd->disk); 3876 3877 dd->bdev = bdget_disk(dd->disk, 0); 3878 /* 3879 * Now that the disk is active, initialize any sysfs attributes 3880 * managed by the protocol layer. 3881 */ 3882 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3883 if (kobj) { 3884 mtip_hw_sysfs_init(dd, kobj); 3885 kobject_put(kobj); 3886 } 3887 3888 if (dd->mtip_svc_handler) { 3889 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 3890 return rv; /* service thread created for handling rebuild */ 3891 } 3892 3893 start_service_thread: 3894 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, 3895 dd, dd->numa_node, 3896 "mtip_svc_thd_%02d", index); 3897 3898 if (IS_ERR(dd->mtip_svc_handler)) { 3899 dev_err(&dd->pdev->dev, "service thread failed to start\n"); 3900 dd->mtip_svc_handler = NULL; 3901 rv = -EFAULT; 3902 goto kthread_run_error; 3903 } 3904 wake_up_process(dd->mtip_svc_handler); 3905 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3906 rv = wait_for_rebuild; 3907 3908 return rv; 3909 3910 kthread_run_error: 3911 bdput(dd->bdev); 3912 dd->bdev = NULL; 3913 3914 /* Delete our gendisk. This also removes the device from /dev */ 3915 del_gendisk(dd->disk); 3916 3917 read_capacity_error: 3918 init_hw_cmds_error: 3919 blk_cleanup_queue(dd->queue); 3920 block_queue_alloc_init_error: 3921 blk_mq_free_tag_set(&dd->tags); 3922 block_queue_alloc_tag_error: 3923 mtip_hw_debugfs_exit(dd); 3924 disk_index_error: 3925 spin_lock(&rssd_index_lock); 3926 ida_remove(&rssd_index_ida, index); 3927 spin_unlock(&rssd_index_lock); 3928 3929 ida_get_error: 3930 put_disk(dd->disk); 3931 3932 alloc_disk_error: 3933 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ 3934 3935 protocol_init_error: 3936 return rv; 3937 } 3938 3939 static void mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv) 3940 { 3941 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3942 3943 cmd->status = BLK_STS_IOERR; 3944 blk_mq_complete_request(rq); 3945 } 3946 3947 /* 3948 * Block layer deinitialization function. 3949 * 3950 * Called by the PCI layer as each P320 device is removed. 3951 * 3952 * @dd Pointer to the driver data structure. 3953 * 3954 * return value 3955 * 0 3956 */ 3957 static int mtip_block_remove(struct driver_data *dd) 3958 { 3959 struct kobject *kobj; 3960 3961 mtip_hw_debugfs_exit(dd); 3962 3963 if (dd->mtip_svc_handler) { 3964 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); 3965 wake_up_interruptible(&dd->port->svc_wait); 3966 kthread_stop(dd->mtip_svc_handler); 3967 } 3968 3969 /* Clean up the sysfs attributes, if created */ 3970 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { 3971 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3972 if (kobj) { 3973 mtip_hw_sysfs_exit(dd, kobj); 3974 kobject_put(kobj); 3975 } 3976 } 3977 3978 if (!dd->sr) { 3979 /* 3980 * Explicitly wait here for IOs to quiesce, 3981 * as mtip_standby_drive usually won't wait for IOs. 3982 */ 3983 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) 3984 mtip_standby_drive(dd); 3985 } 3986 else 3987 dev_info(&dd->pdev->dev, "device %s surprise removal\n", 3988 dd->disk->disk_name); 3989 3990 blk_freeze_queue_start(dd->queue); 3991 blk_mq_quiesce_queue(dd->queue); 3992 blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd); 3993 blk_mq_unquiesce_queue(dd->queue); 3994 3995 /* 3996 * Delete our gendisk structure. This also removes the device 3997 * from /dev 3998 */ 3999 if (dd->bdev) { 4000 bdput(dd->bdev); 4001 dd->bdev = NULL; 4002 } 4003 if (dd->disk) { 4004 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) 4005 del_gendisk(dd->disk); 4006 if (dd->disk->queue) { 4007 blk_cleanup_queue(dd->queue); 4008 blk_mq_free_tag_set(&dd->tags); 4009 dd->queue = NULL; 4010 } 4011 put_disk(dd->disk); 4012 } 4013 dd->disk = NULL; 4014 4015 spin_lock(&rssd_index_lock); 4016 ida_remove(&rssd_index_ida, dd->index); 4017 spin_unlock(&rssd_index_lock); 4018 4019 /* De-initialize the protocol layer. */ 4020 mtip_hw_exit(dd); 4021 4022 return 0; 4023 } 4024 4025 /* 4026 * Function called by the PCI layer when just before the 4027 * machine shuts down. 4028 * 4029 * If a protocol layer shutdown function is present it will be called 4030 * by this function. 4031 * 4032 * @dd Pointer to the driver data structure. 4033 * 4034 * return value 4035 * 0 4036 */ 4037 static int mtip_block_shutdown(struct driver_data *dd) 4038 { 4039 mtip_hw_shutdown(dd); 4040 4041 /* Delete our gendisk structure, and cleanup the blk queue. */ 4042 if (dd->disk) { 4043 dev_info(&dd->pdev->dev, 4044 "Shutting down %s ...\n", dd->disk->disk_name); 4045 4046 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) 4047 del_gendisk(dd->disk); 4048 if (dd->disk->queue) { 4049 blk_cleanup_queue(dd->queue); 4050 blk_mq_free_tag_set(&dd->tags); 4051 } 4052 put_disk(dd->disk); 4053 dd->disk = NULL; 4054 dd->queue = NULL; 4055 } 4056 4057 spin_lock(&rssd_index_lock); 4058 ida_remove(&rssd_index_ida, dd->index); 4059 spin_unlock(&rssd_index_lock); 4060 return 0; 4061 } 4062 4063 static int mtip_block_suspend(struct driver_data *dd) 4064 { 4065 dev_info(&dd->pdev->dev, 4066 "Suspending %s ...\n", dd->disk->disk_name); 4067 mtip_hw_suspend(dd); 4068 return 0; 4069 } 4070 4071 static int mtip_block_resume(struct driver_data *dd) 4072 { 4073 dev_info(&dd->pdev->dev, "Resuming %s ...\n", 4074 dd->disk->disk_name); 4075 mtip_hw_resume(dd); 4076 return 0; 4077 } 4078 4079 static void drop_cpu(int cpu) 4080 { 4081 cpu_use[cpu]--; 4082 } 4083 4084 static int get_least_used_cpu_on_node(int node) 4085 { 4086 int cpu, least_used_cpu, least_cnt; 4087 const struct cpumask *node_mask; 4088 4089 node_mask = cpumask_of_node(node); 4090 least_used_cpu = cpumask_first(node_mask); 4091 least_cnt = cpu_use[least_used_cpu]; 4092 cpu = least_used_cpu; 4093 4094 for_each_cpu(cpu, node_mask) { 4095 if (cpu_use[cpu] < least_cnt) { 4096 least_used_cpu = cpu; 4097 least_cnt = cpu_use[cpu]; 4098 } 4099 } 4100 cpu_use[least_used_cpu]++; 4101 return least_used_cpu; 4102 } 4103 4104 /* Helper for selecting a node in round robin mode */ 4105 static inline int mtip_get_next_rr_node(void) 4106 { 4107 static int next_node = -1; 4108 4109 if (next_node == -1) { 4110 next_node = first_online_node; 4111 return next_node; 4112 } 4113 4114 next_node = next_online_node(next_node); 4115 if (next_node == MAX_NUMNODES) 4116 next_node = first_online_node; 4117 return next_node; 4118 } 4119 4120 static DEFINE_HANDLER(0); 4121 static DEFINE_HANDLER(1); 4122 static DEFINE_HANDLER(2); 4123 static DEFINE_HANDLER(3); 4124 static DEFINE_HANDLER(4); 4125 static DEFINE_HANDLER(5); 4126 static DEFINE_HANDLER(6); 4127 static DEFINE_HANDLER(7); 4128 4129 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) 4130 { 4131 int pos; 4132 unsigned short pcie_dev_ctrl; 4133 4134 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 4135 if (pos) { 4136 pci_read_config_word(pdev, 4137 pos + PCI_EXP_DEVCTL, 4138 &pcie_dev_ctrl); 4139 if (pcie_dev_ctrl & (1 << 11) || 4140 pcie_dev_ctrl & (1 << 4)) { 4141 dev_info(&dd->pdev->dev, 4142 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", 4143 pdev->vendor, pdev->device); 4144 pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN | 4145 PCI_EXP_DEVCTL_RELAX_EN); 4146 pci_write_config_word(pdev, 4147 pos + PCI_EXP_DEVCTL, 4148 pcie_dev_ctrl); 4149 } 4150 } 4151 } 4152 4153 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) 4154 { 4155 /* 4156 * This workaround is specific to AMD/ATI chipset with a PCI upstream 4157 * device with device id 0x5aXX 4158 */ 4159 if (pdev->bus && pdev->bus->self) { 4160 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && 4161 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { 4162 mtip_disable_link_opts(dd, pdev->bus->self); 4163 } else { 4164 /* Check further up the topology */ 4165 struct pci_dev *parent_dev = pdev->bus->self; 4166 if (parent_dev->bus && 4167 parent_dev->bus->parent && 4168 parent_dev->bus->parent->self && 4169 parent_dev->bus->parent->self->vendor == 4170 PCI_VENDOR_ID_ATI && 4171 (parent_dev->bus->parent->self->device & 4172 0xff00) == 0x5a00) { 4173 mtip_disable_link_opts(dd, 4174 parent_dev->bus->parent->self); 4175 } 4176 } 4177 } 4178 } 4179 4180 /* 4181 * Called for each supported PCI device detected. 4182 * 4183 * This function allocates the private data structure, enables the 4184 * PCI device and then calls the block layer initialization function. 4185 * 4186 * return value 4187 * 0 on success else an error code. 4188 */ 4189 static int mtip_pci_probe(struct pci_dev *pdev, 4190 const struct pci_device_id *ent) 4191 { 4192 int rv = 0; 4193 struct driver_data *dd = NULL; 4194 char cpu_list[256]; 4195 const struct cpumask *node_mask; 4196 int cpu, i = 0, j = 0; 4197 int my_node = NUMA_NO_NODE; 4198 unsigned long flags; 4199 4200 /* Allocate memory for this devices private data. */ 4201 my_node = pcibus_to_node(pdev->bus); 4202 if (my_node != NUMA_NO_NODE) { 4203 if (!node_online(my_node)) 4204 my_node = mtip_get_next_rr_node(); 4205 } else { 4206 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); 4207 my_node = mtip_get_next_rr_node(); 4208 } 4209 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", 4210 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), 4211 cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id()); 4212 4213 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); 4214 if (dd == NULL) { 4215 dev_err(&pdev->dev, 4216 "Unable to allocate memory for driver data\n"); 4217 return -ENOMEM; 4218 } 4219 4220 /* Attach the private data to this PCI device. */ 4221 pci_set_drvdata(pdev, dd); 4222 4223 rv = pcim_enable_device(pdev); 4224 if (rv < 0) { 4225 dev_err(&pdev->dev, "Unable to enable device\n"); 4226 goto iomap_err; 4227 } 4228 4229 /* Map BAR5 to memory. */ 4230 rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME); 4231 if (rv < 0) { 4232 dev_err(&pdev->dev, "Unable to map regions\n"); 4233 goto iomap_err; 4234 } 4235 4236 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4237 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4238 4239 if (rv) { 4240 rv = pci_set_consistent_dma_mask(pdev, 4241 DMA_BIT_MASK(32)); 4242 if (rv) { 4243 dev_warn(&pdev->dev, 4244 "64-bit DMA enable failed\n"); 4245 goto setmask_err; 4246 } 4247 } 4248 } 4249 4250 /* Copy the info we may need later into the private data structure. */ 4251 dd->major = mtip_major; 4252 dd->instance = instance; 4253 dd->pdev = pdev; 4254 dd->numa_node = my_node; 4255 4256 INIT_LIST_HEAD(&dd->online_list); 4257 INIT_LIST_HEAD(&dd->remove_list); 4258 4259 memset(dd->workq_name, 0, 32); 4260 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); 4261 4262 dd->isr_workq = create_workqueue(dd->workq_name); 4263 if (!dd->isr_workq) { 4264 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); 4265 rv = -ENOMEM; 4266 goto setmask_err; 4267 } 4268 4269 memset(cpu_list, 0, sizeof(cpu_list)); 4270 4271 node_mask = cpumask_of_node(dd->numa_node); 4272 if (!cpumask_empty(node_mask)) { 4273 for_each_cpu(cpu, node_mask) 4274 { 4275 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); 4276 j = strlen(cpu_list); 4277 } 4278 4279 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", 4280 dd->numa_node, 4281 topology_physical_package_id(cpumask_first(node_mask)), 4282 nr_cpus_node(dd->numa_node), 4283 cpu_list); 4284 } else 4285 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); 4286 4287 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); 4288 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", 4289 cpu_to_node(dd->isr_binding), dd->isr_binding); 4290 4291 /* first worker context always runs in ISR */ 4292 dd->work[0].cpu_binding = dd->isr_binding; 4293 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4294 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4295 dd->work[3].cpu_binding = dd->work[0].cpu_binding; 4296 dd->work[4].cpu_binding = dd->work[1].cpu_binding; 4297 dd->work[5].cpu_binding = dd->work[2].cpu_binding; 4298 dd->work[6].cpu_binding = dd->work[2].cpu_binding; 4299 dd->work[7].cpu_binding = dd->work[1].cpu_binding; 4300 4301 /* Log the bindings */ 4302 for_each_present_cpu(cpu) { 4303 memset(cpu_list, 0, sizeof(cpu_list)); 4304 for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) { 4305 if (dd->work[i].cpu_binding == cpu) { 4306 snprintf(&cpu_list[j], 256 - j, "%d ", i); 4307 j = strlen(cpu_list); 4308 } 4309 } 4310 if (j) 4311 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); 4312 } 4313 4314 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); 4315 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); 4316 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); 4317 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); 4318 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); 4319 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); 4320 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); 4321 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); 4322 4323 pci_set_master(pdev); 4324 rv = pci_enable_msi(pdev); 4325 if (rv) { 4326 dev_warn(&pdev->dev, 4327 "Unable to enable MSI interrupt.\n"); 4328 goto msi_initialize_err; 4329 } 4330 4331 mtip_fix_ero_nosnoop(dd, pdev); 4332 4333 /* Initialize the block layer. */ 4334 rv = mtip_block_initialize(dd); 4335 if (rv < 0) { 4336 dev_err(&pdev->dev, 4337 "Unable to initialize block layer\n"); 4338 goto block_initialize_err; 4339 } 4340 4341 /* 4342 * Increment the instance count so that each device has a unique 4343 * instance number. 4344 */ 4345 instance++; 4346 if (rv != MTIP_FTL_REBUILD_MAGIC) 4347 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 4348 else 4349 rv = 0; /* device in rebuild state, return 0 from probe */ 4350 4351 /* Add to online list even if in ftl rebuild */ 4352 spin_lock_irqsave(&dev_lock, flags); 4353 list_add(&dd->online_list, &online_list); 4354 spin_unlock_irqrestore(&dev_lock, flags); 4355 4356 goto done; 4357 4358 block_initialize_err: 4359 pci_disable_msi(pdev); 4360 4361 msi_initialize_err: 4362 if (dd->isr_workq) { 4363 flush_workqueue(dd->isr_workq); 4364 destroy_workqueue(dd->isr_workq); 4365 drop_cpu(dd->work[0].cpu_binding); 4366 drop_cpu(dd->work[1].cpu_binding); 4367 drop_cpu(dd->work[2].cpu_binding); 4368 } 4369 setmask_err: 4370 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4371 4372 iomap_err: 4373 kfree(dd); 4374 pci_set_drvdata(pdev, NULL); 4375 return rv; 4376 done: 4377 return rv; 4378 } 4379 4380 /* 4381 * Called for each probed device when the device is removed or the 4382 * driver is unloaded. 4383 * 4384 * return value 4385 * None 4386 */ 4387 static void mtip_pci_remove(struct pci_dev *pdev) 4388 { 4389 struct driver_data *dd = pci_get_drvdata(pdev); 4390 unsigned long flags, to; 4391 4392 set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag); 4393 4394 spin_lock_irqsave(&dev_lock, flags); 4395 list_del_init(&dd->online_list); 4396 list_add(&dd->remove_list, &removing_list); 4397 spin_unlock_irqrestore(&dev_lock, flags); 4398 4399 mtip_check_surprise_removal(pdev); 4400 synchronize_irq(dd->pdev->irq); 4401 4402 /* Spin until workers are done */ 4403 to = jiffies + msecs_to_jiffies(4000); 4404 do { 4405 msleep(20); 4406 } while (atomic_read(&dd->irq_workers_active) != 0 && 4407 time_before(jiffies, to)); 4408 4409 if (!dd->sr) 4410 fsync_bdev(dd->bdev); 4411 4412 if (atomic_read(&dd->irq_workers_active) != 0) { 4413 dev_warn(&dd->pdev->dev, 4414 "Completion workers still active!\n"); 4415 } 4416 4417 blk_set_queue_dying(dd->queue); 4418 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); 4419 4420 /* Clean up the block layer. */ 4421 mtip_block_remove(dd); 4422 4423 if (dd->isr_workq) { 4424 flush_workqueue(dd->isr_workq); 4425 destroy_workqueue(dd->isr_workq); 4426 drop_cpu(dd->work[0].cpu_binding); 4427 drop_cpu(dd->work[1].cpu_binding); 4428 drop_cpu(dd->work[2].cpu_binding); 4429 } 4430 4431 pci_disable_msi(pdev); 4432 4433 spin_lock_irqsave(&dev_lock, flags); 4434 list_del_init(&dd->remove_list); 4435 spin_unlock_irqrestore(&dev_lock, flags); 4436 4437 kfree(dd); 4438 4439 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4440 pci_set_drvdata(pdev, NULL); 4441 } 4442 4443 /* 4444 * Called for each probed device when the device is suspended. 4445 * 4446 * return value 4447 * 0 Success 4448 * <0 Error 4449 */ 4450 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) 4451 { 4452 int rv = 0; 4453 struct driver_data *dd = pci_get_drvdata(pdev); 4454 4455 if (!dd) { 4456 dev_err(&pdev->dev, 4457 "Driver private datastructure is NULL\n"); 4458 return -EFAULT; 4459 } 4460 4461 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4462 4463 /* Disable ports & interrupts then send standby immediate */ 4464 rv = mtip_block_suspend(dd); 4465 if (rv < 0) { 4466 dev_err(&pdev->dev, 4467 "Failed to suspend controller\n"); 4468 return rv; 4469 } 4470 4471 /* 4472 * Save the pci config space to pdev structure & 4473 * disable the device 4474 */ 4475 pci_save_state(pdev); 4476 pci_disable_device(pdev); 4477 4478 /* Move to Low power state*/ 4479 pci_set_power_state(pdev, PCI_D3hot); 4480 4481 return rv; 4482 } 4483 4484 /* 4485 * Called for each probed device when the device is resumed. 4486 * 4487 * return value 4488 * 0 Success 4489 * <0 Error 4490 */ 4491 static int mtip_pci_resume(struct pci_dev *pdev) 4492 { 4493 int rv = 0; 4494 struct driver_data *dd; 4495 4496 dd = pci_get_drvdata(pdev); 4497 if (!dd) { 4498 dev_err(&pdev->dev, 4499 "Driver private datastructure is NULL\n"); 4500 return -EFAULT; 4501 } 4502 4503 /* Move the device to active State */ 4504 pci_set_power_state(pdev, PCI_D0); 4505 4506 /* Restore PCI configuration space */ 4507 pci_restore_state(pdev); 4508 4509 /* Enable the PCI device*/ 4510 rv = pcim_enable_device(pdev); 4511 if (rv < 0) { 4512 dev_err(&pdev->dev, 4513 "Failed to enable card during resume\n"); 4514 goto err; 4515 } 4516 pci_set_master(pdev); 4517 4518 /* 4519 * Calls hbaReset, initPort, & startPort function 4520 * then enables interrupts 4521 */ 4522 rv = mtip_block_resume(dd); 4523 if (rv < 0) 4524 dev_err(&pdev->dev, "Unable to resume\n"); 4525 4526 err: 4527 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4528 4529 return rv; 4530 } 4531 4532 /* 4533 * Shutdown routine 4534 * 4535 * return value 4536 * None 4537 */ 4538 static void mtip_pci_shutdown(struct pci_dev *pdev) 4539 { 4540 struct driver_data *dd = pci_get_drvdata(pdev); 4541 if (dd) 4542 mtip_block_shutdown(dd); 4543 } 4544 4545 /* Table of device ids supported by this driver. */ 4546 static const struct pci_device_id mtip_pci_tbl[] = { 4547 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) }, 4548 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) }, 4549 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) }, 4550 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) }, 4551 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) }, 4552 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) }, 4553 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) }, 4554 { 0 } 4555 }; 4556 4557 /* Structure that describes the PCI driver functions. */ 4558 static struct pci_driver mtip_pci_driver = { 4559 .name = MTIP_DRV_NAME, 4560 .id_table = mtip_pci_tbl, 4561 .probe = mtip_pci_probe, 4562 .remove = mtip_pci_remove, 4563 .suspend = mtip_pci_suspend, 4564 .resume = mtip_pci_resume, 4565 .shutdown = mtip_pci_shutdown, 4566 }; 4567 4568 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); 4569 4570 /* 4571 * Module initialization function. 4572 * 4573 * Called once when the module is loaded. This function allocates a major 4574 * block device number to the Cyclone devices and registers the PCI layer 4575 * of the driver. 4576 * 4577 * Return value 4578 * 0 on success else error code. 4579 */ 4580 static int __init mtip_init(void) 4581 { 4582 int error; 4583 4584 pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n"); 4585 4586 spin_lock_init(&dev_lock); 4587 4588 INIT_LIST_HEAD(&online_list); 4589 INIT_LIST_HEAD(&removing_list); 4590 4591 /* Allocate a major block device number to use with this driver. */ 4592 error = register_blkdev(0, MTIP_DRV_NAME); 4593 if (error <= 0) { 4594 pr_err("Unable to register block device (%d)\n", 4595 error); 4596 return -EBUSY; 4597 } 4598 mtip_major = error; 4599 4600 dfs_parent = debugfs_create_dir("rssd", NULL); 4601 if (IS_ERR_OR_NULL(dfs_parent)) { 4602 pr_warn("Error creating debugfs parent\n"); 4603 dfs_parent = NULL; 4604 } 4605 if (dfs_parent) { 4606 dfs_device_status = debugfs_create_file("device_status", 4607 0444, dfs_parent, NULL, 4608 &mtip_device_status_fops); 4609 if (IS_ERR_OR_NULL(dfs_device_status)) { 4610 pr_err("Error creating device_status node\n"); 4611 dfs_device_status = NULL; 4612 } 4613 } 4614 4615 /* Register our PCI operations. */ 4616 error = pci_register_driver(&mtip_pci_driver); 4617 if (error) { 4618 debugfs_remove(dfs_parent); 4619 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4620 } 4621 4622 return error; 4623 } 4624 4625 /* 4626 * Module de-initialization function. 4627 * 4628 * Called once when the module is unloaded. This function deallocates 4629 * the major block device number allocated by mtip_init() and 4630 * unregisters the PCI layer of the driver. 4631 * 4632 * Return value 4633 * none 4634 */ 4635 static void __exit mtip_exit(void) 4636 { 4637 /* Release the allocated major block device number. */ 4638 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4639 4640 /* Unregister the PCI driver. */ 4641 pci_unregister_driver(&mtip_pci_driver); 4642 4643 debugfs_remove_recursive(dfs_parent); 4644 } 4645 4646 MODULE_AUTHOR("Micron Technology, Inc"); 4647 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver"); 4648 MODULE_LICENSE("GPL"); 4649 MODULE_VERSION(MTIP_DRV_VERSION); 4650 4651 module_init(mtip_init); 4652 module_exit(mtip_exit); 4653