1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Block driver for media (i.e., flash cards) 4 * 5 * Copyright 2002 Hewlett-Packard Company 6 * Copyright 2005-2008 Pierre Ossman 7 * 8 * Use consistent with the GNU GPL is permitted, 9 * provided that this copyright notice is 10 * preserved in its entirety in all copies and derived works. 11 * 12 * HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED, 13 * AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS 14 * FITNESS FOR ANY PARTICULAR PURPOSE. 15 * 16 * Many thanks to Alessandro Rubini and Jonathan Corbet! 17 * 18 * Author: Andrew Christian 19 * 28 May 2002 20 */ 21 #include <linux/moduleparam.h> 22 #include <linux/module.h> 23 #include <linux/init.h> 24 25 #include <linux/kernel.h> 26 #include <linux/fs.h> 27 #include <linux/slab.h> 28 #include <linux/errno.h> 29 #include <linux/hdreg.h> 30 #include <linux/kdev_t.h> 31 #include <linux/kref.h> 32 #include <linux/blkdev.h> 33 #include <linux/cdev.h> 34 #include <linux/mutex.h> 35 #include <linux/scatterlist.h> 36 #include <linux/string_helpers.h> 37 #include <linux/delay.h> 38 #include <linux/capability.h> 39 #include <linux/compat.h> 40 #include <linux/pm_runtime.h> 41 #include <linux/idr.h> 42 #include <linux/debugfs.h> 43 44 #include <linux/mmc/ioctl.h> 45 #include <linux/mmc/card.h> 46 #include <linux/mmc/host.h> 47 #include <linux/mmc/mmc.h> 48 #include <linux/mmc/sd.h> 49 50 #include <linux/uaccess.h> 51 52 #include "queue.h" 53 #include "block.h" 54 #include "core.h" 55 #include "card.h" 56 #include "crypto.h" 57 #include "host.h" 58 #include "bus.h" 59 #include "mmc_ops.h" 60 #include "quirks.h" 61 #include "sd_ops.h" 62 63 MODULE_ALIAS("mmc:block"); 64 #ifdef MODULE_PARAM_PREFIX 65 #undef MODULE_PARAM_PREFIX 66 #endif 67 #define MODULE_PARAM_PREFIX "mmcblk." 68 69 /* 70 * Set a 10 second timeout for polling write request busy state. Note, mmc core 71 * is setting a 3 second timeout for SD cards, and SDHCI has long had a 10 72 * second software timer to timeout the whole request, so 10 seconds should be 73 * ample. 74 */ 75 #define MMC_BLK_TIMEOUT_MS (10 * 1000) 76 #define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16) 77 #define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8) 78 79 static DEFINE_MUTEX(block_mutex); 80 81 /* 82 * The defaults come from config options but can be overriden by module 83 * or bootarg options. 84 */ 85 static int perdev_minors = CONFIG_MMC_BLOCK_MINORS; 86 87 /* 88 * We've only got one major, so number of mmcblk devices is 89 * limited to (1 << 20) / number of minors per device. It is also 90 * limited by the MAX_DEVICES below. 91 */ 92 static int max_devices; 93 94 #define MAX_DEVICES 256 95 96 static DEFINE_IDA(mmc_blk_ida); 97 static DEFINE_IDA(mmc_rpmb_ida); 98 99 struct mmc_blk_busy_data { 100 struct mmc_card *card; 101 u32 status; 102 }; 103 104 /* 105 * There is one mmc_blk_data per slot. 106 */ 107 struct mmc_blk_data { 108 struct device *parent; 109 struct gendisk *disk; 110 struct mmc_queue queue; 111 struct list_head part; 112 struct list_head rpmbs; 113 114 unsigned int flags; 115 #define MMC_BLK_CMD23 (1 << 0) /* Can do SET_BLOCK_COUNT for multiblock */ 116 #define MMC_BLK_REL_WR (1 << 1) /* MMC Reliable write support */ 117 118 struct kref kref; 119 unsigned int read_only; 120 unsigned int part_type; 121 unsigned int reset_done; 122 #define MMC_BLK_READ BIT(0) 123 #define MMC_BLK_WRITE BIT(1) 124 #define MMC_BLK_DISCARD BIT(2) 125 #define MMC_BLK_SECDISCARD BIT(3) 126 #define MMC_BLK_CQE_RECOVERY BIT(4) 127 #define MMC_BLK_TRIM BIT(5) 128 129 /* 130 * Only set in main mmc_blk_data associated 131 * with mmc_card with dev_set_drvdata, and keeps 132 * track of the current selected device partition. 133 */ 134 unsigned int part_curr; 135 #define MMC_BLK_PART_INVALID UINT_MAX /* Unknown partition active */ 136 int area_type; 137 138 /* debugfs files (only in main mmc_blk_data) */ 139 struct dentry *status_dentry; 140 struct dentry *ext_csd_dentry; 141 }; 142 143 /* Device type for RPMB character devices */ 144 static dev_t mmc_rpmb_devt; 145 146 /* Bus type for RPMB character devices */ 147 static struct bus_type mmc_rpmb_bus_type = { 148 .name = "mmc_rpmb", 149 }; 150 151 /** 152 * struct mmc_rpmb_data - special RPMB device type for these areas 153 * @dev: the device for the RPMB area 154 * @chrdev: character device for the RPMB area 155 * @id: unique device ID number 156 * @part_index: partition index (0 on first) 157 * @md: parent MMC block device 158 * @node: list item, so we can put this device on a list 159 */ 160 struct mmc_rpmb_data { 161 struct device dev; 162 struct cdev chrdev; 163 int id; 164 unsigned int part_index; 165 struct mmc_blk_data *md; 166 struct list_head node; 167 }; 168 169 static DEFINE_MUTEX(open_lock); 170 171 module_param(perdev_minors, int, 0444); 172 MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device"); 173 174 static inline int mmc_blk_part_switch(struct mmc_card *card, 175 unsigned int part_type); 176 static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq, 177 struct mmc_card *card, 178 int recovery_mode, 179 struct mmc_queue *mq); 180 static void mmc_blk_hsq_req_done(struct mmc_request *mrq); 181 static int mmc_spi_err_check(struct mmc_card *card); 182 static int mmc_blk_busy_cb(void *cb_data, bool *busy); 183 184 static struct mmc_blk_data *mmc_blk_get(struct gendisk *disk) 185 { 186 struct mmc_blk_data *md; 187 188 mutex_lock(&open_lock); 189 md = disk->private_data; 190 if (md && !kref_get_unless_zero(&md->kref)) 191 md = NULL; 192 mutex_unlock(&open_lock); 193 194 return md; 195 } 196 197 static inline int mmc_get_devidx(struct gendisk *disk) 198 { 199 int devidx = disk->first_minor / perdev_minors; 200 return devidx; 201 } 202 203 static void mmc_blk_kref_release(struct kref *ref) 204 { 205 struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref); 206 int devidx; 207 208 devidx = mmc_get_devidx(md->disk); 209 ida_simple_remove(&mmc_blk_ida, devidx); 210 211 mutex_lock(&open_lock); 212 md->disk->private_data = NULL; 213 mutex_unlock(&open_lock); 214 215 put_disk(md->disk); 216 kfree(md); 217 } 218 219 static void mmc_blk_put(struct mmc_blk_data *md) 220 { 221 kref_put(&md->kref, mmc_blk_kref_release); 222 } 223 224 static ssize_t power_ro_lock_show(struct device *dev, 225 struct device_attribute *attr, char *buf) 226 { 227 int ret; 228 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 229 struct mmc_card *card = md->queue.card; 230 int locked = 0; 231 232 if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN) 233 locked = 2; 234 else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN) 235 locked = 1; 236 237 ret = snprintf(buf, PAGE_SIZE, "%d\n", locked); 238 239 mmc_blk_put(md); 240 241 return ret; 242 } 243 244 static ssize_t power_ro_lock_store(struct device *dev, 245 struct device_attribute *attr, const char *buf, size_t count) 246 { 247 int ret; 248 struct mmc_blk_data *md, *part_md; 249 struct mmc_queue *mq; 250 struct request *req; 251 unsigned long set; 252 253 if (kstrtoul(buf, 0, &set)) 254 return -EINVAL; 255 256 if (set != 1) 257 return count; 258 259 md = mmc_blk_get(dev_to_disk(dev)); 260 mq = &md->queue; 261 262 /* Dispatch locking to the block layer */ 263 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0); 264 if (IS_ERR(req)) { 265 count = PTR_ERR(req); 266 goto out_put; 267 } 268 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP; 269 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 270 blk_execute_rq(req, false); 271 ret = req_to_mmc_queue_req(req)->drv_op_result; 272 blk_mq_free_request(req); 273 274 if (!ret) { 275 pr_info("%s: Locking boot partition ro until next power on\n", 276 md->disk->disk_name); 277 set_disk_ro(md->disk, 1); 278 279 list_for_each_entry(part_md, &md->part, part) 280 if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) { 281 pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name); 282 set_disk_ro(part_md->disk, 1); 283 } 284 } 285 out_put: 286 mmc_blk_put(md); 287 return count; 288 } 289 290 static DEVICE_ATTR(ro_lock_until_next_power_on, 0, 291 power_ro_lock_show, power_ro_lock_store); 292 293 static ssize_t force_ro_show(struct device *dev, struct device_attribute *attr, 294 char *buf) 295 { 296 int ret; 297 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 298 299 ret = snprintf(buf, PAGE_SIZE, "%d\n", 300 get_disk_ro(dev_to_disk(dev)) ^ 301 md->read_only); 302 mmc_blk_put(md); 303 return ret; 304 } 305 306 static ssize_t force_ro_store(struct device *dev, struct device_attribute *attr, 307 const char *buf, size_t count) 308 { 309 int ret; 310 char *end; 311 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 312 unsigned long set = simple_strtoul(buf, &end, 0); 313 if (end == buf) { 314 ret = -EINVAL; 315 goto out; 316 } 317 318 set_disk_ro(dev_to_disk(dev), set || md->read_only); 319 ret = count; 320 out: 321 mmc_blk_put(md); 322 return ret; 323 } 324 325 static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store); 326 327 static struct attribute *mmc_disk_attrs[] = { 328 &dev_attr_force_ro.attr, 329 &dev_attr_ro_lock_until_next_power_on.attr, 330 NULL, 331 }; 332 333 static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj, 334 struct attribute *a, int n) 335 { 336 struct device *dev = kobj_to_dev(kobj); 337 struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev)); 338 umode_t mode = a->mode; 339 340 if (a == &dev_attr_ro_lock_until_next_power_on.attr && 341 (md->area_type & MMC_BLK_DATA_AREA_BOOT) && 342 md->queue.card->ext_csd.boot_ro_lockable) { 343 mode = S_IRUGO; 344 if (!(md->queue.card->ext_csd.boot_ro_lock & 345 EXT_CSD_BOOT_WP_B_PWR_WP_DIS)) 346 mode |= S_IWUSR; 347 } 348 349 mmc_blk_put(md); 350 return mode; 351 } 352 353 static const struct attribute_group mmc_disk_attr_group = { 354 .is_visible = mmc_disk_attrs_is_visible, 355 .attrs = mmc_disk_attrs, 356 }; 357 358 static const struct attribute_group *mmc_disk_attr_groups[] = { 359 &mmc_disk_attr_group, 360 NULL, 361 }; 362 363 static int mmc_blk_open(struct gendisk *disk, blk_mode_t mode) 364 { 365 struct mmc_blk_data *md = mmc_blk_get(disk); 366 int ret = -ENXIO; 367 368 mutex_lock(&block_mutex); 369 if (md) { 370 ret = 0; 371 if ((mode & BLK_OPEN_WRITE) && md->read_only) { 372 mmc_blk_put(md); 373 ret = -EROFS; 374 } 375 } 376 mutex_unlock(&block_mutex); 377 378 return ret; 379 } 380 381 static void mmc_blk_release(struct gendisk *disk) 382 { 383 struct mmc_blk_data *md = disk->private_data; 384 385 mutex_lock(&block_mutex); 386 mmc_blk_put(md); 387 mutex_unlock(&block_mutex); 388 } 389 390 static int 391 mmc_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 392 { 393 geo->cylinders = get_capacity(bdev->bd_disk) / (4 * 16); 394 geo->heads = 4; 395 geo->sectors = 16; 396 return 0; 397 } 398 399 struct mmc_blk_ioc_data { 400 struct mmc_ioc_cmd ic; 401 unsigned char *buf; 402 u64 buf_bytes; 403 unsigned int flags; 404 #define MMC_BLK_IOC_DROP BIT(0) /* drop this mrq */ 405 #define MMC_BLK_IOC_SBC BIT(1) /* use mrq.sbc */ 406 407 struct mmc_rpmb_data *rpmb; 408 }; 409 410 static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user( 411 struct mmc_ioc_cmd __user *user) 412 { 413 struct mmc_blk_ioc_data *idata; 414 int err; 415 416 idata = kmalloc(sizeof(*idata), GFP_KERNEL); 417 if (!idata) { 418 err = -ENOMEM; 419 goto out; 420 } 421 422 if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) { 423 err = -EFAULT; 424 goto idata_err; 425 } 426 427 idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks; 428 if (idata->buf_bytes > MMC_IOC_MAX_BYTES) { 429 err = -EOVERFLOW; 430 goto idata_err; 431 } 432 433 if (!idata->buf_bytes) { 434 idata->buf = NULL; 435 return idata; 436 } 437 438 idata->buf = memdup_user((void __user *)(unsigned long) 439 idata->ic.data_ptr, idata->buf_bytes); 440 if (IS_ERR(idata->buf)) { 441 err = PTR_ERR(idata->buf); 442 goto idata_err; 443 } 444 445 return idata; 446 447 idata_err: 448 kfree(idata); 449 out: 450 return ERR_PTR(err); 451 } 452 453 static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr, 454 struct mmc_blk_ioc_data *idata) 455 { 456 struct mmc_ioc_cmd *ic = &idata->ic; 457 458 if (copy_to_user(&(ic_ptr->response), ic->response, 459 sizeof(ic->response))) 460 return -EFAULT; 461 462 if (!idata->ic.write_flag) { 463 if (copy_to_user((void __user *)(unsigned long)ic->data_ptr, 464 idata->buf, idata->buf_bytes)) 465 return -EFAULT; 466 } 467 468 return 0; 469 } 470 471 static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md, 472 struct mmc_blk_ioc_data **idatas, int i) 473 { 474 struct mmc_command cmd = {}, sbc = {}; 475 struct mmc_data data = {}; 476 struct mmc_request mrq = {}; 477 struct scatterlist sg; 478 bool r1b_resp; 479 unsigned int busy_timeout_ms; 480 int err; 481 unsigned int target_part; 482 struct mmc_blk_ioc_data *idata = idatas[i]; 483 struct mmc_blk_ioc_data *prev_idata = NULL; 484 485 if (!card || !md || !idata) 486 return -EINVAL; 487 488 if (idata->flags & MMC_BLK_IOC_DROP) 489 return 0; 490 491 if (idata->flags & MMC_BLK_IOC_SBC) 492 prev_idata = idatas[i - 1]; 493 494 /* 495 * The RPMB accesses comes in from the character device, so we 496 * need to target these explicitly. Else we just target the 497 * partition type for the block device the ioctl() was issued 498 * on. 499 */ 500 if (idata->rpmb) { 501 /* Support multiple RPMB partitions */ 502 target_part = idata->rpmb->part_index; 503 target_part |= EXT_CSD_PART_CONFIG_ACC_RPMB; 504 } else { 505 target_part = md->part_type; 506 } 507 508 cmd.opcode = idata->ic.opcode; 509 cmd.arg = idata->ic.arg; 510 cmd.flags = idata->ic.flags; 511 512 if (idata->buf_bytes) { 513 data.sg = &sg; 514 data.sg_len = 1; 515 data.blksz = idata->ic.blksz; 516 data.blocks = idata->ic.blocks; 517 518 sg_init_one(data.sg, idata->buf, idata->buf_bytes); 519 520 if (idata->ic.write_flag) 521 data.flags = MMC_DATA_WRITE; 522 else 523 data.flags = MMC_DATA_READ; 524 525 /* data.flags must already be set before doing this. */ 526 mmc_set_data_timeout(&data, card); 527 528 /* Allow overriding the timeout_ns for empirical tuning. */ 529 if (idata->ic.data_timeout_ns) 530 data.timeout_ns = idata->ic.data_timeout_ns; 531 532 mrq.data = &data; 533 } 534 535 mrq.cmd = &cmd; 536 537 err = mmc_blk_part_switch(card, target_part); 538 if (err) 539 return err; 540 541 if (idata->ic.is_acmd) { 542 err = mmc_app_cmd(card->host, card); 543 if (err) 544 return err; 545 } 546 547 if (idata->rpmb || prev_idata) { 548 sbc.opcode = MMC_SET_BLOCK_COUNT; 549 /* 550 * We don't do any blockcount validation because the max size 551 * may be increased by a future standard. We just copy the 552 * 'Reliable Write' bit here. 553 */ 554 sbc.arg = data.blocks | (idata->ic.write_flag & BIT(31)); 555 if (prev_idata) 556 sbc.arg = prev_idata->ic.arg; 557 sbc.flags = MMC_RSP_R1 | MMC_CMD_AC; 558 mrq.sbc = &sbc; 559 } 560 561 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) && 562 (cmd.opcode == MMC_SWITCH)) 563 return mmc_sanitize(card, idata->ic.cmd_timeout_ms); 564 565 /* If it's an R1B response we need some more preparations. */ 566 busy_timeout_ms = idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS; 567 r1b_resp = (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B; 568 if (r1b_resp) 569 mmc_prepare_busy_cmd(card->host, &cmd, busy_timeout_ms); 570 571 mmc_wait_for_req(card->host, &mrq); 572 memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp)); 573 574 if (prev_idata) { 575 memcpy(&prev_idata->ic.response, sbc.resp, sizeof(sbc.resp)); 576 if (sbc.error) { 577 dev_err(mmc_dev(card->host), "%s: sbc error %d\n", 578 __func__, sbc.error); 579 return sbc.error; 580 } 581 } 582 583 if (cmd.error) { 584 dev_err(mmc_dev(card->host), "%s: cmd error %d\n", 585 __func__, cmd.error); 586 return cmd.error; 587 } 588 if (data.error) { 589 dev_err(mmc_dev(card->host), "%s: data error %d\n", 590 __func__, data.error); 591 return data.error; 592 } 593 594 /* 595 * Make sure the cache of the PARTITION_CONFIG register and 596 * PARTITION_ACCESS bits is updated in case the ioctl ext_csd write 597 * changed it successfully. 598 */ 599 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) && 600 (cmd.opcode == MMC_SWITCH)) { 601 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev); 602 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg); 603 604 /* 605 * Update cache so the next mmc_blk_part_switch call operates 606 * on up-to-date data. 607 */ 608 card->ext_csd.part_config = value; 609 main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK; 610 } 611 612 /* 613 * Make sure to update CACHE_CTRL in case it was changed. The cache 614 * will get turned back on if the card is re-initialized, e.g. 615 * suspend/resume or hw reset in recovery. 616 */ 617 if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) && 618 (cmd.opcode == MMC_SWITCH)) { 619 u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1; 620 621 card->ext_csd.cache_ctrl = value; 622 } 623 624 /* 625 * According to the SD specs, some commands require a delay after 626 * issuing the command. 627 */ 628 if (idata->ic.postsleep_min_us) 629 usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us); 630 631 if (mmc_host_is_spi(card->host)) { 632 if (idata->ic.write_flag || r1b_resp || cmd.flags & MMC_RSP_SPI_BUSY) 633 return mmc_spi_err_check(card); 634 return err; 635 } 636 637 /* 638 * Ensure RPMB, writes and R1B responses are completed by polling with 639 * CMD13. Note that, usually we don't need to poll when using HW busy 640 * detection, but here it's needed since some commands may indicate the 641 * error through the R1 status bits. 642 */ 643 if (idata->rpmb || idata->ic.write_flag || r1b_resp) { 644 struct mmc_blk_busy_data cb_data = { 645 .card = card, 646 }; 647 648 err = __mmc_poll_for_busy(card->host, 0, busy_timeout_ms, 649 &mmc_blk_busy_cb, &cb_data); 650 651 idata->ic.response[0] = cb_data.status; 652 } 653 654 return err; 655 } 656 657 static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md, 658 struct mmc_ioc_cmd __user *ic_ptr, 659 struct mmc_rpmb_data *rpmb) 660 { 661 struct mmc_blk_ioc_data *idata; 662 struct mmc_blk_ioc_data *idatas[1]; 663 struct mmc_queue *mq; 664 struct mmc_card *card; 665 int err = 0, ioc_err = 0; 666 struct request *req; 667 668 idata = mmc_blk_ioctl_copy_from_user(ic_ptr); 669 if (IS_ERR(idata)) 670 return PTR_ERR(idata); 671 /* This will be NULL on non-RPMB ioctl():s */ 672 idata->rpmb = rpmb; 673 674 card = md->queue.card; 675 if (IS_ERR(card)) { 676 err = PTR_ERR(card); 677 goto cmd_done; 678 } 679 680 /* 681 * Dispatch the ioctl() into the block request queue. 682 */ 683 mq = &md->queue; 684 req = blk_mq_alloc_request(mq->queue, 685 idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); 686 if (IS_ERR(req)) { 687 err = PTR_ERR(req); 688 goto cmd_done; 689 } 690 idatas[0] = idata; 691 req_to_mmc_queue_req(req)->drv_op = 692 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; 693 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 694 req_to_mmc_queue_req(req)->drv_op_data = idatas; 695 req_to_mmc_queue_req(req)->ioc_count = 1; 696 blk_execute_rq(req, false); 697 ioc_err = req_to_mmc_queue_req(req)->drv_op_result; 698 err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata); 699 blk_mq_free_request(req); 700 701 cmd_done: 702 kfree(idata->buf); 703 kfree(idata); 704 return ioc_err ? ioc_err : err; 705 } 706 707 static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md, 708 struct mmc_ioc_multi_cmd __user *user, 709 struct mmc_rpmb_data *rpmb) 710 { 711 struct mmc_blk_ioc_data **idata = NULL; 712 struct mmc_ioc_cmd __user *cmds = user->cmds; 713 struct mmc_card *card; 714 struct mmc_queue *mq; 715 int err = 0, ioc_err = 0; 716 __u64 num_of_cmds; 717 unsigned int i, n; 718 struct request *req; 719 720 if (copy_from_user(&num_of_cmds, &user->num_of_cmds, 721 sizeof(num_of_cmds))) 722 return -EFAULT; 723 724 if (!num_of_cmds) 725 return 0; 726 727 if (num_of_cmds > MMC_IOC_MAX_CMDS) 728 return -EINVAL; 729 730 n = num_of_cmds; 731 idata = kcalloc(n, sizeof(*idata), GFP_KERNEL); 732 if (!idata) 733 return -ENOMEM; 734 735 for (i = 0; i < n; i++) { 736 idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]); 737 if (IS_ERR(idata[i])) { 738 err = PTR_ERR(idata[i]); 739 n = i; 740 goto cmd_err; 741 } 742 /* This will be NULL on non-RPMB ioctl():s */ 743 idata[i]->rpmb = rpmb; 744 } 745 746 card = md->queue.card; 747 if (IS_ERR(card)) { 748 err = PTR_ERR(card); 749 goto cmd_err; 750 } 751 752 753 /* 754 * Dispatch the ioctl()s into the block request queue. 755 */ 756 mq = &md->queue; 757 req = blk_mq_alloc_request(mq->queue, 758 idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0); 759 if (IS_ERR(req)) { 760 err = PTR_ERR(req); 761 goto cmd_err; 762 } 763 req_to_mmc_queue_req(req)->drv_op = 764 rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL; 765 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 766 req_to_mmc_queue_req(req)->drv_op_data = idata; 767 req_to_mmc_queue_req(req)->ioc_count = n; 768 blk_execute_rq(req, false); 769 ioc_err = req_to_mmc_queue_req(req)->drv_op_result; 770 771 /* copy to user if data and response */ 772 for (i = 0; i < n && !err; i++) 773 err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]); 774 775 blk_mq_free_request(req); 776 777 cmd_err: 778 for (i = 0; i < n; i++) { 779 kfree(idata[i]->buf); 780 kfree(idata[i]); 781 } 782 kfree(idata); 783 return ioc_err ? ioc_err : err; 784 } 785 786 static int mmc_blk_check_blkdev(struct block_device *bdev) 787 { 788 /* 789 * The caller must have CAP_SYS_RAWIO, and must be calling this on the 790 * whole block device, not on a partition. This prevents overspray 791 * between sibling partitions. 792 */ 793 if (!capable(CAP_SYS_RAWIO) || bdev_is_partition(bdev)) 794 return -EPERM; 795 return 0; 796 } 797 798 static int mmc_blk_ioctl(struct block_device *bdev, blk_mode_t mode, 799 unsigned int cmd, unsigned long arg) 800 { 801 struct mmc_blk_data *md; 802 int ret; 803 804 switch (cmd) { 805 case MMC_IOC_CMD: 806 ret = mmc_blk_check_blkdev(bdev); 807 if (ret) 808 return ret; 809 md = mmc_blk_get(bdev->bd_disk); 810 if (!md) 811 return -EINVAL; 812 ret = mmc_blk_ioctl_cmd(md, 813 (struct mmc_ioc_cmd __user *)arg, 814 NULL); 815 mmc_blk_put(md); 816 return ret; 817 case MMC_IOC_MULTI_CMD: 818 ret = mmc_blk_check_blkdev(bdev); 819 if (ret) 820 return ret; 821 md = mmc_blk_get(bdev->bd_disk); 822 if (!md) 823 return -EINVAL; 824 ret = mmc_blk_ioctl_multi_cmd(md, 825 (struct mmc_ioc_multi_cmd __user *)arg, 826 NULL); 827 mmc_blk_put(md); 828 return ret; 829 default: 830 return -EINVAL; 831 } 832 } 833 834 #ifdef CONFIG_COMPAT 835 static int mmc_blk_compat_ioctl(struct block_device *bdev, blk_mode_t mode, 836 unsigned int cmd, unsigned long arg) 837 { 838 return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg)); 839 } 840 #endif 841 842 static int mmc_blk_alternative_gpt_sector(struct gendisk *disk, 843 sector_t *sector) 844 { 845 struct mmc_blk_data *md; 846 int ret; 847 848 md = mmc_blk_get(disk); 849 if (!md) 850 return -EINVAL; 851 852 if (md->queue.card) 853 ret = mmc_card_alternative_gpt_sector(md->queue.card, sector); 854 else 855 ret = -ENODEV; 856 857 mmc_blk_put(md); 858 859 return ret; 860 } 861 862 static const struct block_device_operations mmc_bdops = { 863 .open = mmc_blk_open, 864 .release = mmc_blk_release, 865 .getgeo = mmc_blk_getgeo, 866 .owner = THIS_MODULE, 867 .ioctl = mmc_blk_ioctl, 868 #ifdef CONFIG_COMPAT 869 .compat_ioctl = mmc_blk_compat_ioctl, 870 #endif 871 .alternative_gpt_sector = mmc_blk_alternative_gpt_sector, 872 }; 873 874 static int mmc_blk_part_switch_pre(struct mmc_card *card, 875 unsigned int part_type) 876 { 877 const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; 878 int ret = 0; 879 880 if ((part_type & mask) == mask) { 881 if (card->ext_csd.cmdq_en) { 882 ret = mmc_cmdq_disable(card); 883 if (ret) 884 return ret; 885 } 886 mmc_retune_pause(card->host); 887 } 888 889 return ret; 890 } 891 892 static int mmc_blk_part_switch_post(struct mmc_card *card, 893 unsigned int part_type) 894 { 895 const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_RPMB; 896 int ret = 0; 897 898 if ((part_type & mask) == mask) { 899 mmc_retune_unpause(card->host); 900 if (card->reenable_cmdq && !card->ext_csd.cmdq_en) 901 ret = mmc_cmdq_enable(card); 902 } 903 904 return ret; 905 } 906 907 static inline int mmc_blk_part_switch(struct mmc_card *card, 908 unsigned int part_type) 909 { 910 int ret = 0; 911 struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev); 912 913 if (main_md->part_curr == part_type) 914 return 0; 915 916 if (mmc_card_mmc(card)) { 917 u8 part_config = card->ext_csd.part_config; 918 919 ret = mmc_blk_part_switch_pre(card, part_type); 920 if (ret) 921 return ret; 922 923 part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK; 924 part_config |= part_type; 925 926 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 927 EXT_CSD_PART_CONFIG, part_config, 928 card->ext_csd.part_time); 929 if (ret) { 930 mmc_blk_part_switch_post(card, part_type); 931 return ret; 932 } 933 934 card->ext_csd.part_config = part_config; 935 936 ret = mmc_blk_part_switch_post(card, main_md->part_curr); 937 } 938 939 main_md->part_curr = part_type; 940 return ret; 941 } 942 943 static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks) 944 { 945 int err; 946 u32 result; 947 __be32 *blocks; 948 949 struct mmc_request mrq = {}; 950 struct mmc_command cmd = {}; 951 struct mmc_data data = {}; 952 953 struct scatterlist sg; 954 955 err = mmc_app_cmd(card->host, card); 956 if (err) 957 return err; 958 959 cmd.opcode = SD_APP_SEND_NUM_WR_BLKS; 960 cmd.arg = 0; 961 cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC; 962 963 data.blksz = 4; 964 data.blocks = 1; 965 data.flags = MMC_DATA_READ; 966 data.sg = &sg; 967 data.sg_len = 1; 968 mmc_set_data_timeout(&data, card); 969 970 mrq.cmd = &cmd; 971 mrq.data = &data; 972 973 blocks = kmalloc(4, GFP_KERNEL); 974 if (!blocks) 975 return -ENOMEM; 976 977 sg_init_one(&sg, blocks, 4); 978 979 mmc_wait_for_req(card->host, &mrq); 980 981 result = ntohl(*blocks); 982 kfree(blocks); 983 984 if (cmd.error || data.error) 985 return -EIO; 986 987 *written_blocks = result; 988 989 return 0; 990 } 991 992 static unsigned int mmc_blk_clock_khz(struct mmc_host *host) 993 { 994 if (host->actual_clock) 995 return host->actual_clock / 1000; 996 997 /* Clock may be subject to a divisor, fudge it by a factor of 2. */ 998 if (host->ios.clock) 999 return host->ios.clock / 2000; 1000 1001 /* How can there be no clock */ 1002 WARN_ON_ONCE(1); 1003 return 100; /* 100 kHz is minimum possible value */ 1004 } 1005 1006 static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host, 1007 struct mmc_data *data) 1008 { 1009 unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000); 1010 unsigned int khz; 1011 1012 if (data->timeout_clks) { 1013 khz = mmc_blk_clock_khz(host); 1014 ms += DIV_ROUND_UP(data->timeout_clks, khz); 1015 } 1016 1017 return ms; 1018 } 1019 1020 /* 1021 * Attempts to reset the card and get back to the requested partition. 1022 * Therefore any error here must result in cancelling the block layer 1023 * request, it must not be reattempted without going through the mmc_blk 1024 * partition sanity checks. 1025 */ 1026 static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host, 1027 int type) 1028 { 1029 int err; 1030 struct mmc_blk_data *main_md = dev_get_drvdata(&host->card->dev); 1031 1032 if (md->reset_done & type) 1033 return -EEXIST; 1034 1035 md->reset_done |= type; 1036 err = mmc_hw_reset(host->card); 1037 /* 1038 * A successful reset will leave the card in the main partition, but 1039 * upon failure it might not be, so set it to MMC_BLK_PART_INVALID 1040 * in that case. 1041 */ 1042 main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type; 1043 if (err) 1044 return err; 1045 /* Ensure we switch back to the correct partition */ 1046 if (mmc_blk_part_switch(host->card, md->part_type)) 1047 /* 1048 * We have failed to get back into the correct 1049 * partition, so we need to abort the whole request. 1050 */ 1051 return -ENODEV; 1052 return 0; 1053 } 1054 1055 static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type) 1056 { 1057 md->reset_done &= ~type; 1058 } 1059 1060 static void mmc_blk_check_sbc(struct mmc_queue_req *mq_rq) 1061 { 1062 struct mmc_blk_ioc_data **idata = mq_rq->drv_op_data; 1063 int i; 1064 1065 for (i = 1; i < mq_rq->ioc_count; i++) { 1066 if (idata[i - 1]->ic.opcode == MMC_SET_BLOCK_COUNT && 1067 mmc_op_multi(idata[i]->ic.opcode)) { 1068 idata[i - 1]->flags |= MMC_BLK_IOC_DROP; 1069 idata[i]->flags |= MMC_BLK_IOC_SBC; 1070 } 1071 } 1072 } 1073 1074 /* 1075 * The non-block commands come back from the block layer after it queued it and 1076 * processed it with all other requests and then they get issued in this 1077 * function. 1078 */ 1079 static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req) 1080 { 1081 struct mmc_queue_req *mq_rq; 1082 struct mmc_card *card = mq->card; 1083 struct mmc_blk_data *md = mq->blkdata; 1084 struct mmc_blk_ioc_data **idata; 1085 bool rpmb_ioctl; 1086 u8 **ext_csd; 1087 u32 status; 1088 int ret; 1089 int i; 1090 1091 mq_rq = req_to_mmc_queue_req(req); 1092 rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB); 1093 1094 switch (mq_rq->drv_op) { 1095 case MMC_DRV_OP_IOCTL: 1096 if (card->ext_csd.cmdq_en) { 1097 ret = mmc_cmdq_disable(card); 1098 if (ret) 1099 break; 1100 } 1101 1102 mmc_blk_check_sbc(mq_rq); 1103 1104 fallthrough; 1105 case MMC_DRV_OP_IOCTL_RPMB: 1106 idata = mq_rq->drv_op_data; 1107 for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) { 1108 ret = __mmc_blk_ioctl_cmd(card, md, idata, i); 1109 if (ret) 1110 break; 1111 } 1112 /* Always switch back to main area after RPMB access */ 1113 if (rpmb_ioctl) 1114 mmc_blk_part_switch(card, 0); 1115 else if (card->reenable_cmdq && !card->ext_csd.cmdq_en) 1116 mmc_cmdq_enable(card); 1117 break; 1118 case MMC_DRV_OP_BOOT_WP: 1119 ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP, 1120 card->ext_csd.boot_ro_lock | 1121 EXT_CSD_BOOT_WP_B_PWR_WP_EN, 1122 card->ext_csd.part_time); 1123 if (ret) 1124 pr_err("%s: Locking boot partition ro until next power on failed: %d\n", 1125 md->disk->disk_name, ret); 1126 else 1127 card->ext_csd.boot_ro_lock |= 1128 EXT_CSD_BOOT_WP_B_PWR_WP_EN; 1129 break; 1130 case MMC_DRV_OP_GET_CARD_STATUS: 1131 ret = mmc_send_status(card, &status); 1132 if (!ret) 1133 ret = status; 1134 break; 1135 case MMC_DRV_OP_GET_EXT_CSD: 1136 ext_csd = mq_rq->drv_op_data; 1137 ret = mmc_get_ext_csd(card, ext_csd); 1138 break; 1139 default: 1140 pr_err("%s: unknown driver specific operation\n", 1141 md->disk->disk_name); 1142 ret = -EINVAL; 1143 break; 1144 } 1145 mq_rq->drv_op_result = ret; 1146 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK); 1147 } 1148 1149 static void mmc_blk_issue_erase_rq(struct mmc_queue *mq, struct request *req, 1150 int type, unsigned int erase_arg) 1151 { 1152 struct mmc_blk_data *md = mq->blkdata; 1153 struct mmc_card *card = md->queue.card; 1154 unsigned int from, nr; 1155 int err = 0; 1156 blk_status_t status = BLK_STS_OK; 1157 1158 if (!mmc_can_erase(card)) { 1159 status = BLK_STS_NOTSUPP; 1160 goto fail; 1161 } 1162 1163 from = blk_rq_pos(req); 1164 nr = blk_rq_sectors(req); 1165 1166 do { 1167 err = 0; 1168 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1169 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1170 INAND_CMD38_ARG_EXT_CSD, 1171 erase_arg == MMC_TRIM_ARG ? 1172 INAND_CMD38_ARG_TRIM : 1173 INAND_CMD38_ARG_ERASE, 1174 card->ext_csd.generic_cmd6_time); 1175 } 1176 if (!err) 1177 err = mmc_erase(card, from, nr, erase_arg); 1178 } while (err == -EIO && !mmc_blk_reset(md, card->host, type)); 1179 if (err) 1180 status = BLK_STS_IOERR; 1181 else 1182 mmc_blk_reset_success(md, type); 1183 fail: 1184 blk_mq_end_request(req, status); 1185 } 1186 1187 static void mmc_blk_issue_trim_rq(struct mmc_queue *mq, struct request *req) 1188 { 1189 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG); 1190 } 1191 1192 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req) 1193 { 1194 struct mmc_blk_data *md = mq->blkdata; 1195 struct mmc_card *card = md->queue.card; 1196 unsigned int arg = card->erase_arg; 1197 1198 if (mmc_card_broken_sd_discard(card)) 1199 arg = SD_ERASE_ARG; 1200 1201 mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg); 1202 } 1203 1204 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq, 1205 struct request *req) 1206 { 1207 struct mmc_blk_data *md = mq->blkdata; 1208 struct mmc_card *card = md->queue.card; 1209 unsigned int from, nr, arg; 1210 int err = 0, type = MMC_BLK_SECDISCARD; 1211 blk_status_t status = BLK_STS_OK; 1212 1213 if (!(mmc_can_secure_erase_trim(card))) { 1214 status = BLK_STS_NOTSUPP; 1215 goto out; 1216 } 1217 1218 from = blk_rq_pos(req); 1219 nr = blk_rq_sectors(req); 1220 1221 if (mmc_can_trim(card) && !mmc_erase_group_aligned(card, from, nr)) 1222 arg = MMC_SECURE_TRIM1_ARG; 1223 else 1224 arg = MMC_SECURE_ERASE_ARG; 1225 1226 retry: 1227 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1228 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1229 INAND_CMD38_ARG_EXT_CSD, 1230 arg == MMC_SECURE_TRIM1_ARG ? 1231 INAND_CMD38_ARG_SECTRIM1 : 1232 INAND_CMD38_ARG_SECERASE, 1233 card->ext_csd.generic_cmd6_time); 1234 if (err) 1235 goto out_retry; 1236 } 1237 1238 err = mmc_erase(card, from, nr, arg); 1239 if (err == -EIO) 1240 goto out_retry; 1241 if (err) { 1242 status = BLK_STS_IOERR; 1243 goto out; 1244 } 1245 1246 if (arg == MMC_SECURE_TRIM1_ARG) { 1247 if (card->quirks & MMC_QUIRK_INAND_CMD38) { 1248 err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, 1249 INAND_CMD38_ARG_EXT_CSD, 1250 INAND_CMD38_ARG_SECTRIM2, 1251 card->ext_csd.generic_cmd6_time); 1252 if (err) 1253 goto out_retry; 1254 } 1255 1256 err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG); 1257 if (err == -EIO) 1258 goto out_retry; 1259 if (err) { 1260 status = BLK_STS_IOERR; 1261 goto out; 1262 } 1263 } 1264 1265 out_retry: 1266 if (err && !mmc_blk_reset(md, card->host, type)) 1267 goto retry; 1268 if (!err) 1269 mmc_blk_reset_success(md, type); 1270 out: 1271 blk_mq_end_request(req, status); 1272 } 1273 1274 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req) 1275 { 1276 struct mmc_blk_data *md = mq->blkdata; 1277 struct mmc_card *card = md->queue.card; 1278 int ret = 0; 1279 1280 ret = mmc_flush_cache(card->host); 1281 blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK); 1282 } 1283 1284 /* 1285 * Reformat current write as a reliable write, supporting 1286 * both legacy and the enhanced reliable write MMC cards. 1287 * In each transfer we'll handle only as much as a single 1288 * reliable write can handle, thus finish the request in 1289 * partial completions. 1290 */ 1291 static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq, 1292 struct mmc_card *card, 1293 struct request *req) 1294 { 1295 if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) { 1296 /* Legacy mode imposes restrictions on transfers. */ 1297 if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors)) 1298 brq->data.blocks = 1; 1299 1300 if (brq->data.blocks > card->ext_csd.rel_sectors) 1301 brq->data.blocks = card->ext_csd.rel_sectors; 1302 else if (brq->data.blocks < card->ext_csd.rel_sectors) 1303 brq->data.blocks = 1; 1304 } 1305 } 1306 1307 #define CMD_ERRORS_EXCL_OOR \ 1308 (R1_ADDRESS_ERROR | /* Misaligned address */ \ 1309 R1_BLOCK_LEN_ERROR | /* Transferred block length incorrect */\ 1310 R1_WP_VIOLATION | /* Tried to write to protected block */ \ 1311 R1_CARD_ECC_FAILED | /* Card ECC failed */ \ 1312 R1_CC_ERROR | /* Card controller error */ \ 1313 R1_ERROR) /* General/unknown error */ 1314 1315 #define CMD_ERRORS \ 1316 (CMD_ERRORS_EXCL_OOR | \ 1317 R1_OUT_OF_RANGE) /* Command argument out of range */ \ 1318 1319 static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq) 1320 { 1321 u32 val; 1322 1323 /* 1324 * Per the SD specification(physical layer version 4.10)[1], 1325 * section 4.3.3, it explicitly states that "When the last 1326 * block of user area is read using CMD18, the host should 1327 * ignore OUT_OF_RANGE error that may occur even the sequence 1328 * is correct". And JESD84-B51 for eMMC also has a similar 1329 * statement on section 6.8.3. 1330 * 1331 * Multiple block read/write could be done by either predefined 1332 * method, namely CMD23, or open-ending mode. For open-ending mode, 1333 * we should ignore the OUT_OF_RANGE error as it's normal behaviour. 1334 * 1335 * However the spec[1] doesn't tell us whether we should also 1336 * ignore that for predefined method. But per the spec[1], section 1337 * 4.15 Set Block Count Command, it says"If illegal block count 1338 * is set, out of range error will be indicated during read/write 1339 * operation (For example, data transfer is stopped at user area 1340 * boundary)." In another word, we could expect a out of range error 1341 * in the response for the following CMD18/25. And if argument of 1342 * CMD23 + the argument of CMD18/25 exceed the max number of blocks, 1343 * we could also expect to get a -ETIMEDOUT or any error number from 1344 * the host drivers due to missing data response(for write)/data(for 1345 * read), as the cards will stop the data transfer by itself per the 1346 * spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode. 1347 */ 1348 1349 if (!brq->stop.error) { 1350 bool oor_with_open_end; 1351 /* If there is no error yet, check R1 response */ 1352 1353 val = brq->stop.resp[0] & CMD_ERRORS; 1354 oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc; 1355 1356 if (val && !oor_with_open_end) 1357 brq->stop.error = -EIO; 1358 } 1359 } 1360 1361 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq, 1362 int recovery_mode, bool *do_rel_wr_p, 1363 bool *do_data_tag_p) 1364 { 1365 struct mmc_blk_data *md = mq->blkdata; 1366 struct mmc_card *card = md->queue.card; 1367 struct mmc_blk_request *brq = &mqrq->brq; 1368 struct request *req = mmc_queue_req_to_req(mqrq); 1369 bool do_rel_wr, do_data_tag; 1370 1371 /* 1372 * Reliable writes are used to implement Forced Unit Access and 1373 * are supported only on MMCs. 1374 */ 1375 do_rel_wr = (req->cmd_flags & REQ_FUA) && 1376 rq_data_dir(req) == WRITE && 1377 (md->flags & MMC_BLK_REL_WR); 1378 1379 memset(brq, 0, sizeof(struct mmc_blk_request)); 1380 1381 mmc_crypto_prepare_req(mqrq); 1382 1383 brq->mrq.data = &brq->data; 1384 brq->mrq.tag = req->tag; 1385 1386 brq->stop.opcode = MMC_STOP_TRANSMISSION; 1387 brq->stop.arg = 0; 1388 1389 if (rq_data_dir(req) == READ) { 1390 brq->data.flags = MMC_DATA_READ; 1391 brq->stop.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC; 1392 } else { 1393 brq->data.flags = MMC_DATA_WRITE; 1394 brq->stop.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC; 1395 } 1396 1397 brq->data.blksz = 512; 1398 brq->data.blocks = blk_rq_sectors(req); 1399 brq->data.blk_addr = blk_rq_pos(req); 1400 1401 /* 1402 * The command queue supports 2 priorities: "high" (1) and "simple" (0). 1403 * The eMMC will give "high" priority tasks priority over "simple" 1404 * priority tasks. Here we always set "simple" priority by not setting 1405 * MMC_DATA_PRIO. 1406 */ 1407 1408 /* 1409 * The block layer doesn't support all sector count 1410 * restrictions, so we need to be prepared for too big 1411 * requests. 1412 */ 1413 if (brq->data.blocks > card->host->max_blk_count) 1414 brq->data.blocks = card->host->max_blk_count; 1415 1416 if (brq->data.blocks > 1) { 1417 /* 1418 * Some SD cards in SPI mode return a CRC error or even lock up 1419 * completely when trying to read the last block using a 1420 * multiblock read command. 1421 */ 1422 if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) && 1423 (blk_rq_pos(req) + blk_rq_sectors(req) == 1424 get_capacity(md->disk))) 1425 brq->data.blocks--; 1426 1427 /* 1428 * After a read error, we redo the request one (native) sector 1429 * at a time in order to accurately determine which 1430 * sectors can be read successfully. 1431 */ 1432 if (recovery_mode) 1433 brq->data.blocks = queue_physical_block_size(mq->queue) >> 9; 1434 1435 /* 1436 * Some controllers have HW issues while operating 1437 * in multiple I/O mode 1438 */ 1439 if (card->host->ops->multi_io_quirk) 1440 brq->data.blocks = card->host->ops->multi_io_quirk(card, 1441 (rq_data_dir(req) == READ) ? 1442 MMC_DATA_READ : MMC_DATA_WRITE, 1443 brq->data.blocks); 1444 } 1445 1446 if (do_rel_wr) { 1447 mmc_apply_rel_rw(brq, card, req); 1448 brq->data.flags |= MMC_DATA_REL_WR; 1449 } 1450 1451 /* 1452 * Data tag is used only during writing meta data to speed 1453 * up write and any subsequent read of this meta data 1454 */ 1455 do_data_tag = card->ext_csd.data_tag_unit_size && 1456 (req->cmd_flags & REQ_META) && 1457 (rq_data_dir(req) == WRITE) && 1458 ((brq->data.blocks * brq->data.blksz) >= 1459 card->ext_csd.data_tag_unit_size); 1460 1461 if (do_data_tag) 1462 brq->data.flags |= MMC_DATA_DAT_TAG; 1463 1464 mmc_set_data_timeout(&brq->data, card); 1465 1466 brq->data.sg = mqrq->sg; 1467 brq->data.sg_len = mmc_queue_map_sg(mq, mqrq); 1468 1469 /* 1470 * Adjust the sg list so it is the same size as the 1471 * request. 1472 */ 1473 if (brq->data.blocks != blk_rq_sectors(req)) { 1474 int i, data_size = brq->data.blocks << 9; 1475 struct scatterlist *sg; 1476 1477 for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) { 1478 data_size -= sg->length; 1479 if (data_size <= 0) { 1480 sg->length += data_size; 1481 i++; 1482 break; 1483 } 1484 } 1485 brq->data.sg_len = i; 1486 } 1487 1488 if (do_rel_wr_p) 1489 *do_rel_wr_p = do_rel_wr; 1490 1491 if (do_data_tag_p) 1492 *do_data_tag_p = do_data_tag; 1493 } 1494 1495 #define MMC_CQE_RETRIES 2 1496 1497 static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req) 1498 { 1499 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1500 struct mmc_request *mrq = &mqrq->brq.mrq; 1501 struct request_queue *q = req->q; 1502 struct mmc_host *host = mq->card->host; 1503 enum mmc_issue_type issue_type = mmc_issue_type(mq, req); 1504 unsigned long flags; 1505 bool put_card; 1506 int err; 1507 1508 mmc_cqe_post_req(host, mrq); 1509 1510 if (mrq->cmd && mrq->cmd->error) 1511 err = mrq->cmd->error; 1512 else if (mrq->data && mrq->data->error) 1513 err = mrq->data->error; 1514 else 1515 err = 0; 1516 1517 if (err) { 1518 if (mqrq->retries++ < MMC_CQE_RETRIES) 1519 blk_mq_requeue_request(req, true); 1520 else 1521 blk_mq_end_request(req, BLK_STS_IOERR); 1522 } else if (mrq->data) { 1523 if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered)) 1524 blk_mq_requeue_request(req, true); 1525 else 1526 __blk_mq_end_request(req, BLK_STS_OK); 1527 } else if (mq->in_recovery) { 1528 blk_mq_requeue_request(req, true); 1529 } else { 1530 blk_mq_end_request(req, BLK_STS_OK); 1531 } 1532 1533 spin_lock_irqsave(&mq->lock, flags); 1534 1535 mq->in_flight[issue_type] -= 1; 1536 1537 put_card = (mmc_tot_in_flight(mq) == 0); 1538 1539 mmc_cqe_check_busy(mq); 1540 1541 spin_unlock_irqrestore(&mq->lock, flags); 1542 1543 if (!mq->cqe_busy) 1544 blk_mq_run_hw_queues(q, true); 1545 1546 if (put_card) 1547 mmc_put_card(mq->card, &mq->ctx); 1548 } 1549 1550 void mmc_blk_cqe_recovery(struct mmc_queue *mq) 1551 { 1552 struct mmc_card *card = mq->card; 1553 struct mmc_host *host = card->host; 1554 int err; 1555 1556 pr_debug("%s: CQE recovery start\n", mmc_hostname(host)); 1557 1558 err = mmc_cqe_recovery(host); 1559 if (err) 1560 mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY); 1561 mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY); 1562 1563 pr_debug("%s: CQE recovery done\n", mmc_hostname(host)); 1564 } 1565 1566 static void mmc_blk_cqe_req_done(struct mmc_request *mrq) 1567 { 1568 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req, 1569 brq.mrq); 1570 struct request *req = mmc_queue_req_to_req(mqrq); 1571 struct request_queue *q = req->q; 1572 struct mmc_queue *mq = q->queuedata; 1573 1574 /* 1575 * Block layer timeouts race with completions which means the normal 1576 * completion path cannot be used during recovery. 1577 */ 1578 if (mq->in_recovery) 1579 mmc_blk_cqe_complete_rq(mq, req); 1580 else if (likely(!blk_should_fake_timeout(req->q))) 1581 blk_mq_complete_request(req); 1582 } 1583 1584 static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq) 1585 { 1586 mrq->done = mmc_blk_cqe_req_done; 1587 mrq->recovery_notifier = mmc_cqe_recovery_notifier; 1588 1589 return mmc_cqe_start_req(host, mrq); 1590 } 1591 1592 static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq, 1593 struct request *req) 1594 { 1595 struct mmc_blk_request *brq = &mqrq->brq; 1596 1597 memset(brq, 0, sizeof(*brq)); 1598 1599 brq->mrq.cmd = &brq->cmd; 1600 brq->mrq.tag = req->tag; 1601 1602 return &brq->mrq; 1603 } 1604 1605 static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req) 1606 { 1607 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1608 struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req); 1609 1610 mrq->cmd->opcode = MMC_SWITCH; 1611 mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) | 1612 (EXT_CSD_FLUSH_CACHE << 16) | 1613 (1 << 8) | 1614 EXT_CSD_CMD_SET_NORMAL; 1615 mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B; 1616 1617 return mmc_blk_cqe_start_req(mq->card->host, mrq); 1618 } 1619 1620 static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req) 1621 { 1622 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1623 struct mmc_host *host = mq->card->host; 1624 int err; 1625 1626 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); 1627 mqrq->brq.mrq.done = mmc_blk_hsq_req_done; 1628 mmc_pre_req(host, &mqrq->brq.mrq); 1629 1630 err = mmc_cqe_start_req(host, &mqrq->brq.mrq); 1631 if (err) 1632 mmc_post_req(host, &mqrq->brq.mrq, err); 1633 1634 return err; 1635 } 1636 1637 static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req) 1638 { 1639 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1640 struct mmc_host *host = mq->card->host; 1641 1642 if (host->hsq_enabled) 1643 return mmc_blk_hsq_issue_rw_rq(mq, req); 1644 1645 mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL); 1646 1647 return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq); 1648 } 1649 1650 static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq, 1651 struct mmc_card *card, 1652 int recovery_mode, 1653 struct mmc_queue *mq) 1654 { 1655 u32 readcmd, writecmd; 1656 struct mmc_blk_request *brq = &mqrq->brq; 1657 struct request *req = mmc_queue_req_to_req(mqrq); 1658 struct mmc_blk_data *md = mq->blkdata; 1659 bool do_rel_wr, do_data_tag; 1660 1661 mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag); 1662 1663 brq->mrq.cmd = &brq->cmd; 1664 1665 brq->cmd.arg = blk_rq_pos(req); 1666 if (!mmc_card_blockaddr(card)) 1667 brq->cmd.arg <<= 9; 1668 brq->cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_ADTC; 1669 1670 if (brq->data.blocks > 1 || do_rel_wr) { 1671 /* SPI multiblock writes terminate using a special 1672 * token, not a STOP_TRANSMISSION request. 1673 */ 1674 if (!mmc_host_is_spi(card->host) || 1675 rq_data_dir(req) == READ) 1676 brq->mrq.stop = &brq->stop; 1677 readcmd = MMC_READ_MULTIPLE_BLOCK; 1678 writecmd = MMC_WRITE_MULTIPLE_BLOCK; 1679 } else { 1680 brq->mrq.stop = NULL; 1681 readcmd = MMC_READ_SINGLE_BLOCK; 1682 writecmd = MMC_WRITE_BLOCK; 1683 } 1684 brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd; 1685 1686 /* 1687 * Pre-defined multi-block transfers are preferable to 1688 * open ended-ones (and necessary for reliable writes). 1689 * However, it is not sufficient to just send CMD23, 1690 * and avoid the final CMD12, as on an error condition 1691 * CMD12 (stop) needs to be sent anyway. This, coupled 1692 * with Auto-CMD23 enhancements provided by some 1693 * hosts, means that the complexity of dealing 1694 * with this is best left to the host. If CMD23 is 1695 * supported by card and host, we'll fill sbc in and let 1696 * the host deal with handling it correctly. This means 1697 * that for hosts that don't expose MMC_CAP_CMD23, no 1698 * change of behavior will be observed. 1699 * 1700 * N.B: Some MMC cards experience perf degradation. 1701 * We'll avoid using CMD23-bounded multiblock writes for 1702 * these, while retaining features like reliable writes. 1703 */ 1704 if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) && 1705 (do_rel_wr || !(card->quirks & MMC_QUIRK_BLK_NO_CMD23) || 1706 do_data_tag)) { 1707 brq->sbc.opcode = MMC_SET_BLOCK_COUNT; 1708 brq->sbc.arg = brq->data.blocks | 1709 (do_rel_wr ? (1 << 31) : 0) | 1710 (do_data_tag ? (1 << 29) : 0); 1711 brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC; 1712 brq->mrq.sbc = &brq->sbc; 1713 } 1714 } 1715 1716 #define MMC_MAX_RETRIES 5 1717 #define MMC_DATA_RETRIES 2 1718 #define MMC_NO_RETRIES (MMC_MAX_RETRIES + 1) 1719 1720 static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout) 1721 { 1722 struct mmc_command cmd = { 1723 .opcode = MMC_STOP_TRANSMISSION, 1724 .flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC, 1725 /* Some hosts wait for busy anyway, so provide a busy timeout */ 1726 .busy_timeout = timeout, 1727 }; 1728 1729 return mmc_wait_for_cmd(card->host, &cmd, 5); 1730 } 1731 1732 static int mmc_blk_fix_state(struct mmc_card *card, struct request *req) 1733 { 1734 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1735 struct mmc_blk_request *brq = &mqrq->brq; 1736 unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data); 1737 int err; 1738 1739 mmc_retune_hold_now(card->host); 1740 1741 mmc_blk_send_stop(card, timeout); 1742 1743 err = mmc_poll_for_busy(card, timeout, false, MMC_BUSY_IO); 1744 1745 mmc_retune_release(card->host); 1746 1747 return err; 1748 } 1749 1750 #define MMC_READ_SINGLE_RETRIES 2 1751 1752 /* Single (native) sector read during recovery */ 1753 static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req) 1754 { 1755 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1756 struct mmc_request *mrq = &mqrq->brq.mrq; 1757 struct mmc_card *card = mq->card; 1758 struct mmc_host *host = card->host; 1759 blk_status_t error = BLK_STS_OK; 1760 size_t bytes_per_read = queue_physical_block_size(mq->queue); 1761 1762 do { 1763 u32 status; 1764 int err; 1765 int retries = 0; 1766 1767 while (retries++ <= MMC_READ_SINGLE_RETRIES) { 1768 mmc_blk_rw_rq_prep(mqrq, card, 1, mq); 1769 1770 mmc_wait_for_req(host, mrq); 1771 1772 err = mmc_send_status(card, &status); 1773 if (err) 1774 goto error_exit; 1775 1776 if (!mmc_host_is_spi(host) && 1777 !mmc_ready_for_data(status)) { 1778 err = mmc_blk_fix_state(card, req); 1779 if (err) 1780 goto error_exit; 1781 } 1782 1783 if (!mrq->cmd->error) 1784 break; 1785 } 1786 1787 if (mrq->cmd->error || 1788 mrq->data->error || 1789 (!mmc_host_is_spi(host) && 1790 (mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS))) 1791 error = BLK_STS_IOERR; 1792 else 1793 error = BLK_STS_OK; 1794 1795 } while (blk_update_request(req, error, bytes_per_read)); 1796 1797 return; 1798 1799 error_exit: 1800 mrq->data->bytes_xfered = 0; 1801 blk_update_request(req, BLK_STS_IOERR, bytes_per_read); 1802 /* Let it try the remaining request again */ 1803 if (mqrq->retries > MMC_MAX_RETRIES - 1) 1804 mqrq->retries = MMC_MAX_RETRIES - 1; 1805 } 1806 1807 static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq) 1808 { 1809 return !!brq->mrq.sbc; 1810 } 1811 1812 static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq) 1813 { 1814 return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR; 1815 } 1816 1817 /* 1818 * Check for errors the host controller driver might not have seen such as 1819 * response mode errors or invalid card state. 1820 */ 1821 static bool mmc_blk_status_error(struct request *req, u32 status) 1822 { 1823 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1824 struct mmc_blk_request *brq = &mqrq->brq; 1825 struct mmc_queue *mq = req->q->queuedata; 1826 u32 stop_err_bits; 1827 1828 if (mmc_host_is_spi(mq->card->host)) 1829 return false; 1830 1831 stop_err_bits = mmc_blk_stop_err_bits(brq); 1832 1833 return brq->cmd.resp[0] & CMD_ERRORS || 1834 brq->stop.resp[0] & stop_err_bits || 1835 status & stop_err_bits || 1836 (rq_data_dir(req) == WRITE && !mmc_ready_for_data(status)); 1837 } 1838 1839 static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq) 1840 { 1841 return !brq->sbc.error && !brq->cmd.error && 1842 !(brq->cmd.resp[0] & CMD_ERRORS); 1843 } 1844 1845 /* 1846 * Requests are completed by mmc_blk_mq_complete_rq() which sets simple 1847 * policy: 1848 * 1. A request that has transferred at least some data is considered 1849 * successful and will be requeued if there is remaining data to 1850 * transfer. 1851 * 2. Otherwise the number of retries is incremented and the request 1852 * will be requeued if there are remaining retries. 1853 * 3. Otherwise the request will be errored out. 1854 * That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and 1855 * mqrq->retries. So there are only 4 possible actions here: 1856 * 1. do not accept the bytes_xfered value i.e. set it to zero 1857 * 2. change mqrq->retries to determine the number of retries 1858 * 3. try to reset the card 1859 * 4. read one sector at a time 1860 */ 1861 static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req) 1862 { 1863 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE; 1864 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 1865 struct mmc_blk_request *brq = &mqrq->brq; 1866 struct mmc_blk_data *md = mq->blkdata; 1867 struct mmc_card *card = mq->card; 1868 u32 status; 1869 u32 blocks; 1870 int err; 1871 1872 /* 1873 * Some errors the host driver might not have seen. Set the number of 1874 * bytes transferred to zero in that case. 1875 */ 1876 err = __mmc_send_status(card, &status, 0); 1877 if (err || mmc_blk_status_error(req, status)) 1878 brq->data.bytes_xfered = 0; 1879 1880 mmc_retune_release(card->host); 1881 1882 /* 1883 * Try again to get the status. This also provides an opportunity for 1884 * re-tuning. 1885 */ 1886 if (err) 1887 err = __mmc_send_status(card, &status, 0); 1888 1889 /* 1890 * Nothing more to do after the number of bytes transferred has been 1891 * updated and there is no card. 1892 */ 1893 if (err && mmc_detect_card_removed(card->host)) 1894 return; 1895 1896 /* Try to get back to "tran" state */ 1897 if (!mmc_host_is_spi(mq->card->host) && 1898 (err || !mmc_ready_for_data(status))) 1899 err = mmc_blk_fix_state(mq->card, req); 1900 1901 /* 1902 * Special case for SD cards where the card might record the number of 1903 * blocks written. 1904 */ 1905 if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) && 1906 rq_data_dir(req) == WRITE) { 1907 if (mmc_sd_num_wr_blocks(card, &blocks)) 1908 brq->data.bytes_xfered = 0; 1909 else 1910 brq->data.bytes_xfered = blocks << 9; 1911 } 1912 1913 /* Reset if the card is in a bad state */ 1914 if (!mmc_host_is_spi(mq->card->host) && 1915 err && mmc_blk_reset(md, card->host, type)) { 1916 pr_err("%s: recovery failed!\n", req->q->disk->disk_name); 1917 mqrq->retries = MMC_NO_RETRIES; 1918 return; 1919 } 1920 1921 /* 1922 * If anything was done, just return and if there is anything remaining 1923 * on the request it will get requeued. 1924 */ 1925 if (brq->data.bytes_xfered) 1926 return; 1927 1928 /* Reset before last retry */ 1929 if (mqrq->retries + 1 == MMC_MAX_RETRIES && 1930 mmc_blk_reset(md, card->host, type)) 1931 return; 1932 1933 /* Command errors fail fast, so use all MMC_MAX_RETRIES */ 1934 if (brq->sbc.error || brq->cmd.error) 1935 return; 1936 1937 /* Reduce the remaining retries for data errors */ 1938 if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) { 1939 mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES; 1940 return; 1941 } 1942 1943 if (rq_data_dir(req) == READ && brq->data.blocks > 1944 queue_physical_block_size(mq->queue) >> 9) { 1945 /* Read one (native) sector at a time */ 1946 mmc_blk_read_single(mq, req); 1947 return; 1948 } 1949 } 1950 1951 static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq) 1952 { 1953 mmc_blk_eval_resp_error(brq); 1954 1955 return brq->sbc.error || brq->cmd.error || brq->stop.error || 1956 brq->data.error || brq->cmd.resp[0] & CMD_ERRORS; 1957 } 1958 1959 static int mmc_spi_err_check(struct mmc_card *card) 1960 { 1961 u32 status = 0; 1962 int err; 1963 1964 /* 1965 * SPI does not have a TRAN state we have to wait on, instead the 1966 * card is ready again when it no longer holds the line LOW. 1967 * We still have to ensure two things here before we know the write 1968 * was successful: 1969 * 1. The card has not disconnected during busy and we actually read our 1970 * own pull-up, thinking it was still connected, so ensure it 1971 * still responds. 1972 * 2. Check for any error bits, in particular R1_SPI_IDLE to catch a 1973 * just reconnected card after being disconnected during busy. 1974 */ 1975 err = __mmc_send_status(card, &status, 0); 1976 if (err) 1977 return err; 1978 /* All R1 and R2 bits of SPI are errors in our case */ 1979 if (status) 1980 return -EIO; 1981 return 0; 1982 } 1983 1984 static int mmc_blk_busy_cb(void *cb_data, bool *busy) 1985 { 1986 struct mmc_blk_busy_data *data = cb_data; 1987 u32 status = 0; 1988 int err; 1989 1990 err = mmc_send_status(data->card, &status); 1991 if (err) 1992 return err; 1993 1994 /* Accumulate response error bits. */ 1995 data->status |= status; 1996 1997 *busy = !mmc_ready_for_data(status); 1998 return 0; 1999 } 2000 2001 static int mmc_blk_card_busy(struct mmc_card *card, struct request *req) 2002 { 2003 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2004 struct mmc_blk_busy_data cb_data; 2005 int err; 2006 2007 if (rq_data_dir(req) == READ) 2008 return 0; 2009 2010 if (mmc_host_is_spi(card->host)) { 2011 err = mmc_spi_err_check(card); 2012 if (err) 2013 mqrq->brq.data.bytes_xfered = 0; 2014 return err; 2015 } 2016 2017 cb_data.card = card; 2018 cb_data.status = 0; 2019 err = __mmc_poll_for_busy(card->host, 0, MMC_BLK_TIMEOUT_MS, 2020 &mmc_blk_busy_cb, &cb_data); 2021 2022 /* 2023 * Do not assume data transferred correctly if there are any error bits 2024 * set. 2025 */ 2026 if (cb_data.status & mmc_blk_stop_err_bits(&mqrq->brq)) { 2027 mqrq->brq.data.bytes_xfered = 0; 2028 err = err ? err : -EIO; 2029 } 2030 2031 /* Copy the exception bit so it will be seen later on */ 2032 if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT) 2033 mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT; 2034 2035 return err; 2036 } 2037 2038 static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq, 2039 struct request *req) 2040 { 2041 int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE; 2042 2043 mmc_blk_reset_success(mq->blkdata, type); 2044 } 2045 2046 static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req) 2047 { 2048 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2049 unsigned int nr_bytes = mqrq->brq.data.bytes_xfered; 2050 2051 if (nr_bytes) { 2052 if (blk_update_request(req, BLK_STS_OK, nr_bytes)) 2053 blk_mq_requeue_request(req, true); 2054 else 2055 __blk_mq_end_request(req, BLK_STS_OK); 2056 } else if (!blk_rq_bytes(req)) { 2057 __blk_mq_end_request(req, BLK_STS_IOERR); 2058 } else if (mqrq->retries++ < MMC_MAX_RETRIES) { 2059 blk_mq_requeue_request(req, true); 2060 } else { 2061 if (mmc_card_removed(mq->card)) 2062 req->rq_flags |= RQF_QUIET; 2063 blk_mq_end_request(req, BLK_STS_IOERR); 2064 } 2065 } 2066 2067 static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq, 2068 struct mmc_queue_req *mqrq) 2069 { 2070 return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) && 2071 (mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT || 2072 mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT); 2073 } 2074 2075 static void mmc_blk_urgent_bkops(struct mmc_queue *mq, 2076 struct mmc_queue_req *mqrq) 2077 { 2078 if (mmc_blk_urgent_bkops_needed(mq, mqrq)) 2079 mmc_run_bkops(mq->card); 2080 } 2081 2082 static void mmc_blk_hsq_req_done(struct mmc_request *mrq) 2083 { 2084 struct mmc_queue_req *mqrq = 2085 container_of(mrq, struct mmc_queue_req, brq.mrq); 2086 struct request *req = mmc_queue_req_to_req(mqrq); 2087 struct request_queue *q = req->q; 2088 struct mmc_queue *mq = q->queuedata; 2089 struct mmc_host *host = mq->card->host; 2090 unsigned long flags; 2091 2092 if (mmc_blk_rq_error(&mqrq->brq) || 2093 mmc_blk_urgent_bkops_needed(mq, mqrq)) { 2094 spin_lock_irqsave(&mq->lock, flags); 2095 mq->recovery_needed = true; 2096 mq->recovery_req = req; 2097 spin_unlock_irqrestore(&mq->lock, flags); 2098 2099 host->cqe_ops->cqe_recovery_start(host); 2100 2101 schedule_work(&mq->recovery_work); 2102 return; 2103 } 2104 2105 mmc_blk_rw_reset_success(mq, req); 2106 2107 /* 2108 * Block layer timeouts race with completions which means the normal 2109 * completion path cannot be used during recovery. 2110 */ 2111 if (mq->in_recovery) 2112 mmc_blk_cqe_complete_rq(mq, req); 2113 else if (likely(!blk_should_fake_timeout(req->q))) 2114 blk_mq_complete_request(req); 2115 } 2116 2117 void mmc_blk_mq_complete(struct request *req) 2118 { 2119 struct mmc_queue *mq = req->q->queuedata; 2120 struct mmc_host *host = mq->card->host; 2121 2122 if (host->cqe_enabled) 2123 mmc_blk_cqe_complete_rq(mq, req); 2124 else if (likely(!blk_should_fake_timeout(req->q))) 2125 mmc_blk_mq_complete_rq(mq, req); 2126 } 2127 2128 static void mmc_blk_mq_poll_completion(struct mmc_queue *mq, 2129 struct request *req) 2130 { 2131 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2132 struct mmc_host *host = mq->card->host; 2133 2134 if (mmc_blk_rq_error(&mqrq->brq) || 2135 mmc_blk_card_busy(mq->card, req)) { 2136 mmc_blk_mq_rw_recovery(mq, req); 2137 } else { 2138 mmc_blk_rw_reset_success(mq, req); 2139 mmc_retune_release(host); 2140 } 2141 2142 mmc_blk_urgent_bkops(mq, mqrq); 2143 } 2144 2145 static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type) 2146 { 2147 unsigned long flags; 2148 bool put_card; 2149 2150 spin_lock_irqsave(&mq->lock, flags); 2151 2152 mq->in_flight[issue_type] -= 1; 2153 2154 put_card = (mmc_tot_in_flight(mq) == 0); 2155 2156 spin_unlock_irqrestore(&mq->lock, flags); 2157 2158 if (put_card) 2159 mmc_put_card(mq->card, &mq->ctx); 2160 } 2161 2162 static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req, 2163 bool can_sleep) 2164 { 2165 enum mmc_issue_type issue_type = mmc_issue_type(mq, req); 2166 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2167 struct mmc_request *mrq = &mqrq->brq.mrq; 2168 struct mmc_host *host = mq->card->host; 2169 2170 mmc_post_req(host, mrq, 0); 2171 2172 /* 2173 * Block layer timeouts race with completions which means the normal 2174 * completion path cannot be used during recovery. 2175 */ 2176 if (mq->in_recovery) { 2177 mmc_blk_mq_complete_rq(mq, req); 2178 } else if (likely(!blk_should_fake_timeout(req->q))) { 2179 if (can_sleep) 2180 blk_mq_complete_request_direct(req, mmc_blk_mq_complete); 2181 else 2182 blk_mq_complete_request(req); 2183 } 2184 2185 mmc_blk_mq_dec_in_flight(mq, issue_type); 2186 } 2187 2188 void mmc_blk_mq_recovery(struct mmc_queue *mq) 2189 { 2190 struct request *req = mq->recovery_req; 2191 struct mmc_host *host = mq->card->host; 2192 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2193 2194 mq->recovery_req = NULL; 2195 mq->rw_wait = false; 2196 2197 if (mmc_blk_rq_error(&mqrq->brq)) { 2198 mmc_retune_hold_now(host); 2199 mmc_blk_mq_rw_recovery(mq, req); 2200 } 2201 2202 mmc_blk_urgent_bkops(mq, mqrq); 2203 2204 mmc_blk_mq_post_req(mq, req, true); 2205 } 2206 2207 static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq, 2208 struct request **prev_req) 2209 { 2210 if (mmc_host_done_complete(mq->card->host)) 2211 return; 2212 2213 mutex_lock(&mq->complete_lock); 2214 2215 if (!mq->complete_req) 2216 goto out_unlock; 2217 2218 mmc_blk_mq_poll_completion(mq, mq->complete_req); 2219 2220 if (prev_req) 2221 *prev_req = mq->complete_req; 2222 else 2223 mmc_blk_mq_post_req(mq, mq->complete_req, true); 2224 2225 mq->complete_req = NULL; 2226 2227 out_unlock: 2228 mutex_unlock(&mq->complete_lock); 2229 } 2230 2231 void mmc_blk_mq_complete_work(struct work_struct *work) 2232 { 2233 struct mmc_queue *mq = container_of(work, struct mmc_queue, 2234 complete_work); 2235 2236 mmc_blk_mq_complete_prev_req(mq, NULL); 2237 } 2238 2239 static void mmc_blk_mq_req_done(struct mmc_request *mrq) 2240 { 2241 struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req, 2242 brq.mrq); 2243 struct request *req = mmc_queue_req_to_req(mqrq); 2244 struct request_queue *q = req->q; 2245 struct mmc_queue *mq = q->queuedata; 2246 struct mmc_host *host = mq->card->host; 2247 unsigned long flags; 2248 2249 if (!mmc_host_done_complete(host)) { 2250 bool waiting; 2251 2252 /* 2253 * We cannot complete the request in this context, so record 2254 * that there is a request to complete, and that a following 2255 * request does not need to wait (although it does need to 2256 * complete complete_req first). 2257 */ 2258 spin_lock_irqsave(&mq->lock, flags); 2259 mq->complete_req = req; 2260 mq->rw_wait = false; 2261 waiting = mq->waiting; 2262 spin_unlock_irqrestore(&mq->lock, flags); 2263 2264 /* 2265 * If 'waiting' then the waiting task will complete this 2266 * request, otherwise queue a work to do it. Note that 2267 * complete_work may still race with the dispatch of a following 2268 * request. 2269 */ 2270 if (waiting) 2271 wake_up(&mq->wait); 2272 else 2273 queue_work(mq->card->complete_wq, &mq->complete_work); 2274 2275 return; 2276 } 2277 2278 /* Take the recovery path for errors or urgent background operations */ 2279 if (mmc_blk_rq_error(&mqrq->brq) || 2280 mmc_blk_urgent_bkops_needed(mq, mqrq)) { 2281 spin_lock_irqsave(&mq->lock, flags); 2282 mq->recovery_needed = true; 2283 mq->recovery_req = req; 2284 spin_unlock_irqrestore(&mq->lock, flags); 2285 wake_up(&mq->wait); 2286 schedule_work(&mq->recovery_work); 2287 return; 2288 } 2289 2290 mmc_blk_rw_reset_success(mq, req); 2291 2292 mq->rw_wait = false; 2293 wake_up(&mq->wait); 2294 2295 /* context unknown */ 2296 mmc_blk_mq_post_req(mq, req, false); 2297 } 2298 2299 static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err) 2300 { 2301 unsigned long flags; 2302 bool done; 2303 2304 /* 2305 * Wait while there is another request in progress, but not if recovery 2306 * is needed. Also indicate whether there is a request waiting to start. 2307 */ 2308 spin_lock_irqsave(&mq->lock, flags); 2309 if (mq->recovery_needed) { 2310 *err = -EBUSY; 2311 done = true; 2312 } else { 2313 done = !mq->rw_wait; 2314 } 2315 mq->waiting = !done; 2316 spin_unlock_irqrestore(&mq->lock, flags); 2317 2318 return done; 2319 } 2320 2321 static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req) 2322 { 2323 int err = 0; 2324 2325 wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err)); 2326 2327 /* Always complete the previous request if there is one */ 2328 mmc_blk_mq_complete_prev_req(mq, prev_req); 2329 2330 return err; 2331 } 2332 2333 static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq, 2334 struct request *req) 2335 { 2336 struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req); 2337 struct mmc_host *host = mq->card->host; 2338 struct request *prev_req = NULL; 2339 int err = 0; 2340 2341 mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq); 2342 2343 mqrq->brq.mrq.done = mmc_blk_mq_req_done; 2344 2345 mmc_pre_req(host, &mqrq->brq.mrq); 2346 2347 err = mmc_blk_rw_wait(mq, &prev_req); 2348 if (err) 2349 goto out_post_req; 2350 2351 mq->rw_wait = true; 2352 2353 err = mmc_start_request(host, &mqrq->brq.mrq); 2354 2355 if (prev_req) 2356 mmc_blk_mq_post_req(mq, prev_req, true); 2357 2358 if (err) 2359 mq->rw_wait = false; 2360 2361 /* Release re-tuning here where there is no synchronization required */ 2362 if (err || mmc_host_done_complete(host)) 2363 mmc_retune_release(host); 2364 2365 out_post_req: 2366 if (err) 2367 mmc_post_req(host, &mqrq->brq.mrq, err); 2368 2369 return err; 2370 } 2371 2372 static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host) 2373 { 2374 if (host->cqe_enabled) 2375 return host->cqe_ops->cqe_wait_for_idle(host); 2376 2377 return mmc_blk_rw_wait(mq, NULL); 2378 } 2379 2380 enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req) 2381 { 2382 struct mmc_blk_data *md = mq->blkdata; 2383 struct mmc_card *card = md->queue.card; 2384 struct mmc_host *host = card->host; 2385 int ret; 2386 2387 ret = mmc_blk_part_switch(card, md->part_type); 2388 if (ret) 2389 return MMC_REQ_FAILED_TO_START; 2390 2391 switch (mmc_issue_type(mq, req)) { 2392 case MMC_ISSUE_SYNC: 2393 ret = mmc_blk_wait_for_idle(mq, host); 2394 if (ret) 2395 return MMC_REQ_BUSY; 2396 switch (req_op(req)) { 2397 case REQ_OP_DRV_IN: 2398 case REQ_OP_DRV_OUT: 2399 mmc_blk_issue_drv_op(mq, req); 2400 break; 2401 case REQ_OP_DISCARD: 2402 mmc_blk_issue_discard_rq(mq, req); 2403 break; 2404 case REQ_OP_SECURE_ERASE: 2405 mmc_blk_issue_secdiscard_rq(mq, req); 2406 break; 2407 case REQ_OP_WRITE_ZEROES: 2408 mmc_blk_issue_trim_rq(mq, req); 2409 break; 2410 case REQ_OP_FLUSH: 2411 mmc_blk_issue_flush(mq, req); 2412 break; 2413 default: 2414 WARN_ON_ONCE(1); 2415 return MMC_REQ_FAILED_TO_START; 2416 } 2417 return MMC_REQ_FINISHED; 2418 case MMC_ISSUE_DCMD: 2419 case MMC_ISSUE_ASYNC: 2420 switch (req_op(req)) { 2421 case REQ_OP_FLUSH: 2422 if (!mmc_cache_enabled(host)) { 2423 blk_mq_end_request(req, BLK_STS_OK); 2424 return MMC_REQ_FINISHED; 2425 } 2426 ret = mmc_blk_cqe_issue_flush(mq, req); 2427 break; 2428 case REQ_OP_WRITE: 2429 card->written_flag = true; 2430 fallthrough; 2431 case REQ_OP_READ: 2432 if (host->cqe_enabled) 2433 ret = mmc_blk_cqe_issue_rw_rq(mq, req); 2434 else 2435 ret = mmc_blk_mq_issue_rw_rq(mq, req); 2436 break; 2437 default: 2438 WARN_ON_ONCE(1); 2439 ret = -EINVAL; 2440 } 2441 if (!ret) 2442 return MMC_REQ_STARTED; 2443 return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START; 2444 default: 2445 WARN_ON_ONCE(1); 2446 return MMC_REQ_FAILED_TO_START; 2447 } 2448 } 2449 2450 static inline int mmc_blk_readonly(struct mmc_card *card) 2451 { 2452 return mmc_card_readonly(card) || 2453 !(card->csd.cmdclass & CCC_BLOCK_WRITE); 2454 } 2455 2456 static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card, 2457 struct device *parent, 2458 sector_t size, 2459 bool default_ro, 2460 const char *subname, 2461 int area_type, 2462 unsigned int part_type) 2463 { 2464 struct mmc_blk_data *md; 2465 int devidx, ret; 2466 char cap_str[10]; 2467 bool cache_enabled = false; 2468 bool fua_enabled = false; 2469 2470 devidx = ida_simple_get(&mmc_blk_ida, 0, max_devices, GFP_KERNEL); 2471 if (devidx < 0) { 2472 /* 2473 * We get -ENOSPC because there are no more any available 2474 * devidx. The reason may be that, either userspace haven't yet 2475 * unmounted the partitions, which postpones mmc_blk_release() 2476 * from being called, or the device has more partitions than 2477 * what we support. 2478 */ 2479 if (devidx == -ENOSPC) 2480 dev_err(mmc_dev(card->host), 2481 "no more device IDs available\n"); 2482 2483 return ERR_PTR(devidx); 2484 } 2485 2486 md = kzalloc(sizeof(struct mmc_blk_data), GFP_KERNEL); 2487 if (!md) { 2488 ret = -ENOMEM; 2489 goto out; 2490 } 2491 2492 md->area_type = area_type; 2493 2494 /* 2495 * Set the read-only status based on the supported commands 2496 * and the write protect switch. 2497 */ 2498 md->read_only = mmc_blk_readonly(card); 2499 2500 md->disk = mmc_init_queue(&md->queue, card); 2501 if (IS_ERR(md->disk)) { 2502 ret = PTR_ERR(md->disk); 2503 goto err_kfree; 2504 } 2505 2506 INIT_LIST_HEAD(&md->part); 2507 INIT_LIST_HEAD(&md->rpmbs); 2508 kref_init(&md->kref); 2509 2510 md->queue.blkdata = md; 2511 md->part_type = part_type; 2512 2513 md->disk->major = MMC_BLOCK_MAJOR; 2514 md->disk->minors = perdev_minors; 2515 md->disk->first_minor = devidx * perdev_minors; 2516 md->disk->fops = &mmc_bdops; 2517 md->disk->private_data = md; 2518 md->parent = parent; 2519 set_disk_ro(md->disk, md->read_only || default_ro); 2520 if (area_type & (MMC_BLK_DATA_AREA_RPMB | MMC_BLK_DATA_AREA_BOOT)) 2521 md->disk->flags |= GENHD_FL_NO_PART; 2522 2523 /* 2524 * As discussed on lkml, GENHD_FL_REMOVABLE should: 2525 * 2526 * - be set for removable media with permanent block devices 2527 * - be unset for removable block devices with permanent media 2528 * 2529 * Since MMC block devices clearly fall under the second 2530 * case, we do not set GENHD_FL_REMOVABLE. Userspace 2531 * should use the block device creation/destruction hotplug 2532 * messages to tell when the card is present. 2533 */ 2534 2535 snprintf(md->disk->disk_name, sizeof(md->disk->disk_name), 2536 "mmcblk%u%s", card->host->index, subname ? subname : ""); 2537 2538 set_capacity(md->disk, size); 2539 2540 if (mmc_host_cmd23(card->host)) { 2541 if ((mmc_card_mmc(card) && 2542 card->csd.mmca_vsn >= CSD_SPEC_VER_3) || 2543 (mmc_card_sd(card) && 2544 card->scr.cmds & SD_SCR_CMD23_SUPPORT)) 2545 md->flags |= MMC_BLK_CMD23; 2546 } 2547 2548 if (md->flags & MMC_BLK_CMD23 && 2549 ((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) || 2550 card->ext_csd.rel_sectors)) { 2551 md->flags |= MMC_BLK_REL_WR; 2552 fua_enabled = true; 2553 cache_enabled = true; 2554 } 2555 if (mmc_cache_enabled(card->host)) 2556 cache_enabled = true; 2557 2558 blk_queue_write_cache(md->queue.queue, cache_enabled, fua_enabled); 2559 2560 string_get_size((u64)size, 512, STRING_UNITS_2, 2561 cap_str, sizeof(cap_str)); 2562 pr_info("%s: %s %s %s%s\n", 2563 md->disk->disk_name, mmc_card_id(card), mmc_card_name(card), 2564 cap_str, md->read_only ? " (ro)" : ""); 2565 2566 /* used in ->open, must be set before add_disk: */ 2567 if (area_type == MMC_BLK_DATA_AREA_MAIN) 2568 dev_set_drvdata(&card->dev, md); 2569 ret = device_add_disk(md->parent, md->disk, mmc_disk_attr_groups); 2570 if (ret) 2571 goto err_put_disk; 2572 return md; 2573 2574 err_put_disk: 2575 put_disk(md->disk); 2576 blk_mq_free_tag_set(&md->queue.tag_set); 2577 err_kfree: 2578 kfree(md); 2579 out: 2580 ida_simple_remove(&mmc_blk_ida, devidx); 2581 return ERR_PTR(ret); 2582 } 2583 2584 static struct mmc_blk_data *mmc_blk_alloc(struct mmc_card *card) 2585 { 2586 sector_t size; 2587 2588 if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) { 2589 /* 2590 * The EXT_CSD sector count is in number or 512 byte 2591 * sectors. 2592 */ 2593 size = card->ext_csd.sectors; 2594 } else { 2595 /* 2596 * The CSD capacity field is in units of read_blkbits. 2597 * set_capacity takes units of 512 bytes. 2598 */ 2599 size = (typeof(sector_t))card->csd.capacity 2600 << (card->csd.read_blkbits - 9); 2601 } 2602 2603 return mmc_blk_alloc_req(card, &card->dev, size, false, NULL, 2604 MMC_BLK_DATA_AREA_MAIN, 0); 2605 } 2606 2607 static int mmc_blk_alloc_part(struct mmc_card *card, 2608 struct mmc_blk_data *md, 2609 unsigned int part_type, 2610 sector_t size, 2611 bool default_ro, 2612 const char *subname, 2613 int area_type) 2614 { 2615 struct mmc_blk_data *part_md; 2616 2617 part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro, 2618 subname, area_type, part_type); 2619 if (IS_ERR(part_md)) 2620 return PTR_ERR(part_md); 2621 list_add(&part_md->part, &md->part); 2622 2623 return 0; 2624 } 2625 2626 /** 2627 * mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev 2628 * @filp: the character device file 2629 * @cmd: the ioctl() command 2630 * @arg: the argument from userspace 2631 * 2632 * This will essentially just redirect the ioctl()s coming in over to 2633 * the main block device spawning the RPMB character device. 2634 */ 2635 static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd, 2636 unsigned long arg) 2637 { 2638 struct mmc_rpmb_data *rpmb = filp->private_data; 2639 int ret; 2640 2641 switch (cmd) { 2642 case MMC_IOC_CMD: 2643 ret = mmc_blk_ioctl_cmd(rpmb->md, 2644 (struct mmc_ioc_cmd __user *)arg, 2645 rpmb); 2646 break; 2647 case MMC_IOC_MULTI_CMD: 2648 ret = mmc_blk_ioctl_multi_cmd(rpmb->md, 2649 (struct mmc_ioc_multi_cmd __user *)arg, 2650 rpmb); 2651 break; 2652 default: 2653 ret = -EINVAL; 2654 break; 2655 } 2656 2657 return ret; 2658 } 2659 2660 #ifdef CONFIG_COMPAT 2661 static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd, 2662 unsigned long arg) 2663 { 2664 return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); 2665 } 2666 #endif 2667 2668 static int mmc_rpmb_chrdev_open(struct inode *inode, struct file *filp) 2669 { 2670 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev, 2671 struct mmc_rpmb_data, chrdev); 2672 2673 get_device(&rpmb->dev); 2674 filp->private_data = rpmb; 2675 mmc_blk_get(rpmb->md->disk); 2676 2677 return nonseekable_open(inode, filp); 2678 } 2679 2680 static int mmc_rpmb_chrdev_release(struct inode *inode, struct file *filp) 2681 { 2682 struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev, 2683 struct mmc_rpmb_data, chrdev); 2684 2685 mmc_blk_put(rpmb->md); 2686 put_device(&rpmb->dev); 2687 2688 return 0; 2689 } 2690 2691 static const struct file_operations mmc_rpmb_fileops = { 2692 .release = mmc_rpmb_chrdev_release, 2693 .open = mmc_rpmb_chrdev_open, 2694 .owner = THIS_MODULE, 2695 .llseek = no_llseek, 2696 .unlocked_ioctl = mmc_rpmb_ioctl, 2697 #ifdef CONFIG_COMPAT 2698 .compat_ioctl = mmc_rpmb_ioctl_compat, 2699 #endif 2700 }; 2701 2702 static void mmc_blk_rpmb_device_release(struct device *dev) 2703 { 2704 struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev); 2705 2706 ida_simple_remove(&mmc_rpmb_ida, rpmb->id); 2707 kfree(rpmb); 2708 } 2709 2710 static int mmc_blk_alloc_rpmb_part(struct mmc_card *card, 2711 struct mmc_blk_data *md, 2712 unsigned int part_index, 2713 sector_t size, 2714 const char *subname) 2715 { 2716 int devidx, ret; 2717 char rpmb_name[DISK_NAME_LEN]; 2718 char cap_str[10]; 2719 struct mmc_rpmb_data *rpmb; 2720 2721 /* This creates the minor number for the RPMB char device */ 2722 devidx = ida_simple_get(&mmc_rpmb_ida, 0, max_devices, GFP_KERNEL); 2723 if (devidx < 0) 2724 return devidx; 2725 2726 rpmb = kzalloc(sizeof(*rpmb), GFP_KERNEL); 2727 if (!rpmb) { 2728 ida_simple_remove(&mmc_rpmb_ida, devidx); 2729 return -ENOMEM; 2730 } 2731 2732 snprintf(rpmb_name, sizeof(rpmb_name), 2733 "mmcblk%u%s", card->host->index, subname ? subname : ""); 2734 2735 rpmb->id = devidx; 2736 rpmb->part_index = part_index; 2737 rpmb->dev.init_name = rpmb_name; 2738 rpmb->dev.bus = &mmc_rpmb_bus_type; 2739 rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id); 2740 rpmb->dev.parent = &card->dev; 2741 rpmb->dev.release = mmc_blk_rpmb_device_release; 2742 device_initialize(&rpmb->dev); 2743 dev_set_drvdata(&rpmb->dev, rpmb); 2744 rpmb->md = md; 2745 2746 cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops); 2747 rpmb->chrdev.owner = THIS_MODULE; 2748 ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev); 2749 if (ret) { 2750 pr_err("%s: could not add character device\n", rpmb_name); 2751 goto out_put_device; 2752 } 2753 2754 list_add(&rpmb->node, &md->rpmbs); 2755 2756 string_get_size((u64)size, 512, STRING_UNITS_2, 2757 cap_str, sizeof(cap_str)); 2758 2759 pr_info("%s: %s %s %s, chardev (%d:%d)\n", 2760 rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str, 2761 MAJOR(mmc_rpmb_devt), rpmb->id); 2762 2763 return 0; 2764 2765 out_put_device: 2766 put_device(&rpmb->dev); 2767 return ret; 2768 } 2769 2770 static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb) 2771 2772 { 2773 cdev_device_del(&rpmb->chrdev, &rpmb->dev); 2774 put_device(&rpmb->dev); 2775 } 2776 2777 /* MMC Physical partitions consist of two boot partitions and 2778 * up to four general purpose partitions. 2779 * For each partition enabled in EXT_CSD a block device will be allocatedi 2780 * to provide access to the partition. 2781 */ 2782 2783 static int mmc_blk_alloc_parts(struct mmc_card *card, struct mmc_blk_data *md) 2784 { 2785 int idx, ret; 2786 2787 if (!mmc_card_mmc(card)) 2788 return 0; 2789 2790 for (idx = 0; idx < card->nr_parts; idx++) { 2791 if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) { 2792 /* 2793 * RPMB partitions does not provide block access, they 2794 * are only accessed using ioctl():s. Thus create 2795 * special RPMB block devices that do not have a 2796 * backing block queue for these. 2797 */ 2798 ret = mmc_blk_alloc_rpmb_part(card, md, 2799 card->part[idx].part_cfg, 2800 card->part[idx].size >> 9, 2801 card->part[idx].name); 2802 if (ret) 2803 return ret; 2804 } else if (card->part[idx].size) { 2805 ret = mmc_blk_alloc_part(card, md, 2806 card->part[idx].part_cfg, 2807 card->part[idx].size >> 9, 2808 card->part[idx].force_ro, 2809 card->part[idx].name, 2810 card->part[idx].area_type); 2811 if (ret) 2812 return ret; 2813 } 2814 } 2815 2816 return 0; 2817 } 2818 2819 static void mmc_blk_remove_req(struct mmc_blk_data *md) 2820 { 2821 /* 2822 * Flush remaining requests and free queues. It is freeing the queue 2823 * that stops new requests from being accepted. 2824 */ 2825 del_gendisk(md->disk); 2826 mmc_cleanup_queue(&md->queue); 2827 mmc_blk_put(md); 2828 } 2829 2830 static void mmc_blk_remove_parts(struct mmc_card *card, 2831 struct mmc_blk_data *md) 2832 { 2833 struct list_head *pos, *q; 2834 struct mmc_blk_data *part_md; 2835 struct mmc_rpmb_data *rpmb; 2836 2837 /* Remove RPMB partitions */ 2838 list_for_each_safe(pos, q, &md->rpmbs) { 2839 rpmb = list_entry(pos, struct mmc_rpmb_data, node); 2840 list_del(pos); 2841 mmc_blk_remove_rpmb_part(rpmb); 2842 } 2843 /* Remove block partitions */ 2844 list_for_each_safe(pos, q, &md->part) { 2845 part_md = list_entry(pos, struct mmc_blk_data, part); 2846 list_del(pos); 2847 mmc_blk_remove_req(part_md); 2848 } 2849 } 2850 2851 #ifdef CONFIG_DEBUG_FS 2852 2853 static int mmc_dbg_card_status_get(void *data, u64 *val) 2854 { 2855 struct mmc_card *card = data; 2856 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 2857 struct mmc_queue *mq = &md->queue; 2858 struct request *req; 2859 int ret; 2860 2861 /* Ask the block layer about the card status */ 2862 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); 2863 if (IS_ERR(req)) 2864 return PTR_ERR(req); 2865 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS; 2866 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 2867 blk_execute_rq(req, false); 2868 ret = req_to_mmc_queue_req(req)->drv_op_result; 2869 if (ret >= 0) { 2870 *val = ret; 2871 ret = 0; 2872 } 2873 blk_mq_free_request(req); 2874 2875 return ret; 2876 } 2877 DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get, 2878 NULL, "%08llx\n"); 2879 2880 /* That is two digits * 512 + 1 for newline */ 2881 #define EXT_CSD_STR_LEN 1025 2882 2883 static int mmc_ext_csd_open(struct inode *inode, struct file *filp) 2884 { 2885 struct mmc_card *card = inode->i_private; 2886 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 2887 struct mmc_queue *mq = &md->queue; 2888 struct request *req; 2889 char *buf; 2890 ssize_t n = 0; 2891 u8 *ext_csd; 2892 int err, i; 2893 2894 buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL); 2895 if (!buf) 2896 return -ENOMEM; 2897 2898 /* Ask the block layer for the EXT CSD */ 2899 req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0); 2900 if (IS_ERR(req)) { 2901 err = PTR_ERR(req); 2902 goto out_free; 2903 } 2904 req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD; 2905 req_to_mmc_queue_req(req)->drv_op_result = -EIO; 2906 req_to_mmc_queue_req(req)->drv_op_data = &ext_csd; 2907 blk_execute_rq(req, false); 2908 err = req_to_mmc_queue_req(req)->drv_op_result; 2909 blk_mq_free_request(req); 2910 if (err) { 2911 pr_err("FAILED %d\n", err); 2912 goto out_free; 2913 } 2914 2915 for (i = 0; i < 512; i++) 2916 n += sprintf(buf + n, "%02x", ext_csd[i]); 2917 n += sprintf(buf + n, "\n"); 2918 2919 if (n != EXT_CSD_STR_LEN) { 2920 err = -EINVAL; 2921 kfree(ext_csd); 2922 goto out_free; 2923 } 2924 2925 filp->private_data = buf; 2926 kfree(ext_csd); 2927 return 0; 2928 2929 out_free: 2930 kfree(buf); 2931 return err; 2932 } 2933 2934 static ssize_t mmc_ext_csd_read(struct file *filp, char __user *ubuf, 2935 size_t cnt, loff_t *ppos) 2936 { 2937 char *buf = filp->private_data; 2938 2939 return simple_read_from_buffer(ubuf, cnt, ppos, 2940 buf, EXT_CSD_STR_LEN); 2941 } 2942 2943 static int mmc_ext_csd_release(struct inode *inode, struct file *file) 2944 { 2945 kfree(file->private_data); 2946 return 0; 2947 } 2948 2949 static const struct file_operations mmc_dbg_ext_csd_fops = { 2950 .open = mmc_ext_csd_open, 2951 .read = mmc_ext_csd_read, 2952 .release = mmc_ext_csd_release, 2953 .llseek = default_llseek, 2954 }; 2955 2956 static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md) 2957 { 2958 struct dentry *root; 2959 2960 if (!card->debugfs_root) 2961 return; 2962 2963 root = card->debugfs_root; 2964 2965 if (mmc_card_mmc(card) || mmc_card_sd(card)) { 2966 md->status_dentry = 2967 debugfs_create_file_unsafe("status", 0400, root, 2968 card, 2969 &mmc_dbg_card_status_fops); 2970 } 2971 2972 if (mmc_card_mmc(card)) { 2973 md->ext_csd_dentry = 2974 debugfs_create_file("ext_csd", S_IRUSR, root, card, 2975 &mmc_dbg_ext_csd_fops); 2976 } 2977 } 2978 2979 static void mmc_blk_remove_debugfs(struct mmc_card *card, 2980 struct mmc_blk_data *md) 2981 { 2982 if (!card->debugfs_root) 2983 return; 2984 2985 debugfs_remove(md->status_dentry); 2986 md->status_dentry = NULL; 2987 2988 debugfs_remove(md->ext_csd_dentry); 2989 md->ext_csd_dentry = NULL; 2990 } 2991 2992 #else 2993 2994 static void mmc_blk_add_debugfs(struct mmc_card *card, struct mmc_blk_data *md) 2995 { 2996 } 2997 2998 static void mmc_blk_remove_debugfs(struct mmc_card *card, 2999 struct mmc_blk_data *md) 3000 { 3001 } 3002 3003 #endif /* CONFIG_DEBUG_FS */ 3004 3005 static int mmc_blk_probe(struct mmc_card *card) 3006 { 3007 struct mmc_blk_data *md; 3008 int ret = 0; 3009 3010 /* 3011 * Check that the card supports the command class(es) we need. 3012 */ 3013 if (!(card->csd.cmdclass & CCC_BLOCK_READ)) 3014 return -ENODEV; 3015 3016 mmc_fixup_device(card, mmc_blk_fixups); 3017 3018 card->complete_wq = alloc_workqueue("mmc_complete", 3019 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 3020 if (!card->complete_wq) { 3021 pr_err("Failed to create mmc completion workqueue"); 3022 return -ENOMEM; 3023 } 3024 3025 md = mmc_blk_alloc(card); 3026 if (IS_ERR(md)) { 3027 ret = PTR_ERR(md); 3028 goto out_free; 3029 } 3030 3031 ret = mmc_blk_alloc_parts(card, md); 3032 if (ret) 3033 goto out; 3034 3035 /* Add two debugfs entries */ 3036 mmc_blk_add_debugfs(card, md); 3037 3038 pm_runtime_set_autosuspend_delay(&card->dev, 3000); 3039 pm_runtime_use_autosuspend(&card->dev); 3040 3041 /* 3042 * Don't enable runtime PM for SD-combo cards here. Leave that 3043 * decision to be taken during the SDIO init sequence instead. 3044 */ 3045 if (!mmc_card_sd_combo(card)) { 3046 pm_runtime_set_active(&card->dev); 3047 pm_runtime_enable(&card->dev); 3048 } 3049 3050 return 0; 3051 3052 out: 3053 mmc_blk_remove_parts(card, md); 3054 mmc_blk_remove_req(md); 3055 out_free: 3056 destroy_workqueue(card->complete_wq); 3057 return ret; 3058 } 3059 3060 static void mmc_blk_remove(struct mmc_card *card) 3061 { 3062 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3063 3064 mmc_blk_remove_debugfs(card, md); 3065 mmc_blk_remove_parts(card, md); 3066 pm_runtime_get_sync(&card->dev); 3067 if (md->part_curr != md->part_type) { 3068 mmc_claim_host(card->host); 3069 mmc_blk_part_switch(card, md->part_type); 3070 mmc_release_host(card->host); 3071 } 3072 if (!mmc_card_sd_combo(card)) 3073 pm_runtime_disable(&card->dev); 3074 pm_runtime_put_noidle(&card->dev); 3075 mmc_blk_remove_req(md); 3076 destroy_workqueue(card->complete_wq); 3077 } 3078 3079 static int _mmc_blk_suspend(struct mmc_card *card) 3080 { 3081 struct mmc_blk_data *part_md; 3082 struct mmc_blk_data *md = dev_get_drvdata(&card->dev); 3083 3084 if (md) { 3085 mmc_queue_suspend(&md->queue); 3086 list_for_each_entry(part_md, &md->part, part) { 3087 mmc_queue_suspend(&part_md->queue); 3088 } 3089 } 3090 return 0; 3091 } 3092 3093 static void mmc_blk_shutdown(struct mmc_card *card) 3094 { 3095 _mmc_blk_suspend(card); 3096 } 3097 3098 #ifdef CONFIG_PM_SLEEP 3099 static int mmc_blk_suspend(struct device *dev) 3100 { 3101 struct mmc_card *card = mmc_dev_to_card(dev); 3102 3103 return _mmc_blk_suspend(card); 3104 } 3105 3106 static int mmc_blk_resume(struct device *dev) 3107 { 3108 struct mmc_blk_data *part_md; 3109 struct mmc_blk_data *md = dev_get_drvdata(dev); 3110 3111 if (md) { 3112 /* 3113 * Resume involves the card going into idle state, 3114 * so current partition is always the main one. 3115 */ 3116 md->part_curr = md->part_type; 3117 mmc_queue_resume(&md->queue); 3118 list_for_each_entry(part_md, &md->part, part) { 3119 mmc_queue_resume(&part_md->queue); 3120 } 3121 } 3122 return 0; 3123 } 3124 #endif 3125 3126 static SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume); 3127 3128 static struct mmc_driver mmc_driver = { 3129 .drv = { 3130 .name = "mmcblk", 3131 .pm = &mmc_blk_pm_ops, 3132 }, 3133 .probe = mmc_blk_probe, 3134 .remove = mmc_blk_remove, 3135 .shutdown = mmc_blk_shutdown, 3136 }; 3137 3138 static int __init mmc_blk_init(void) 3139 { 3140 int res; 3141 3142 res = bus_register(&mmc_rpmb_bus_type); 3143 if (res < 0) { 3144 pr_err("mmcblk: could not register RPMB bus type\n"); 3145 return res; 3146 } 3147 res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb"); 3148 if (res < 0) { 3149 pr_err("mmcblk: failed to allocate rpmb chrdev region\n"); 3150 goto out_bus_unreg; 3151 } 3152 3153 if (perdev_minors != CONFIG_MMC_BLOCK_MINORS) 3154 pr_info("mmcblk: using %d minors per device\n", perdev_minors); 3155 3156 max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors); 3157 3158 res = register_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3159 if (res) 3160 goto out_chrdev_unreg; 3161 3162 res = mmc_register_driver(&mmc_driver); 3163 if (res) 3164 goto out_blkdev_unreg; 3165 3166 return 0; 3167 3168 out_blkdev_unreg: 3169 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3170 out_chrdev_unreg: 3171 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES); 3172 out_bus_unreg: 3173 bus_unregister(&mmc_rpmb_bus_type); 3174 return res; 3175 } 3176 3177 static void __exit mmc_blk_exit(void) 3178 { 3179 mmc_unregister_driver(&mmc_driver); 3180 unregister_blkdev(MMC_BLOCK_MAJOR, "mmc"); 3181 unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES); 3182 bus_unregister(&mmc_rpmb_bus_type); 3183 } 3184 3185 module_init(mmc_blk_init); 3186 module_exit(mmc_blk_exit); 3187 3188 MODULE_LICENSE("GPL"); 3189 MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver"); 3190