core.c (9f2482b91bcd02ac2999cf04b3fb1b89e1c4d559) core.c (576d55d625664a20ee4bae6500952febfb2d7b10)
1/*
2 * NVM Express device driver
3 * Copyright (c) 2011-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 22 unchanged lines hidden (view full) ---

31
32#include "nvme.h"
33
34#define NVME_MINORS (1U << MINORBITS)
35
36unsigned char admin_timeout = 60;
37module_param(admin_timeout, byte, 0644);
38MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
1/*
2 * NVM Express device driver
3 * Copyright (c) 2011-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 22 unchanged lines hidden (view full) ---

31
32#include "nvme.h"
33
34#define NVME_MINORS (1U << MINORBITS)
35
36unsigned char admin_timeout = 60;
37module_param(admin_timeout, byte, 0644);
38MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
39EXPORT_SYMBOL_GPL(admin_timeout);
39
40unsigned char nvme_io_timeout = 30;
41module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
42MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
40
41unsigned char nvme_io_timeout = 30;
42module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
43MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
44EXPORT_SYMBOL_GPL(nvme_io_timeout);
43
44unsigned char shutdown_timeout = 5;
45module_param(shutdown_timeout, byte, 0644);
46MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
47
48static int nvme_major;
49module_param(nvme_major, int, 0);
50

--- 54 unchanged lines hidden (view full) ---

105 unsigned long flags;
106
107 blk_mq_requeue_request(req);
108 spin_lock_irqsave(req->q->queue_lock, flags);
109 if (!blk_queue_stopped(req->q))
110 blk_mq_kick_requeue_list(req->q);
111 spin_unlock_irqrestore(req->q->queue_lock, flags);
112}
45
46unsigned char shutdown_timeout = 5;
47module_param(shutdown_timeout, byte, 0644);
48MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
49
50static int nvme_major;
51module_param(nvme_major, int, 0);
52

--- 54 unchanged lines hidden (view full) ---

107 unsigned long flags;
108
109 blk_mq_requeue_request(req);
110 spin_lock_irqsave(req->q->queue_lock, flags);
111 if (!blk_queue_stopped(req->q))
112 blk_mq_kick_requeue_list(req->q);
113 spin_unlock_irqrestore(req->q->queue_lock, flags);
114}
115EXPORT_SYMBOL_GPL(nvme_requeue_req);
113
114struct request *nvme_alloc_request(struct request_queue *q,
115 struct nvme_command *cmd, unsigned int flags)
116{
117 bool write = cmd->common.opcode & 1;
118 struct request *req;
119
120 req = blk_mq_alloc_request(q, write, flags);

--- 7 unchanged lines hidden (view full) ---

128 req->bio = req->biotail = NULL;
129
130 req->cmd = (unsigned char *)cmd;
131 req->cmd_len = sizeof(struct nvme_command);
132 req->special = (void *)0;
133
134 return req;
135}
116
117struct request *nvme_alloc_request(struct request_queue *q,
118 struct nvme_command *cmd, unsigned int flags)
119{
120 bool write = cmd->common.opcode & 1;
121 struct request *req;
122
123 req = blk_mq_alloc_request(q, write, flags);

--- 7 unchanged lines hidden (view full) ---

131 req->bio = req->biotail = NULL;
132
133 req->cmd = (unsigned char *)cmd;
134 req->cmd_len = sizeof(struct nvme_command);
135 req->special = (void *)0;
136
137 return req;
138}
139EXPORT_SYMBOL_GPL(nvme_alloc_request);
136
137/*
138 * Returns 0 on success. If the result is negative, it's a Linux error code;
139 * if the result is positive, it's an NVM Express status code
140 */
141int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
142 void *buffer, unsigned bufflen, u32 *result, unsigned timeout)
143{

--- 21 unchanged lines hidden (view full) ---

165 return ret;
166}
167
168int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
169 void *buffer, unsigned bufflen)
170{
171 return __nvme_submit_sync_cmd(q, cmd, buffer, bufflen, NULL, 0);
172}
140
141/*
142 * Returns 0 on success. If the result is negative, it's a Linux error code;
143 * if the result is positive, it's an NVM Express status code
144 */
145int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
146 void *buffer, unsigned bufflen, u32 *result, unsigned timeout)
147{

--- 21 unchanged lines hidden (view full) ---

169 return ret;
170}
171
172int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
173 void *buffer, unsigned bufflen)
174{
175 return __nvme_submit_sync_cmd(q, cmd, buffer, bufflen, NULL, 0);
176}
177EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
173
174int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
175 void __user *ubuffer, unsigned bufflen,
176 void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
177 u32 *result, unsigned timeout)
178{
179 bool write = cmd->common.opcode & 1;
180 struct nvme_ns *ns = q->queuedata;

--- 199 unchanged lines hidden (view full) ---

380 &result);
381 if (status)
382 return status;
383
384 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
385 *count = min(*count, nr_io_queues);
386 return 0;
387}
178
179int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
180 void __user *ubuffer, unsigned bufflen,
181 void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
182 u32 *result, unsigned timeout)
183{
184 bool write = cmd->common.opcode & 1;
185 struct nvme_ns *ns = q->queuedata;

--- 199 unchanged lines hidden (view full) ---

385 &result);
386 if (status)
387 return status;
388
389 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
390 *count = min(*count, nr_io_queues);
391 return 0;
392}
393EXPORT_SYMBOL_GPL(nvme_set_queue_count);
388
389static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
390{
391 struct nvme_user_io io;
392 struct nvme_command c;
393 unsigned length, meta_len;
394 void __user *metadata;
395

--- 393 unchanged lines hidden (view full) ---

789 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
790 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
791
792 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
793 if (ret)
794 return ret;
795 return nvme_wait_ready(ctrl, cap, false);
796}
394
395static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
396{
397 struct nvme_user_io io;
398 struct nvme_command c;
399 unsigned length, meta_len;
400 void __user *metadata;
401

--- 393 unchanged lines hidden (view full) ---

795 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
796 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
797
798 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
799 if (ret)
800 return ret;
801 return nvme_wait_ready(ctrl, cap, false);
802}
803EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
797
798int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
799{
800 /*
801 * Default to a 4K page size, with the intention to update this
802 * path in the future to accomodate architectures with differing
803 * kernel and IO page sizes.
804 */

--- 15 unchanged lines hidden (view full) ---

820 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
821 ctrl->ctrl_config |= NVME_CC_ENABLE;
822
823 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
824 if (ret)
825 return ret;
826 return nvme_wait_ready(ctrl, cap, true);
827}
804
805int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
806{
807 /*
808 * Default to a 4K page size, with the intention to update this
809 * path in the future to accomodate architectures with differing
810 * kernel and IO page sizes.
811 */

--- 15 unchanged lines hidden (view full) ---

827 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
828 ctrl->ctrl_config |= NVME_CC_ENABLE;
829
830 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
831 if (ret)
832 return ret;
833 return nvme_wait_ready(ctrl, cap, true);
834}
835EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
828
829int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
830{
831 unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
832 u32 csts;
833 int ret;
834
835 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;

--- 14 unchanged lines hidden (view full) ---

850 dev_err(ctrl->device,
851 "Device shutdown incomplete; abort shutdown\n");
852 return -ENODEV;
853 }
854 }
855
856 return ret;
857}
836
837int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
838{
839 unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
840 u32 csts;
841 int ret;
842
843 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;

--- 14 unchanged lines hidden (view full) ---

858 dev_err(ctrl->device,
859 "Device shutdown incomplete; abort shutdown\n");
860 return -ENODEV;
861 }
862 }
863
864 return ret;
865}
866EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
858
859/*
860 * Initialize the cached copies of the Identify data and various controller
861 * register in our nvme_ctrl structure. This should be called as soon as
862 * the admin queue is fully up and running.
863 */
864int nvme_init_identify(struct nvme_ctrl *ctrl)
865{

--- 45 unchanged lines hidden (view full) ---

911 } else {
912 ctrl->max_hw_sectors = max_hw_sectors;
913 }
914 }
915
916 kfree(id);
917 return 0;
918}
867
868/*
869 * Initialize the cached copies of the Identify data and various controller
870 * register in our nvme_ctrl structure. This should be called as soon as
871 * the admin queue is fully up and running.
872 */
873int nvme_init_identify(struct nvme_ctrl *ctrl)
874{

--- 45 unchanged lines hidden (view full) ---

920 } else {
921 ctrl->max_hw_sectors = max_hw_sectors;
922 }
923 }
924
925 kfree(id);
926 return 0;
927}
928EXPORT_SYMBOL_GPL(nvme_init_identify);
919
920static int nvme_dev_open(struct inode *inode, struct file *file)
921{
922 struct nvme_ctrl *ctrl;
923 int instance = iminor(inode);
924 int ret = -ENODEV;
925
926 spin_lock(&dev_list_lock);

--- 389 unchanged lines hidden (view full) ---

1316 goto done;
1317 }
1318 __nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
1319 done:
1320 list_sort(NULL, &ctrl->namespaces, ns_cmp);
1321 mutex_unlock(&ctrl->namespaces_mutex);
1322 kfree(id);
1323}
929
930static int nvme_dev_open(struct inode *inode, struct file *file)
931{
932 struct nvme_ctrl *ctrl;
933 int instance = iminor(inode);
934 int ret = -ENODEV;
935
936 spin_lock(&dev_list_lock);

--- 389 unchanged lines hidden (view full) ---

1326 goto done;
1327 }
1328 __nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
1329 done:
1330 list_sort(NULL, &ctrl->namespaces, ns_cmp);
1331 mutex_unlock(&ctrl->namespaces_mutex);
1332 kfree(id);
1333}
1334EXPORT_SYMBOL_GPL(nvme_scan_namespaces);
1324
1325void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
1326{
1327 struct nvme_ns *ns, *next;
1328
1329 mutex_lock(&ctrl->namespaces_mutex);
1330 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
1331 nvme_ns_remove(ns);
1332 mutex_unlock(&ctrl->namespaces_mutex);
1333}
1335
1336void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
1337{
1338 struct nvme_ns *ns, *next;
1339
1340 mutex_lock(&ctrl->namespaces_mutex);
1341 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
1342 nvme_ns_remove(ns);
1343 mutex_unlock(&ctrl->namespaces_mutex);
1344}
1345EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
1334
1335static DEFINE_IDA(nvme_instance_ida);
1336
1337static int nvme_set_instance(struct nvme_ctrl *ctrl)
1338{
1339 int instance, error;
1340
1341 do {

--- 15 unchanged lines hidden (view full) ---

1357static void nvme_release_instance(struct nvme_ctrl *ctrl)
1358{
1359 spin_lock(&dev_list_lock);
1360 ida_remove(&nvme_instance_ida, ctrl->instance);
1361 spin_unlock(&dev_list_lock);
1362}
1363
1364void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
1346
1347static DEFINE_IDA(nvme_instance_ida);
1348
1349static int nvme_set_instance(struct nvme_ctrl *ctrl)
1350{
1351 int instance, error;
1352
1353 do {

--- 15 unchanged lines hidden (view full) ---

1369static void nvme_release_instance(struct nvme_ctrl *ctrl)
1370{
1371 spin_lock(&dev_list_lock);
1372 ida_remove(&nvme_instance_ida, ctrl->instance);
1373 spin_unlock(&dev_list_lock);
1374}
1375
1376void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
1365 {
1377{
1366 device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
1367
1368 spin_lock(&dev_list_lock);
1369 list_del(&ctrl->node);
1370 spin_unlock(&dev_list_lock);
1371}
1378 device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
1379
1380 spin_lock(&dev_list_lock);
1381 list_del(&ctrl->node);
1382 spin_unlock(&dev_list_lock);
1383}
1384EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
1372
1373static void nvme_free_ctrl(struct kref *kref)
1374{
1375 struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
1376
1377 put_device(ctrl->device);
1378 nvme_release_instance(ctrl);
1379
1380 ctrl->ops->free_ctrl(ctrl);
1381}
1382
1383void nvme_put_ctrl(struct nvme_ctrl *ctrl)
1384{
1385 kref_put(&ctrl->kref, nvme_free_ctrl);
1386}
1385
1386static void nvme_free_ctrl(struct kref *kref)
1387{
1388 struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
1389
1390 put_device(ctrl->device);
1391 nvme_release_instance(ctrl);
1392
1393 ctrl->ops->free_ctrl(ctrl);
1394}
1395
1396void nvme_put_ctrl(struct nvme_ctrl *ctrl)
1397{
1398 kref_put(&ctrl->kref, nvme_free_ctrl);
1399}
1400EXPORT_SYMBOL_GPL(nvme_put_ctrl);
1387
1388/*
1389 * Initialize a NVMe controller structures. This needs to be called during
1390 * earliest initialization so that we have the initialized structured around
1391 * during probing.
1392 */
1393int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
1394 const struct nvme_ctrl_ops *ops, unsigned long quirks)

--- 26 unchanged lines hidden (view full) ---

1421 spin_unlock(&dev_list_lock);
1422
1423 return 0;
1424out_release_instance:
1425 nvme_release_instance(ctrl);
1426out:
1427 return ret;
1428}
1401
1402/*
1403 * Initialize a NVMe controller structures. This needs to be called during
1404 * earliest initialization so that we have the initialized structured around
1405 * during probing.
1406 */
1407int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
1408 const struct nvme_ctrl_ops *ops, unsigned long quirks)

--- 26 unchanged lines hidden (view full) ---

1435 spin_unlock(&dev_list_lock);
1436
1437 return 0;
1438out_release_instance:
1439 nvme_release_instance(ctrl);
1440out:
1441 return ret;
1442}
1443EXPORT_SYMBOL_GPL(nvme_init_ctrl);
1429
1430void nvme_stop_queues(struct nvme_ctrl *ctrl)
1431{
1432 struct nvme_ns *ns;
1433
1434 mutex_lock(&ctrl->namespaces_mutex);
1435 list_for_each_entry(ns, &ctrl->namespaces, list) {
1436 spin_lock_irq(ns->queue->queue_lock);
1437 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1438 spin_unlock_irq(ns->queue->queue_lock);
1439
1440 blk_mq_cancel_requeue_work(ns->queue);
1441 blk_mq_stop_hw_queues(ns->queue);
1442 }
1443 mutex_unlock(&ctrl->namespaces_mutex);
1444}
1444
1445void nvme_stop_queues(struct nvme_ctrl *ctrl)
1446{
1447 struct nvme_ns *ns;
1448
1449 mutex_lock(&ctrl->namespaces_mutex);
1450 list_for_each_entry(ns, &ctrl->namespaces, list) {
1451 spin_lock_irq(ns->queue->queue_lock);
1452 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1453 spin_unlock_irq(ns->queue->queue_lock);
1454
1455 blk_mq_cancel_requeue_work(ns->queue);
1456 blk_mq_stop_hw_queues(ns->queue);
1457 }
1458 mutex_unlock(&ctrl->namespaces_mutex);
1459}
1460EXPORT_SYMBOL_GPL(nvme_stop_queues);
1445
1446void nvme_start_queues(struct nvme_ctrl *ctrl)
1447{
1448 struct nvme_ns *ns;
1449
1450 mutex_lock(&ctrl->namespaces_mutex);
1451 list_for_each_entry(ns, &ctrl->namespaces, list) {
1452 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
1453 blk_mq_start_stopped_hw_queues(ns->queue, true);
1454 blk_mq_kick_requeue_list(ns->queue);
1455 }
1456 mutex_unlock(&ctrl->namespaces_mutex);
1457}
1461
1462void nvme_start_queues(struct nvme_ctrl *ctrl)
1463{
1464 struct nvme_ns *ns;
1465
1466 mutex_lock(&ctrl->namespaces_mutex);
1467 list_for_each_entry(ns, &ctrl->namespaces, list) {
1468 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
1469 blk_mq_start_stopped_hw_queues(ns->queue, true);
1470 blk_mq_kick_requeue_list(ns->queue);
1471 }
1472 mutex_unlock(&ctrl->namespaces_mutex);
1473}
1474EXPORT_SYMBOL_GPL(nvme_start_queues);
1458
1459int __init nvme_core_init(void)
1460{
1461 int result;
1462
1463 result = register_blkdev(nvme_major, "nvme");
1464 if (result < 0)
1465 return result;

--- 23 unchanged lines hidden (view full) ---

1489}
1490
1491void nvme_core_exit(void)
1492{
1493 unregister_blkdev(nvme_major, "nvme");
1494 class_destroy(nvme_class);
1495 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
1496}
1475
1476int __init nvme_core_init(void)
1477{
1478 int result;
1479
1480 result = register_blkdev(nvme_major, "nvme");
1481 if (result < 0)
1482 return result;

--- 23 unchanged lines hidden (view full) ---

1506}
1507
1508void nvme_core_exit(void)
1509{
1510 unregister_blkdev(nvme_major, "nvme");
1511 class_destroy(nvme_class);
1512 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
1513}
1514
1515MODULE_LICENSE("GPL");
1516MODULE_VERSION("1.0");
1517module_init(nvme_core_init);
1518module_exit(nvme_core_exit);