1f7a7a5c2SJack Wang // SPDX-License-Identifier: GPL-2.0-or-later 2f7a7a5c2SJack Wang /* 3f7a7a5c2SJack Wang * RDMA Network Block Driver 4f7a7a5c2SJack Wang * 5f7a7a5c2SJack Wang * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. 6f7a7a5c2SJack Wang * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. 7f7a7a5c2SJack Wang * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. 8f7a7a5c2SJack Wang */ 9f7a7a5c2SJack Wang 10f7a7a5c2SJack Wang #undef pr_fmt 11f7a7a5c2SJack Wang #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt 12f7a7a5c2SJack Wang 13f7a7a5c2SJack Wang #include <linux/module.h> 14f7a7a5c2SJack Wang #include <linux/blkdev.h> 15f7a7a5c2SJack Wang #include <linux/hdreg.h> 16f7a7a5c2SJack Wang #include <linux/scatterlist.h> 17f7a7a5c2SJack Wang #include <linux/idr.h> 18f7a7a5c2SJack Wang 19f7a7a5c2SJack Wang #include "rnbd-clt.h" 20f7a7a5c2SJack Wang 21f7a7a5c2SJack Wang MODULE_DESCRIPTION("RDMA Network Block Device Client"); 22f7a7a5c2SJack Wang MODULE_LICENSE("GPL"); 23f7a7a5c2SJack Wang 24f7a7a5c2SJack Wang static int rnbd_client_major; 25f7a7a5c2SJack Wang static DEFINE_IDA(index_ida); 26f7a7a5c2SJack Wang static DEFINE_MUTEX(ida_lock); 27f7a7a5c2SJack Wang static DEFINE_MUTEX(sess_lock); 28f7a7a5c2SJack Wang static LIST_HEAD(sess_list); 29f7a7a5c2SJack Wang 30f7a7a5c2SJack Wang /* 31f7a7a5c2SJack Wang * Maximum number of partitions an instance can have. 32f7a7a5c2SJack Wang * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself) 33f7a7a5c2SJack Wang */ 34f7a7a5c2SJack Wang #define RNBD_PART_BITS 6 35f7a7a5c2SJack Wang 36f7a7a5c2SJack Wang static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess) 37f7a7a5c2SJack Wang { 38f7a7a5c2SJack Wang return refcount_inc_not_zero(&sess->refcount); 39f7a7a5c2SJack Wang } 40f7a7a5c2SJack Wang 41f7a7a5c2SJack Wang static void free_sess(struct rnbd_clt_session *sess); 42f7a7a5c2SJack Wang 43f7a7a5c2SJack Wang static void rnbd_clt_put_sess(struct rnbd_clt_session *sess) 44f7a7a5c2SJack Wang { 45f7a7a5c2SJack Wang might_sleep(); 46f7a7a5c2SJack Wang 47f7a7a5c2SJack Wang if (refcount_dec_and_test(&sess->refcount)) 48f7a7a5c2SJack Wang free_sess(sess); 49f7a7a5c2SJack Wang } 50f7a7a5c2SJack Wang 51f7a7a5c2SJack Wang static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev) 52f7a7a5c2SJack Wang { 53f7a7a5c2SJack Wang might_sleep(); 54f7a7a5c2SJack Wang 55f7a7a5c2SJack Wang if (!refcount_dec_and_test(&dev->refcount)) 56f7a7a5c2SJack Wang return; 57f7a7a5c2SJack Wang 58f7a7a5c2SJack Wang mutex_lock(&ida_lock); 59f7a7a5c2SJack Wang ida_simple_remove(&index_ida, dev->clt_device_id); 60f7a7a5c2SJack Wang mutex_unlock(&ida_lock); 61f7a7a5c2SJack Wang kfree(dev->hw_queues); 6264e8a6ecSMd Haris Iqbal kfree(dev->pathname); 63f7a7a5c2SJack Wang rnbd_clt_put_sess(dev->sess); 64f7a7a5c2SJack Wang mutex_destroy(&dev->lock); 65f7a7a5c2SJack Wang kfree(dev); 66f7a7a5c2SJack Wang } 67f7a7a5c2SJack Wang 68f7a7a5c2SJack Wang static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev) 69f7a7a5c2SJack Wang { 70f7a7a5c2SJack Wang return refcount_inc_not_zero(&dev->refcount); 71f7a7a5c2SJack Wang } 72f7a7a5c2SJack Wang 73f7a7a5c2SJack Wang static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev, 74f7a7a5c2SJack Wang const struct rnbd_msg_open_rsp *rsp) 75f7a7a5c2SJack Wang { 76f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 77f7a7a5c2SJack Wang 78f7a7a5c2SJack Wang if (!rsp->logical_block_size) 79f7a7a5c2SJack Wang return -EINVAL; 80f7a7a5c2SJack Wang 81f7a7a5c2SJack Wang dev->device_id = le32_to_cpu(rsp->device_id); 82f7a7a5c2SJack Wang dev->nsectors = le64_to_cpu(rsp->nsectors); 83f7a7a5c2SJack Wang dev->logical_block_size = le16_to_cpu(rsp->logical_block_size); 84f7a7a5c2SJack Wang dev->physical_block_size = le16_to_cpu(rsp->physical_block_size); 85f7a7a5c2SJack Wang dev->max_write_same_sectors = le32_to_cpu(rsp->max_write_same_sectors); 86f7a7a5c2SJack Wang dev->max_discard_sectors = le32_to_cpu(rsp->max_discard_sectors); 87f7a7a5c2SJack Wang dev->discard_granularity = le32_to_cpu(rsp->discard_granularity); 88f7a7a5c2SJack Wang dev->discard_alignment = le32_to_cpu(rsp->discard_alignment); 89f7a7a5c2SJack Wang dev->secure_discard = le16_to_cpu(rsp->secure_discard); 90f7a7a5c2SJack Wang dev->rotational = rsp->rotational; 91512c781fSGioh Kim dev->wc = !!(rsp->cache_policy & RNBD_WRITEBACK); 92512c781fSGioh Kim dev->fua = !!(rsp->cache_policy & RNBD_FUA); 93f7a7a5c2SJack Wang 94f7a7a5c2SJack Wang dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE; 957404bddeSJack Wang dev->max_segments = sess->max_segments; 96f7a7a5c2SJack Wang 97f7a7a5c2SJack Wang return 0; 98f7a7a5c2SJack Wang } 99f7a7a5c2SJack Wang 100f7a7a5c2SJack Wang static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev, 101f7a7a5c2SJack Wang size_t new_nsectors) 102f7a7a5c2SJack Wang { 103f7a7a5c2SJack Wang rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n", 104f7a7a5c2SJack Wang dev->nsectors, new_nsectors); 105f7a7a5c2SJack Wang dev->nsectors = new_nsectors; 106230272b4SChristoph Hellwig set_capacity_and_notify(dev->gd, dev->nsectors); 107659e56baSChristoph Hellwig return 0; 108f7a7a5c2SJack Wang } 109f7a7a5c2SJack Wang 110f7a7a5c2SJack Wang static int process_msg_open_rsp(struct rnbd_clt_dev *dev, 111f7a7a5c2SJack Wang struct rnbd_msg_open_rsp *rsp) 112f7a7a5c2SJack Wang { 1133ba1c693SMd Haris Iqbal struct kobject *gd_kobj; 114f7a7a5c2SJack Wang int err = 0; 115f7a7a5c2SJack Wang 116f7a7a5c2SJack Wang mutex_lock(&dev->lock); 117f7a7a5c2SJack Wang if (dev->dev_state == DEV_STATE_UNMAPPED) { 118f7a7a5c2SJack Wang rnbd_clt_info(dev, 119f7a7a5c2SJack Wang "Ignoring Open-Response message from server for unmapped device\n"); 120f7a7a5c2SJack Wang err = -ENOENT; 121f7a7a5c2SJack Wang goto out; 122f7a7a5c2SJack Wang } 123f7a7a5c2SJack Wang if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) { 124f7a7a5c2SJack Wang u64 nsectors = le64_to_cpu(rsp->nsectors); 125f7a7a5c2SJack Wang 126f7a7a5c2SJack Wang /* 127f7a7a5c2SJack Wang * If the device was remapped and the size changed in the 128f7a7a5c2SJack Wang * meantime we need to revalidate it 129f7a7a5c2SJack Wang */ 130f7a7a5c2SJack Wang if (dev->nsectors != nsectors) 131f7a7a5c2SJack Wang rnbd_clt_change_capacity(dev, nsectors); 1323ba1c693SMd Haris Iqbal gd_kobj = &disk_to_dev(dev->gd)->kobj; 1333ba1c693SMd Haris Iqbal kobject_uevent(gd_kobj, KOBJ_ONLINE); 134f7a7a5c2SJack Wang rnbd_clt_info(dev, "Device online, device remapped successfully\n"); 135f7a7a5c2SJack Wang } 136f7a7a5c2SJack Wang err = rnbd_clt_set_dev_attr(dev, rsp); 137f7a7a5c2SJack Wang if (err) 138f7a7a5c2SJack Wang goto out; 139f7a7a5c2SJack Wang dev->dev_state = DEV_STATE_MAPPED; 140f7a7a5c2SJack Wang 141f7a7a5c2SJack Wang out: 142f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 143f7a7a5c2SJack Wang 144f7a7a5c2SJack Wang return err; 145f7a7a5c2SJack Wang } 146f7a7a5c2SJack Wang 147f7a7a5c2SJack Wang int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize) 148f7a7a5c2SJack Wang { 149f7a7a5c2SJack Wang int ret = 0; 150f7a7a5c2SJack Wang 151f7a7a5c2SJack Wang mutex_lock(&dev->lock); 152f7a7a5c2SJack Wang if (dev->dev_state != DEV_STATE_MAPPED) { 153f7a7a5c2SJack Wang pr_err("Failed to set new size of the device, device is not opened\n"); 154f7a7a5c2SJack Wang ret = -ENOENT; 155f7a7a5c2SJack Wang goto out; 156f7a7a5c2SJack Wang } 157f7a7a5c2SJack Wang ret = rnbd_clt_change_capacity(dev, newsize); 158f7a7a5c2SJack Wang 159f7a7a5c2SJack Wang out: 160f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 161f7a7a5c2SJack Wang 162f7a7a5c2SJack Wang return ret; 163f7a7a5c2SJack Wang } 164f7a7a5c2SJack Wang 165f7a7a5c2SJack Wang static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q) 166f7a7a5c2SJack Wang { 167f7a7a5c2SJack Wang if (WARN_ON(!q->hctx)) 168f7a7a5c2SJack Wang return; 169f7a7a5c2SJack Wang 170f7a7a5c2SJack Wang /* We can come here from interrupt, thus async=true */ 171f7a7a5c2SJack Wang blk_mq_run_hw_queue(q->hctx, true); 172f7a7a5c2SJack Wang } 173f7a7a5c2SJack Wang 174f7a7a5c2SJack Wang enum { 175f7a7a5c2SJack Wang RNBD_DELAY_IFBUSY = -1, 176f7a7a5c2SJack Wang }; 177f7a7a5c2SJack Wang 178f7a7a5c2SJack Wang /** 179f7a7a5c2SJack Wang * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun 180f7a7a5c2SJack Wang * @sess: Session to find a queue for 181f7a7a5c2SJack Wang * @cpu: Cpu to start the search from 182f7a7a5c2SJack Wang * 183f7a7a5c2SJack Wang * Description: 184f7a7a5c2SJack Wang * Each CPU has a list of HW queues, which needs to be rerun. If a list 185f7a7a5c2SJack Wang * is not empty - it is marked with a bit. This function finds first 186f7a7a5c2SJack Wang * set bit in a bitmap and returns corresponding CPU list. 187f7a7a5c2SJack Wang */ 188f7a7a5c2SJack Wang static struct rnbd_cpu_qlist * 189f7a7a5c2SJack Wang rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu) 190f7a7a5c2SJack Wang { 191f7a7a5c2SJack Wang int bit; 192f7a7a5c2SJack Wang 193f7a7a5c2SJack Wang /* Search from cpu to nr_cpu_ids */ 194f7a7a5c2SJack Wang bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu); 195f7a7a5c2SJack Wang if (bit < nr_cpu_ids) { 196f7a7a5c2SJack Wang return per_cpu_ptr(sess->cpu_queues, bit); 197f7a7a5c2SJack Wang } else if (cpu != 0) { 198f7a7a5c2SJack Wang /* Search from 0 to cpu */ 199f7a7a5c2SJack Wang bit = find_next_bit(sess->cpu_queues_bm, cpu, 0); 200f7a7a5c2SJack Wang if (bit < cpu) 201f7a7a5c2SJack Wang return per_cpu_ptr(sess->cpu_queues, bit); 202f7a7a5c2SJack Wang } 203f7a7a5c2SJack Wang 204f7a7a5c2SJack Wang return NULL; 205f7a7a5c2SJack Wang } 206f7a7a5c2SJack Wang 207f7a7a5c2SJack Wang static inline int nxt_cpu(int cpu) 208f7a7a5c2SJack Wang { 209f7a7a5c2SJack Wang return (cpu + 1) % nr_cpu_ids; 210f7a7a5c2SJack Wang } 211f7a7a5c2SJack Wang 212f7a7a5c2SJack Wang /** 213f7a7a5c2SJack Wang * rnbd_rerun_if_needed() - rerun next queue marked as stopped 214f7a7a5c2SJack Wang * @sess: Session to rerun a queue on 215f7a7a5c2SJack Wang * 216f7a7a5c2SJack Wang * Description: 217f7a7a5c2SJack Wang * Each CPU has it's own list of HW queues, which should be rerun. 218f7a7a5c2SJack Wang * Function finds such list with HW queues, takes a list lock, picks up 219f7a7a5c2SJack Wang * the first HW queue out of the list and requeues it. 220f7a7a5c2SJack Wang * 221f7a7a5c2SJack Wang * Return: 222f7a7a5c2SJack Wang * True if the queue was requeued, false otherwise. 223f7a7a5c2SJack Wang * 224f7a7a5c2SJack Wang * Context: 225f7a7a5c2SJack Wang * Does not matter. 226f7a7a5c2SJack Wang */ 227f7a7a5c2SJack Wang static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess) 228f7a7a5c2SJack Wang { 229f7a7a5c2SJack Wang struct rnbd_queue *q = NULL; 230f7a7a5c2SJack Wang struct rnbd_cpu_qlist *cpu_q; 231f7a7a5c2SJack Wang unsigned long flags; 232f7a7a5c2SJack Wang int *cpup; 233f7a7a5c2SJack Wang 234f7a7a5c2SJack Wang /* 235f7a7a5c2SJack Wang * To keep fairness and not to let other queues starve we always 236f7a7a5c2SJack Wang * try to wake up someone else in round-robin manner. That of course 237f7a7a5c2SJack Wang * increases latency but queues always have a chance to be executed. 238f7a7a5c2SJack Wang */ 239f7a7a5c2SJack Wang cpup = get_cpu_ptr(sess->cpu_rr); 240f7a7a5c2SJack Wang for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q; 241f7a7a5c2SJack Wang cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) { 242f7a7a5c2SJack Wang if (!spin_trylock_irqsave(&cpu_q->requeue_lock, flags)) 243f7a7a5c2SJack Wang continue; 2441e31016bSGioh Kim if (!test_bit(cpu_q->cpu, sess->cpu_queues_bm)) 245f7a7a5c2SJack Wang goto unlock; 246f7a7a5c2SJack Wang q = list_first_entry_or_null(&cpu_q->requeue_list, 247f7a7a5c2SJack Wang typeof(*q), requeue_list); 248f7a7a5c2SJack Wang if (WARN_ON(!q)) 249f7a7a5c2SJack Wang goto clear_bit; 250f7a7a5c2SJack Wang list_del_init(&q->requeue_list); 251f7a7a5c2SJack Wang clear_bit_unlock(0, &q->in_list); 252f7a7a5c2SJack Wang 253f7a7a5c2SJack Wang if (list_empty(&cpu_q->requeue_list)) { 254f7a7a5c2SJack Wang /* Clear bit if nothing is left */ 255f7a7a5c2SJack Wang clear_bit: 256f7a7a5c2SJack Wang clear_bit(cpu_q->cpu, sess->cpu_queues_bm); 257f7a7a5c2SJack Wang } 258f7a7a5c2SJack Wang unlock: 259f7a7a5c2SJack Wang spin_unlock_irqrestore(&cpu_q->requeue_lock, flags); 260f7a7a5c2SJack Wang 261f7a7a5c2SJack Wang if (q) 262f7a7a5c2SJack Wang break; 263f7a7a5c2SJack Wang } 264f7a7a5c2SJack Wang 265f7a7a5c2SJack Wang /** 266f7a7a5c2SJack Wang * Saves the CPU that is going to be requeued on the per-cpu var. Just 267f7a7a5c2SJack Wang * incrementing it doesn't work because rnbd_get_cpu_qlist() will 268f7a7a5c2SJack Wang * always return the first CPU with something on the queue list when the 269f7a7a5c2SJack Wang * value stored on the var is greater than the last CPU with something 270f7a7a5c2SJack Wang * on the list. 271f7a7a5c2SJack Wang */ 272f7a7a5c2SJack Wang if (cpu_q) 273f7a7a5c2SJack Wang *cpup = cpu_q->cpu; 27494dace8cSGioh Kim put_cpu_ptr(sess->cpu_rr); 275f7a7a5c2SJack Wang 276f7a7a5c2SJack Wang if (q) 277f7a7a5c2SJack Wang rnbd_clt_dev_requeue(q); 278f7a7a5c2SJack Wang 279f7a7a5c2SJack Wang return q; 280f7a7a5c2SJack Wang } 281f7a7a5c2SJack Wang 282f7a7a5c2SJack Wang /** 283f7a7a5c2SJack Wang * rnbd_rerun_all_if_idle() - rerun all queues left in the list if 284f7a7a5c2SJack Wang * session is idling (there are no requests 285f7a7a5c2SJack Wang * in-flight). 286f7a7a5c2SJack Wang * @sess: Session to rerun the queues on 287f7a7a5c2SJack Wang * 288f7a7a5c2SJack Wang * Description: 289f7a7a5c2SJack Wang * This function tries to rerun all stopped queues if there are no 290f7a7a5c2SJack Wang * requests in-flight anymore. This function tries to solve an obvious 291f7a7a5c2SJack Wang * problem, when number of tags < than number of queues (hctx), which 292f7a7a5c2SJack Wang * are stopped and put to sleep. If last permit, which has been just put, 293f7a7a5c2SJack Wang * does not wake up all left queues (hctxs), IO requests hang forever. 294f7a7a5c2SJack Wang * 295f7a7a5c2SJack Wang * That can happen when all number of permits, say N, have been exhausted 296f7a7a5c2SJack Wang * from one CPU, and we have many block devices per session, say M. 297f7a7a5c2SJack Wang * Each block device has it's own queue (hctx) for each CPU, so eventually 298f7a7a5c2SJack Wang * we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids. 299f7a7a5c2SJack Wang * If number of permits N < M x nr_cpu_ids finally we will get an IO hang. 300f7a7a5c2SJack Wang * 301f7a7a5c2SJack Wang * To avoid this hang last caller of rnbd_put_permit() (last caller is the 302f7a7a5c2SJack Wang * one who observes sess->busy == 0) must wake up all remaining queues. 303f7a7a5c2SJack Wang * 304f7a7a5c2SJack Wang * Context: 305f7a7a5c2SJack Wang * Does not matter. 306f7a7a5c2SJack Wang */ 307f7a7a5c2SJack Wang static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess) 308f7a7a5c2SJack Wang { 309f7a7a5c2SJack Wang bool requeued; 310f7a7a5c2SJack Wang 311f7a7a5c2SJack Wang do { 312f7a7a5c2SJack Wang requeued = rnbd_rerun_if_needed(sess); 313f7a7a5c2SJack Wang } while (atomic_read(&sess->busy) == 0 && requeued); 314f7a7a5c2SJack Wang } 315f7a7a5c2SJack Wang 316f7a7a5c2SJack Wang static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess, 317f7a7a5c2SJack Wang enum rtrs_clt_con_type con_type, 3189f455eeaSGioh Kim enum wait_type wait) 319f7a7a5c2SJack Wang { 320f7a7a5c2SJack Wang struct rtrs_permit *permit; 321f7a7a5c2SJack Wang 3229f455eeaSGioh Kim permit = rtrs_clt_get_permit(sess->rtrs, con_type, wait); 3231e31016bSGioh Kim if (permit) 324f7a7a5c2SJack Wang /* We have a subtle rare case here, when all permits can be 325f7a7a5c2SJack Wang * consumed before busy counter increased. This is safe, 326f7a7a5c2SJack Wang * because loser will get NULL as a permit, observe 0 busy 327f7a7a5c2SJack Wang * counter and immediately restart the queue himself. 328f7a7a5c2SJack Wang */ 329f7a7a5c2SJack Wang atomic_inc(&sess->busy); 330f7a7a5c2SJack Wang 331f7a7a5c2SJack Wang return permit; 332f7a7a5c2SJack Wang } 333f7a7a5c2SJack Wang 334f7a7a5c2SJack Wang static void rnbd_put_permit(struct rnbd_clt_session *sess, 335f7a7a5c2SJack Wang struct rtrs_permit *permit) 336f7a7a5c2SJack Wang { 337f7a7a5c2SJack Wang rtrs_clt_put_permit(sess->rtrs, permit); 338f7a7a5c2SJack Wang atomic_dec(&sess->busy); 339f7a7a5c2SJack Wang /* Paired with rnbd_clt_dev_add_to_requeue(). Decrement first 340f7a7a5c2SJack Wang * and then check queue bits. 341f7a7a5c2SJack Wang */ 342f7a7a5c2SJack Wang smp_mb__after_atomic(); 343f7a7a5c2SJack Wang rnbd_rerun_all_if_idle(sess); 344f7a7a5c2SJack Wang } 345f7a7a5c2SJack Wang 346f7a7a5c2SJack Wang static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess, 347f7a7a5c2SJack Wang enum rtrs_clt_con_type con_type, 3489f455eeaSGioh Kim enum wait_type wait) 349f7a7a5c2SJack Wang { 350f7a7a5c2SJack Wang struct rnbd_iu *iu; 351f7a7a5c2SJack Wang struct rtrs_permit *permit; 352f7a7a5c2SJack Wang 3539aaf9a2aSGioh Kim iu = kzalloc(sizeof(*iu), GFP_KERNEL); 354292660faSDima Stepanov if (!iu) 3559aaf9a2aSGioh Kim return NULL; 3569aaf9a2aSGioh Kim 3579f455eeaSGioh Kim permit = rnbd_get_permit(sess, con_type, wait); 3581e31016bSGioh Kim if (!permit) { 3599aaf9a2aSGioh Kim kfree(iu); 360f7a7a5c2SJack Wang return NULL; 3619aaf9a2aSGioh Kim } 3629aaf9a2aSGioh Kim 363f7a7a5c2SJack Wang iu->permit = permit; 364f7a7a5c2SJack Wang /* 365f7a7a5c2SJack Wang * 1st reference is dropped after finishing sending a "user" message, 366f7a7a5c2SJack Wang * 2nd reference is dropped after confirmation with the response is 367f7a7a5c2SJack Wang * returned. 368f7a7a5c2SJack Wang * 1st and 2nd can happen in any order, so the rnbd_iu should be 3693877ece0SJack Wang * released (rtrs_permit returned to rtrs) only after both 370f7a7a5c2SJack Wang * are finished. 371f7a7a5c2SJack Wang */ 372f7a7a5c2SJack Wang atomic_set(&iu->refcount, 2); 373f7a7a5c2SJack Wang init_waitqueue_head(&iu->comp.wait); 374f7a7a5c2SJack Wang iu->comp.errno = INT_MAX; 375f7a7a5c2SJack Wang 37680f99093SGuoqing Jiang if (sg_alloc_table(&iu->sgt, 1, GFP_KERNEL)) { 37780f99093SGuoqing Jiang rnbd_put_permit(sess, permit); 37880f99093SGuoqing Jiang kfree(iu); 37980f99093SGuoqing Jiang return NULL; 38080f99093SGuoqing Jiang } 38180f99093SGuoqing Jiang 382f7a7a5c2SJack Wang return iu; 383f7a7a5c2SJack Wang } 384f7a7a5c2SJack Wang 385f7a7a5c2SJack Wang static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu) 386f7a7a5c2SJack Wang { 3879aaf9a2aSGioh Kim if (atomic_dec_and_test(&iu->refcount)) { 38880f99093SGuoqing Jiang sg_free_table(&iu->sgt); 389f7a7a5c2SJack Wang rnbd_put_permit(sess, iu->permit); 3909aaf9a2aSGioh Kim kfree(iu); 3919aaf9a2aSGioh Kim } 392f7a7a5c2SJack Wang } 393f7a7a5c2SJack Wang 394f7a7a5c2SJack Wang static void rnbd_softirq_done_fn(struct request *rq) 395f7a7a5c2SJack Wang { 396f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = rq->rq_disk->private_data; 397f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 398f7a7a5c2SJack Wang struct rnbd_iu *iu; 399f7a7a5c2SJack Wang 400f7a7a5c2SJack Wang iu = blk_mq_rq_to_pdu(rq); 4015a1328d0SGioh Kim sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT); 402f7a7a5c2SJack Wang rnbd_put_permit(sess, iu->permit); 403f7a7a5c2SJack Wang blk_mq_end_request(rq, errno_to_blk_status(iu->errno)); 404f7a7a5c2SJack Wang } 405f7a7a5c2SJack Wang 406f7a7a5c2SJack Wang static void msg_io_conf(void *priv, int errno) 407f7a7a5c2SJack Wang { 408f7a7a5c2SJack Wang struct rnbd_iu *iu = priv; 409f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = iu->dev; 410f7a7a5c2SJack Wang struct request *rq = iu->rq; 411f7a7a5c2SJack Wang int rw = rq_data_dir(rq); 412f7a7a5c2SJack Wang 413f7a7a5c2SJack Wang iu->errno = errno; 414f7a7a5c2SJack Wang 415f7a7a5c2SJack Wang blk_mq_complete_request(rq); 416f7a7a5c2SJack Wang 417f7a7a5c2SJack Wang if (errno) 418f7a7a5c2SJack Wang rnbd_clt_info_rl(dev, "%s I/O failed with err: %d\n", 419f7a7a5c2SJack Wang rw == READ ? "read" : "write", errno); 420f7a7a5c2SJack Wang } 421f7a7a5c2SJack Wang 422f7a7a5c2SJack Wang static void wake_up_iu_comp(struct rnbd_iu *iu, int errno) 423f7a7a5c2SJack Wang { 424f7a7a5c2SJack Wang iu->comp.errno = errno; 425f7a7a5c2SJack Wang wake_up(&iu->comp.wait); 426f7a7a5c2SJack Wang } 427f7a7a5c2SJack Wang 428f7a7a5c2SJack Wang static void msg_conf(void *priv, int errno) 429f7a7a5c2SJack Wang { 430f7a7a5c2SJack Wang struct rnbd_iu *iu = priv; 431f7a7a5c2SJack Wang 432f7a7a5c2SJack Wang iu->errno = errno; 433f7a7a5c2SJack Wang schedule_work(&iu->work); 434f7a7a5c2SJack Wang } 435f7a7a5c2SJack Wang 436f7a7a5c2SJack Wang static int send_usr_msg(struct rtrs_clt *rtrs, int dir, 43746a99e0cSGuoqing Jiang struct rnbd_iu *iu, struct kvec *vec, 438f7a7a5c2SJack Wang size_t len, struct scatterlist *sg, unsigned int sg_len, 439f7a7a5c2SJack Wang void (*conf)(struct work_struct *work), 4409f455eeaSGioh Kim int *errno, int wait) 441f7a7a5c2SJack Wang { 442f7a7a5c2SJack Wang int err; 443f7a7a5c2SJack Wang struct rtrs_clt_req_ops req_ops; 444f7a7a5c2SJack Wang 445f7a7a5c2SJack Wang INIT_WORK(&iu->work, conf); 446f7a7a5c2SJack Wang req_ops = (struct rtrs_clt_req_ops) { 447f7a7a5c2SJack Wang .priv = iu, 448f7a7a5c2SJack Wang .conf_fn = msg_conf, 449f7a7a5c2SJack Wang }; 450f7a7a5c2SJack Wang err = rtrs_clt_request(dir, &req_ops, rtrs, iu->permit, 45146a99e0cSGuoqing Jiang vec, 1, len, sg, sg_len); 452f7a7a5c2SJack Wang if (!err && wait) { 453f7a7a5c2SJack Wang wait_event(iu->comp.wait, iu->comp.errno != INT_MAX); 454f7a7a5c2SJack Wang *errno = iu->comp.errno; 455f7a7a5c2SJack Wang } else { 456f7a7a5c2SJack Wang *errno = 0; 457f7a7a5c2SJack Wang } 458f7a7a5c2SJack Wang 459f7a7a5c2SJack Wang return err; 460f7a7a5c2SJack Wang } 461f7a7a5c2SJack Wang 462f7a7a5c2SJack Wang static void msg_close_conf(struct work_struct *work) 463f7a7a5c2SJack Wang { 464f7a7a5c2SJack Wang struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work); 465f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = iu->dev; 466f7a7a5c2SJack Wang 467f7a7a5c2SJack Wang wake_up_iu_comp(iu, iu->errno); 468f7a7a5c2SJack Wang rnbd_put_iu(dev->sess, iu); 469f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 470f7a7a5c2SJack Wang } 471f7a7a5c2SJack Wang 4729f455eeaSGioh Kim static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id, 4739f455eeaSGioh Kim enum wait_type wait) 474f7a7a5c2SJack Wang { 475f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 476f7a7a5c2SJack Wang struct rnbd_msg_close msg; 477f7a7a5c2SJack Wang struct rnbd_iu *iu; 478f7a7a5c2SJack Wang struct kvec vec = { 479f7a7a5c2SJack Wang .iov_base = &msg, 480f7a7a5c2SJack Wang .iov_len = sizeof(msg) 481f7a7a5c2SJack Wang }; 482f7a7a5c2SJack Wang int err, errno; 483f7a7a5c2SJack Wang 484f7a7a5c2SJack Wang iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT); 485f7a7a5c2SJack Wang if (!iu) 486f7a7a5c2SJack Wang return -ENOMEM; 487f7a7a5c2SJack Wang 488f7a7a5c2SJack Wang iu->buf = NULL; 489f7a7a5c2SJack Wang iu->dev = dev; 490f7a7a5c2SJack Wang 491f7a7a5c2SJack Wang msg.hdr.type = cpu_to_le16(RNBD_MSG_CLOSE); 492f7a7a5c2SJack Wang msg.device_id = cpu_to_le32(device_id); 493f7a7a5c2SJack Wang 494f7a7a5c2SJack Wang WARN_ON(!rnbd_clt_get_dev(dev)); 49546a99e0cSGuoqing Jiang err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 0, NULL, 0, 496f7a7a5c2SJack Wang msg_close_conf, &errno, wait); 497f7a7a5c2SJack Wang if (err) { 498f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 499f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 500f7a7a5c2SJack Wang } else { 501f7a7a5c2SJack Wang err = errno; 502f7a7a5c2SJack Wang } 503f7a7a5c2SJack Wang 504f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 505f7a7a5c2SJack Wang return err; 506f7a7a5c2SJack Wang } 507f7a7a5c2SJack Wang 508f7a7a5c2SJack Wang static void msg_open_conf(struct work_struct *work) 509f7a7a5c2SJack Wang { 510f7a7a5c2SJack Wang struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work); 511f7a7a5c2SJack Wang struct rnbd_msg_open_rsp *rsp = iu->buf; 512f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = iu->dev; 513f7a7a5c2SJack Wang int errno = iu->errno; 514f7a7a5c2SJack Wang 515f7a7a5c2SJack Wang if (errno) { 516f7a7a5c2SJack Wang rnbd_clt_err(dev, 517f7a7a5c2SJack Wang "Opening failed, server responded: %d\n", 518f7a7a5c2SJack Wang errno); 519f7a7a5c2SJack Wang } else { 520f7a7a5c2SJack Wang errno = process_msg_open_rsp(dev, rsp); 521f7a7a5c2SJack Wang if (errno) { 522f7a7a5c2SJack Wang u32 device_id = le32_to_cpu(rsp->device_id); 523f7a7a5c2SJack Wang /* 524f7a7a5c2SJack Wang * If server thinks its fine, but we fail to process 525f7a7a5c2SJack Wang * then be nice and send a close to server. 526f7a7a5c2SJack Wang */ 5279f455eeaSGioh Kim send_msg_close(dev, device_id, RTRS_PERMIT_NOWAIT); 528f7a7a5c2SJack Wang } 529f7a7a5c2SJack Wang } 530f7a7a5c2SJack Wang kfree(rsp); 531f7a7a5c2SJack Wang wake_up_iu_comp(iu, errno); 532f7a7a5c2SJack Wang rnbd_put_iu(dev->sess, iu); 533f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 534f7a7a5c2SJack Wang } 535f7a7a5c2SJack Wang 536f7a7a5c2SJack Wang static void msg_sess_info_conf(struct work_struct *work) 537f7a7a5c2SJack Wang { 538f7a7a5c2SJack Wang struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work); 539f7a7a5c2SJack Wang struct rnbd_msg_sess_info_rsp *rsp = iu->buf; 540f7a7a5c2SJack Wang struct rnbd_clt_session *sess = iu->sess; 541f7a7a5c2SJack Wang 542f7a7a5c2SJack Wang if (!iu->errno) 543f7a7a5c2SJack Wang sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR); 544f7a7a5c2SJack Wang 545f7a7a5c2SJack Wang kfree(rsp); 546f7a7a5c2SJack Wang wake_up_iu_comp(iu, iu->errno); 547f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 548f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 549f7a7a5c2SJack Wang } 550f7a7a5c2SJack Wang 5519f455eeaSGioh Kim static int send_msg_open(struct rnbd_clt_dev *dev, enum wait_type wait) 552f7a7a5c2SJack Wang { 553f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 554f7a7a5c2SJack Wang struct rnbd_msg_open_rsp *rsp; 555f7a7a5c2SJack Wang struct rnbd_msg_open msg; 556f7a7a5c2SJack Wang struct rnbd_iu *iu; 557f7a7a5c2SJack Wang struct kvec vec = { 558f7a7a5c2SJack Wang .iov_base = &msg, 559f7a7a5c2SJack Wang .iov_len = sizeof(msg) 560f7a7a5c2SJack Wang }; 561f7a7a5c2SJack Wang int err, errno; 562f7a7a5c2SJack Wang 563f7a7a5c2SJack Wang rsp = kzalloc(sizeof(*rsp), GFP_KERNEL); 564f7a7a5c2SJack Wang if (!rsp) 565f7a7a5c2SJack Wang return -ENOMEM; 566f7a7a5c2SJack Wang 567f7a7a5c2SJack Wang iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT); 568f7a7a5c2SJack Wang if (!iu) { 569f7a7a5c2SJack Wang kfree(rsp); 570f7a7a5c2SJack Wang return -ENOMEM; 571f7a7a5c2SJack Wang } 572f7a7a5c2SJack Wang 573f7a7a5c2SJack Wang iu->buf = rsp; 574f7a7a5c2SJack Wang iu->dev = dev; 575f7a7a5c2SJack Wang 5765a1328d0SGioh Kim sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp)); 577f7a7a5c2SJack Wang 578f7a7a5c2SJack Wang msg.hdr.type = cpu_to_le16(RNBD_MSG_OPEN); 579f7a7a5c2SJack Wang msg.access_mode = dev->access_mode; 58057b93ed4SDima Stepanov strscpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name)); 581f7a7a5c2SJack Wang 582f7a7a5c2SJack Wang WARN_ON(!rnbd_clt_get_dev(dev)); 583f7a7a5c2SJack Wang err = send_usr_msg(sess->rtrs, READ, iu, 5845a1328d0SGioh Kim &vec, sizeof(*rsp), iu->sgt.sgl, 1, 585f7a7a5c2SJack Wang msg_open_conf, &errno, wait); 586f7a7a5c2SJack Wang if (err) { 587f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 588f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 589f7a7a5c2SJack Wang kfree(rsp); 590f7a7a5c2SJack Wang } else { 591f7a7a5c2SJack Wang err = errno; 592f7a7a5c2SJack Wang } 593f7a7a5c2SJack Wang 594f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 595f7a7a5c2SJack Wang return err; 596f7a7a5c2SJack Wang } 597f7a7a5c2SJack Wang 5989f455eeaSGioh Kim static int send_msg_sess_info(struct rnbd_clt_session *sess, enum wait_type wait) 599f7a7a5c2SJack Wang { 600f7a7a5c2SJack Wang struct rnbd_msg_sess_info_rsp *rsp; 601f7a7a5c2SJack Wang struct rnbd_msg_sess_info msg; 602f7a7a5c2SJack Wang struct rnbd_iu *iu; 603f7a7a5c2SJack Wang struct kvec vec = { 604f7a7a5c2SJack Wang .iov_base = &msg, 605f7a7a5c2SJack Wang .iov_len = sizeof(msg) 606f7a7a5c2SJack Wang }; 607f7a7a5c2SJack Wang int err, errno; 608f7a7a5c2SJack Wang 609f7a7a5c2SJack Wang rsp = kzalloc(sizeof(*rsp), GFP_KERNEL); 610f7a7a5c2SJack Wang if (!rsp) 611f7a7a5c2SJack Wang return -ENOMEM; 612f7a7a5c2SJack Wang 613f7a7a5c2SJack Wang iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT); 614f7a7a5c2SJack Wang if (!iu) { 615f7a7a5c2SJack Wang kfree(rsp); 616f7a7a5c2SJack Wang return -ENOMEM; 617f7a7a5c2SJack Wang } 618f7a7a5c2SJack Wang 619f7a7a5c2SJack Wang iu->buf = rsp; 620f7a7a5c2SJack Wang iu->sess = sess; 6215a1328d0SGioh Kim sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp)); 622f7a7a5c2SJack Wang 623f7a7a5c2SJack Wang msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO); 624f7a7a5c2SJack Wang msg.ver = RNBD_PROTO_VER_MAJOR; 625f7a7a5c2SJack Wang 626f7a7a5c2SJack Wang if (!rnbd_clt_get_sess(sess)) { 627f7a7a5c2SJack Wang /* 628f7a7a5c2SJack Wang * That can happen only in one case, when RTRS has restablished 629f7a7a5c2SJack Wang * the connection and link_ev() is called, but session is almost 630f7a7a5c2SJack Wang * dead, last reference on session is put and caller is waiting 631f7a7a5c2SJack Wang * for RTRS to close everything. 632f7a7a5c2SJack Wang */ 633f7a7a5c2SJack Wang err = -ENODEV; 634f7a7a5c2SJack Wang goto put_iu; 635f7a7a5c2SJack Wang } 636f7a7a5c2SJack Wang err = send_usr_msg(sess->rtrs, READ, iu, 6375a1328d0SGioh Kim &vec, sizeof(*rsp), iu->sgt.sgl, 1, 638f7a7a5c2SJack Wang msg_sess_info_conf, &errno, wait); 639f7a7a5c2SJack Wang if (err) { 640f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 641f7a7a5c2SJack Wang put_iu: 642f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 643f7a7a5c2SJack Wang kfree(rsp); 644f7a7a5c2SJack Wang } else { 645f7a7a5c2SJack Wang err = errno; 646f7a7a5c2SJack Wang } 647f7a7a5c2SJack Wang rnbd_put_iu(sess, iu); 648f7a7a5c2SJack Wang return err; 649f7a7a5c2SJack Wang } 650f7a7a5c2SJack Wang 651f7a7a5c2SJack Wang static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess) 652f7a7a5c2SJack Wang { 653f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 6543ba1c693SMd Haris Iqbal struct kobject *gd_kobj; 655f7a7a5c2SJack Wang 656f7a7a5c2SJack Wang mutex_lock(&sess->lock); 657f7a7a5c2SJack Wang list_for_each_entry(dev, &sess->devs_list, list) { 658f7a7a5c2SJack Wang rnbd_clt_err(dev, "Device disconnected.\n"); 659f7a7a5c2SJack Wang 660f7a7a5c2SJack Wang mutex_lock(&dev->lock); 6613ba1c693SMd Haris Iqbal if (dev->dev_state == DEV_STATE_MAPPED) { 662f7a7a5c2SJack Wang dev->dev_state = DEV_STATE_MAPPED_DISCONNECTED; 6633ba1c693SMd Haris Iqbal gd_kobj = &disk_to_dev(dev->gd)->kobj; 6643ba1c693SMd Haris Iqbal kobject_uevent(gd_kobj, KOBJ_OFFLINE); 6653ba1c693SMd Haris Iqbal } 666f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 667f7a7a5c2SJack Wang } 668f7a7a5c2SJack Wang mutex_unlock(&sess->lock); 669f7a7a5c2SJack Wang } 670f7a7a5c2SJack Wang 671f7a7a5c2SJack Wang static void remap_devs(struct rnbd_clt_session *sess) 672f7a7a5c2SJack Wang { 673f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 674f7a7a5c2SJack Wang struct rtrs_attrs attrs; 675f7a7a5c2SJack Wang int err; 676f7a7a5c2SJack Wang 677f7a7a5c2SJack Wang /* 678f7a7a5c2SJack Wang * Careful here: we are called from RTRS link event directly, 679f7a7a5c2SJack Wang * thus we can't send any RTRS request and wait for response 680f7a7a5c2SJack Wang * or RTRS will not be able to complete request with failure 681f7a7a5c2SJack Wang * if something goes wrong (failing of outstanding requests 682f7a7a5c2SJack Wang * happens exactly from the context where we are blocking now). 683f7a7a5c2SJack Wang * 684f7a7a5c2SJack Wang * So to avoid deadlocks each usr message sent from here must 685f7a7a5c2SJack Wang * be asynchronous. 686f7a7a5c2SJack Wang */ 687f7a7a5c2SJack Wang 6889f455eeaSGioh Kim err = send_msg_sess_info(sess, RTRS_PERMIT_NOWAIT); 689f7a7a5c2SJack Wang if (err) { 690f7a7a5c2SJack Wang pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err); 691f7a7a5c2SJack Wang return; 692f7a7a5c2SJack Wang } 693f7a7a5c2SJack Wang 6941056ad82SMd Haris Iqbal err = rtrs_clt_query(sess->rtrs, &attrs); 6951056ad82SMd Haris Iqbal if (err) { 6961056ad82SMd Haris Iqbal pr_err("rtrs_clt_query(\"%s\"): %d\n", sess->sessname, err); 6971056ad82SMd Haris Iqbal return; 6981056ad82SMd Haris Iqbal } 699f7a7a5c2SJack Wang mutex_lock(&sess->lock); 700f7a7a5c2SJack Wang sess->max_io_size = attrs.max_io_size; 701f7a7a5c2SJack Wang 702f7a7a5c2SJack Wang list_for_each_entry(dev, &sess->devs_list, list) { 703f7a7a5c2SJack Wang bool skip; 704f7a7a5c2SJack Wang 705f7a7a5c2SJack Wang mutex_lock(&dev->lock); 706f7a7a5c2SJack Wang skip = (dev->dev_state == DEV_STATE_INIT); 707f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 708f7a7a5c2SJack Wang if (skip) 709f7a7a5c2SJack Wang /* 710f7a7a5c2SJack Wang * When device is establishing connection for the first 711f7a7a5c2SJack Wang * time - do not remap, it will be closed soon. 712f7a7a5c2SJack Wang */ 713f7a7a5c2SJack Wang continue; 714f7a7a5c2SJack Wang 715f7a7a5c2SJack Wang rnbd_clt_info(dev, "session reconnected, remapping device\n"); 7169f455eeaSGioh Kim err = send_msg_open(dev, RTRS_PERMIT_NOWAIT); 717f7a7a5c2SJack Wang if (err) { 718f7a7a5c2SJack Wang rnbd_clt_err(dev, "send_msg_open(): %d\n", err); 719f7a7a5c2SJack Wang break; 720f7a7a5c2SJack Wang } 721f7a7a5c2SJack Wang } 722f7a7a5c2SJack Wang mutex_unlock(&sess->lock); 723f7a7a5c2SJack Wang } 724f7a7a5c2SJack Wang 725f7a7a5c2SJack Wang static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev) 726f7a7a5c2SJack Wang { 727f7a7a5c2SJack Wang struct rnbd_clt_session *sess = priv; 728f7a7a5c2SJack Wang 729f7a7a5c2SJack Wang switch (ev) { 730f7a7a5c2SJack Wang case RTRS_CLT_LINK_EV_DISCONNECTED: 731f7a7a5c2SJack Wang set_dev_states_to_disconnected(sess); 732f7a7a5c2SJack Wang break; 733f7a7a5c2SJack Wang case RTRS_CLT_LINK_EV_RECONNECTED: 734f7a7a5c2SJack Wang remap_devs(sess); 735f7a7a5c2SJack Wang break; 736f7a7a5c2SJack Wang default: 737f7a7a5c2SJack Wang pr_err("Unknown session event received (%d), session: %s\n", 738f7a7a5c2SJack Wang ev, sess->sessname); 739f7a7a5c2SJack Wang } 740f7a7a5c2SJack Wang } 741f7a7a5c2SJack Wang 742f7a7a5c2SJack Wang static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues) 743f7a7a5c2SJack Wang { 744f7a7a5c2SJack Wang unsigned int cpu; 745f7a7a5c2SJack Wang struct rnbd_cpu_qlist *cpu_q; 746f7a7a5c2SJack Wang 747f7a7a5c2SJack Wang for_each_possible_cpu(cpu) { 748f7a7a5c2SJack Wang cpu_q = per_cpu_ptr(cpu_queues, cpu); 749f7a7a5c2SJack Wang 750f7a7a5c2SJack Wang cpu_q->cpu = cpu; 751f7a7a5c2SJack Wang INIT_LIST_HEAD(&cpu_q->requeue_list); 752f7a7a5c2SJack Wang spin_lock_init(&cpu_q->requeue_lock); 753f7a7a5c2SJack Wang } 754f7a7a5c2SJack Wang } 755f7a7a5c2SJack Wang 756f7a7a5c2SJack Wang static void destroy_mq_tags(struct rnbd_clt_session *sess) 757f7a7a5c2SJack Wang { 758f7a7a5c2SJack Wang if (sess->tag_set.tags) 759f7a7a5c2SJack Wang blk_mq_free_tag_set(&sess->tag_set); 760f7a7a5c2SJack Wang } 761f7a7a5c2SJack Wang 762f7a7a5c2SJack Wang static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess) 763f7a7a5c2SJack Wang { 764f7a7a5c2SJack Wang sess->rtrs_ready = true; 765f7a7a5c2SJack Wang wake_up_all(&sess->rtrs_waitq); 766f7a7a5c2SJack Wang } 767f7a7a5c2SJack Wang 768f7a7a5c2SJack Wang static void close_rtrs(struct rnbd_clt_session *sess) 769f7a7a5c2SJack Wang { 770f7a7a5c2SJack Wang might_sleep(); 771f7a7a5c2SJack Wang 772f7a7a5c2SJack Wang if (!IS_ERR_OR_NULL(sess->rtrs)) { 773f7a7a5c2SJack Wang rtrs_clt_close(sess->rtrs); 774f7a7a5c2SJack Wang sess->rtrs = NULL; 775f7a7a5c2SJack Wang wake_up_rtrs_waiters(sess); 776f7a7a5c2SJack Wang } 777f7a7a5c2SJack Wang } 778f7a7a5c2SJack Wang 779f7a7a5c2SJack Wang static void free_sess(struct rnbd_clt_session *sess) 780f7a7a5c2SJack Wang { 781f7a7a5c2SJack Wang WARN_ON(!list_empty(&sess->devs_list)); 782f7a7a5c2SJack Wang 783f7a7a5c2SJack Wang might_sleep(); 784f7a7a5c2SJack Wang 785f7a7a5c2SJack Wang close_rtrs(sess); 786f7a7a5c2SJack Wang destroy_mq_tags(sess); 787f7a7a5c2SJack Wang if (!list_empty(&sess->list)) { 788f7a7a5c2SJack Wang mutex_lock(&sess_lock); 789f7a7a5c2SJack Wang list_del(&sess->list); 790f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 791f7a7a5c2SJack Wang } 792f7a7a5c2SJack Wang free_percpu(sess->cpu_queues); 793f7a7a5c2SJack Wang free_percpu(sess->cpu_rr); 794f7a7a5c2SJack Wang mutex_destroy(&sess->lock); 795f7a7a5c2SJack Wang kfree(sess); 796f7a7a5c2SJack Wang } 797f7a7a5c2SJack Wang 798f7a7a5c2SJack Wang static struct rnbd_clt_session *alloc_sess(const char *sessname) 799f7a7a5c2SJack Wang { 800f7a7a5c2SJack Wang struct rnbd_clt_session *sess; 801f7a7a5c2SJack Wang int err, cpu; 802f7a7a5c2SJack Wang 803f7a7a5c2SJack Wang sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE); 804f7a7a5c2SJack Wang if (!sess) 805f7a7a5c2SJack Wang return ERR_PTR(-ENOMEM); 80657b93ed4SDima Stepanov strscpy(sess->sessname, sessname, sizeof(sess->sessname)); 807f7a7a5c2SJack Wang atomic_set(&sess->busy, 0); 808f7a7a5c2SJack Wang mutex_init(&sess->lock); 809f7a7a5c2SJack Wang INIT_LIST_HEAD(&sess->devs_list); 810f7a7a5c2SJack Wang INIT_LIST_HEAD(&sess->list); 811292660faSDima Stepanov bitmap_zero(sess->cpu_queues_bm, num_possible_cpus()); 812f7a7a5c2SJack Wang init_waitqueue_head(&sess->rtrs_waitq); 813f7a7a5c2SJack Wang refcount_set(&sess->refcount, 1); 814f7a7a5c2SJack Wang 815f7a7a5c2SJack Wang sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist); 816f7a7a5c2SJack Wang if (!sess->cpu_queues) { 817f7a7a5c2SJack Wang err = -ENOMEM; 818f7a7a5c2SJack Wang goto err; 819f7a7a5c2SJack Wang } 820f7a7a5c2SJack Wang rnbd_init_cpu_qlists(sess->cpu_queues); 821f7a7a5c2SJack Wang 822f7a7a5c2SJack Wang /* 8233877ece0SJack Wang * That is simple percpu variable which stores cpu indices, which are 824f7a7a5c2SJack Wang * incremented on each access. We need that for the sake of fairness 825f7a7a5c2SJack Wang * to wake up queues in a round-robin manner. 826f7a7a5c2SJack Wang */ 827f7a7a5c2SJack Wang sess->cpu_rr = alloc_percpu(int); 828f7a7a5c2SJack Wang if (!sess->cpu_rr) { 829f7a7a5c2SJack Wang err = -ENOMEM; 830f7a7a5c2SJack Wang goto err; 831f7a7a5c2SJack Wang } 832f7a7a5c2SJack Wang for_each_possible_cpu(cpu) 833f7a7a5c2SJack Wang * per_cpu_ptr(sess->cpu_rr, cpu) = cpu; 834f7a7a5c2SJack Wang 835f7a7a5c2SJack Wang return sess; 836f7a7a5c2SJack Wang 837f7a7a5c2SJack Wang err: 838f7a7a5c2SJack Wang free_sess(sess); 839f7a7a5c2SJack Wang 840f7a7a5c2SJack Wang return ERR_PTR(err); 841f7a7a5c2SJack Wang } 842f7a7a5c2SJack Wang 843f7a7a5c2SJack Wang static int wait_for_rtrs_connection(struct rnbd_clt_session *sess) 844f7a7a5c2SJack Wang { 845f7a7a5c2SJack Wang wait_event(sess->rtrs_waitq, sess->rtrs_ready); 846f7a7a5c2SJack Wang if (IS_ERR_OR_NULL(sess->rtrs)) 847f7a7a5c2SJack Wang return -ECONNRESET; 848f7a7a5c2SJack Wang 849f7a7a5c2SJack Wang return 0; 850f7a7a5c2SJack Wang } 851f7a7a5c2SJack Wang 852f7a7a5c2SJack Wang static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess) 853f7a7a5c2SJack Wang __releases(&sess_lock) 854f7a7a5c2SJack Wang __acquires(&sess_lock) 855f7a7a5c2SJack Wang { 856f7a7a5c2SJack Wang DEFINE_WAIT(wait); 857f7a7a5c2SJack Wang 858f7a7a5c2SJack Wang prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE); 859f7a7a5c2SJack Wang if (IS_ERR_OR_NULL(sess->rtrs)) { 860f7a7a5c2SJack Wang finish_wait(&sess->rtrs_waitq, &wait); 861f7a7a5c2SJack Wang return; 862f7a7a5c2SJack Wang } 863f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 864f7a7a5c2SJack Wang /* loop in caller, see __find_and_get_sess(). 865f7a7a5c2SJack Wang * You can't leave mutex locked and call schedule(), you will catch a 866f7a7a5c2SJack Wang * deadlock with a caller of free_sess(), which has just put the last 867f7a7a5c2SJack Wang * reference and is about to take the sess_lock in order to delete 868f7a7a5c2SJack Wang * the session from the list. 869f7a7a5c2SJack Wang */ 870f7a7a5c2SJack Wang schedule(); 871f7a7a5c2SJack Wang mutex_lock(&sess_lock); 872f7a7a5c2SJack Wang } 873f7a7a5c2SJack Wang 874f7a7a5c2SJack Wang static struct rnbd_clt_session *__find_and_get_sess(const char *sessname) 875f7a7a5c2SJack Wang __releases(&sess_lock) 876f7a7a5c2SJack Wang __acquires(&sess_lock) 877f7a7a5c2SJack Wang { 878f7a7a5c2SJack Wang struct rnbd_clt_session *sess, *sn; 879f7a7a5c2SJack Wang int err; 880f7a7a5c2SJack Wang 881f7a7a5c2SJack Wang again: 882f7a7a5c2SJack Wang list_for_each_entry_safe(sess, sn, &sess_list, list) { 883f7a7a5c2SJack Wang if (strcmp(sessname, sess->sessname)) 884f7a7a5c2SJack Wang continue; 885f7a7a5c2SJack Wang 886f7a7a5c2SJack Wang if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs)) 887f7a7a5c2SJack Wang /* 888f7a7a5c2SJack Wang * No RTRS connection, session is dying. 889f7a7a5c2SJack Wang */ 890f7a7a5c2SJack Wang continue; 891f7a7a5c2SJack Wang 892f7a7a5c2SJack Wang if (rnbd_clt_get_sess(sess)) { 893f7a7a5c2SJack Wang /* 894f7a7a5c2SJack Wang * Alive session is found, wait for RTRS connection. 895f7a7a5c2SJack Wang */ 896f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 897f7a7a5c2SJack Wang err = wait_for_rtrs_connection(sess); 898f7a7a5c2SJack Wang if (err) 899f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 900f7a7a5c2SJack Wang mutex_lock(&sess_lock); 901f7a7a5c2SJack Wang 902f7a7a5c2SJack Wang if (err) 903f7a7a5c2SJack Wang /* Session is dying, repeat the loop */ 904f7a7a5c2SJack Wang goto again; 905f7a7a5c2SJack Wang 906f7a7a5c2SJack Wang return sess; 907f7a7a5c2SJack Wang } 908f7a7a5c2SJack Wang /* 909f7a7a5c2SJack Wang * Ref is 0, session is dying, wait for RTRS disconnect 910f7a7a5c2SJack Wang * in order to avoid session names clashes. 911f7a7a5c2SJack Wang */ 912f7a7a5c2SJack Wang wait_for_rtrs_disconnection(sess); 913f7a7a5c2SJack Wang /* 914f7a7a5c2SJack Wang * RTRS is disconnected and soon session will be freed, 915f7a7a5c2SJack Wang * so repeat a loop. 916f7a7a5c2SJack Wang */ 917f7a7a5c2SJack Wang goto again; 918f7a7a5c2SJack Wang } 919f7a7a5c2SJack Wang 920f7a7a5c2SJack Wang return NULL; 921f7a7a5c2SJack Wang } 922f7a7a5c2SJack Wang 923ce9d2b4fSTom Rix /* caller is responsible for initializing 'first' to false */ 924f7a7a5c2SJack Wang static struct 925f7a7a5c2SJack Wang rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first) 926f7a7a5c2SJack Wang { 927f7a7a5c2SJack Wang struct rnbd_clt_session *sess = NULL; 928f7a7a5c2SJack Wang 929f7a7a5c2SJack Wang mutex_lock(&sess_lock); 930f7a7a5c2SJack Wang sess = __find_and_get_sess(sessname); 931f7a7a5c2SJack Wang if (!sess) { 932f7a7a5c2SJack Wang sess = alloc_sess(sessname); 93347393fb5SDan Carpenter if (IS_ERR(sess)) { 93447393fb5SDan Carpenter mutex_unlock(&sess_lock); 93547393fb5SDan Carpenter return sess; 93647393fb5SDan Carpenter } 937f7a7a5c2SJack Wang list_add(&sess->list, &sess_list); 938f7a7a5c2SJack Wang *first = true; 939ce9d2b4fSTom Rix } 940f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 941f7a7a5c2SJack Wang 942f7a7a5c2SJack Wang return sess; 943f7a7a5c2SJack Wang } 944f7a7a5c2SJack Wang 945f7a7a5c2SJack Wang static int rnbd_client_open(struct block_device *block_device, fmode_t mode) 946f7a7a5c2SJack Wang { 947f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = block_device->bd_disk->private_data; 948f7a7a5c2SJack Wang 949f7a7a5c2SJack Wang if (dev->read_only && (mode & FMODE_WRITE)) 950f7a7a5c2SJack Wang return -EPERM; 951f7a7a5c2SJack Wang 952f7a7a5c2SJack Wang if (dev->dev_state == DEV_STATE_UNMAPPED || 953f7a7a5c2SJack Wang !rnbd_clt_get_dev(dev)) 954f7a7a5c2SJack Wang return -EIO; 955f7a7a5c2SJack Wang 956f7a7a5c2SJack Wang return 0; 957f7a7a5c2SJack Wang } 958f7a7a5c2SJack Wang 959f7a7a5c2SJack Wang static void rnbd_client_release(struct gendisk *gen, fmode_t mode) 960f7a7a5c2SJack Wang { 961f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = gen->private_data; 962f7a7a5c2SJack Wang 963f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 964f7a7a5c2SJack Wang } 965f7a7a5c2SJack Wang 966f7a7a5c2SJack Wang static int rnbd_client_getgeo(struct block_device *block_device, 967f7a7a5c2SJack Wang struct hd_geometry *geo) 968f7a7a5c2SJack Wang { 969f7a7a5c2SJack Wang u64 size; 970f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 971f7a7a5c2SJack Wang 972f7a7a5c2SJack Wang dev = block_device->bd_disk->private_data; 973f7a7a5c2SJack Wang size = dev->size * (dev->logical_block_size / SECTOR_SIZE); 974f7a7a5c2SJack Wang geo->cylinders = size >> 6; /* size/64 */ 975f7a7a5c2SJack Wang geo->heads = 4; 976f7a7a5c2SJack Wang geo->sectors = 16; 977f7a7a5c2SJack Wang geo->start = 0; 978f7a7a5c2SJack Wang 979f7a7a5c2SJack Wang return 0; 980f7a7a5c2SJack Wang } 981f7a7a5c2SJack Wang 982f7a7a5c2SJack Wang static const struct block_device_operations rnbd_client_ops = { 983f7a7a5c2SJack Wang .owner = THIS_MODULE, 984f7a7a5c2SJack Wang .open = rnbd_client_open, 985f7a7a5c2SJack Wang .release = rnbd_client_release, 986f7a7a5c2SJack Wang .getgeo = rnbd_client_getgeo 987f7a7a5c2SJack Wang }; 988f7a7a5c2SJack Wang 989f7a7a5c2SJack Wang /* The amount of data that belongs to an I/O and the amount of data that 990f7a7a5c2SJack Wang * should be read or written to the disk (bi_size) can differ. 991f7a7a5c2SJack Wang * 992f7a7a5c2SJack Wang * E.g. When WRITE_SAME is used, only a small amount of data is 993f7a7a5c2SJack Wang * transferred that is then written repeatedly over a lot of sectors. 994f7a7a5c2SJack Wang * 995f7a7a5c2SJack Wang * Get the size of data to be transferred via RTRS by summing up the size 996f7a7a5c2SJack Wang * of the scather-gather list entries. 997f7a7a5c2SJack Wang */ 998f7a7a5c2SJack Wang static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len) 999f7a7a5c2SJack Wang { 1000f7a7a5c2SJack Wang struct scatterlist *sg; 1001f7a7a5c2SJack Wang size_t tsize = 0; 1002f7a7a5c2SJack Wang int i; 1003f7a7a5c2SJack Wang 1004f7a7a5c2SJack Wang for_each_sg(sglist, sg, len, i) 1005f7a7a5c2SJack Wang tsize += sg->length; 1006f7a7a5c2SJack Wang return tsize; 1007f7a7a5c2SJack Wang } 1008f7a7a5c2SJack Wang 1009f7a7a5c2SJack Wang static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev, 1010f7a7a5c2SJack Wang struct request *rq, 1011f7a7a5c2SJack Wang struct rnbd_iu *iu) 1012f7a7a5c2SJack Wang { 1013f7a7a5c2SJack Wang struct rtrs_clt *rtrs = dev->sess->rtrs; 1014f7a7a5c2SJack Wang struct rtrs_permit *permit = iu->permit; 1015f7a7a5c2SJack Wang struct rnbd_msg_io msg; 1016f7a7a5c2SJack Wang struct rtrs_clt_req_ops req_ops; 1017f7a7a5c2SJack Wang unsigned int sg_cnt = 0; 1018f7a7a5c2SJack Wang struct kvec vec; 1019f7a7a5c2SJack Wang size_t size; 1020f7a7a5c2SJack Wang int err; 1021f7a7a5c2SJack Wang 1022f7a7a5c2SJack Wang iu->rq = rq; 1023f7a7a5c2SJack Wang iu->dev = dev; 1024f7a7a5c2SJack Wang msg.sector = cpu_to_le64(blk_rq_pos(rq)); 1025f7a7a5c2SJack Wang msg.bi_size = cpu_to_le32(blk_rq_bytes(rq)); 1026f7a7a5c2SJack Wang msg.rw = cpu_to_le32(rq_to_rnbd_flags(rq)); 1027f7a7a5c2SJack Wang msg.prio = cpu_to_le16(req_get_ioprio(rq)); 1028f7a7a5c2SJack Wang 1029f7a7a5c2SJack Wang /* 1030f7a7a5c2SJack Wang * We only support discards with single segment for now. 1031f7a7a5c2SJack Wang * See queue limits. 1032f7a7a5c2SJack Wang */ 1033f7a7a5c2SJack Wang if (req_op(rq) != REQ_OP_DISCARD) 10345a1328d0SGioh Kim sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sgt.sgl); 1035f7a7a5c2SJack Wang 1036f7a7a5c2SJack Wang if (sg_cnt == 0) 10375a1328d0SGioh Kim sg_mark_end(&iu->sgt.sgl[0]); 1038f7a7a5c2SJack Wang 1039f7a7a5c2SJack Wang msg.hdr.type = cpu_to_le16(RNBD_MSG_IO); 1040f7a7a5c2SJack Wang msg.device_id = cpu_to_le32(dev->device_id); 1041f7a7a5c2SJack Wang 1042f7a7a5c2SJack Wang vec = (struct kvec) { 1043f7a7a5c2SJack Wang .iov_base = &msg, 1044f7a7a5c2SJack Wang .iov_len = sizeof(msg) 1045f7a7a5c2SJack Wang }; 10465a1328d0SGioh Kim size = rnbd_clt_get_sg_size(iu->sgt.sgl, sg_cnt); 1047f7a7a5c2SJack Wang req_ops = (struct rtrs_clt_req_ops) { 1048f7a7a5c2SJack Wang .priv = iu, 1049f7a7a5c2SJack Wang .conf_fn = msg_io_conf, 1050f7a7a5c2SJack Wang }; 1051f7a7a5c2SJack Wang err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit, 10525a1328d0SGioh Kim &vec, 1, size, iu->sgt.sgl, sg_cnt); 10531e31016bSGioh Kim if (err) { 1054f7a7a5c2SJack Wang rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n", 1055f7a7a5c2SJack Wang err); 1056f7a7a5c2SJack Wang return err; 1057f7a7a5c2SJack Wang } 1058f7a7a5c2SJack Wang 1059f7a7a5c2SJack Wang return 0; 1060f7a7a5c2SJack Wang } 1061f7a7a5c2SJack Wang 1062f7a7a5c2SJack Wang /** 1063f7a7a5c2SJack Wang * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy 1064f7a7a5c2SJack Wang * @dev: Device to be checked 1065f7a7a5c2SJack Wang * @q: Queue to be added to the requeue list if required 1066f7a7a5c2SJack Wang * 1067f7a7a5c2SJack Wang * Description: 1068f7a7a5c2SJack Wang * If session is busy, that means someone will requeue us when resources 1069f7a7a5c2SJack Wang * are freed. If session is not doing anything - device is not added to 1070f7a7a5c2SJack Wang * the list and @false is returned. 1071f7a7a5c2SJack Wang */ 1072f7a7a5c2SJack Wang static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev, 1073f7a7a5c2SJack Wang struct rnbd_queue *q) 1074f7a7a5c2SJack Wang { 1075f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 1076f7a7a5c2SJack Wang struct rnbd_cpu_qlist *cpu_q; 1077f7a7a5c2SJack Wang unsigned long flags; 1078f7a7a5c2SJack Wang bool added = true; 1079f7a7a5c2SJack Wang bool need_set; 1080f7a7a5c2SJack Wang 1081f7a7a5c2SJack Wang cpu_q = get_cpu_ptr(sess->cpu_queues); 1082f7a7a5c2SJack Wang spin_lock_irqsave(&cpu_q->requeue_lock, flags); 1083f7a7a5c2SJack Wang 10841e31016bSGioh Kim if (!test_and_set_bit_lock(0, &q->in_list)) { 1085f7a7a5c2SJack Wang if (WARN_ON(!list_empty(&q->requeue_list))) 1086f7a7a5c2SJack Wang goto unlock; 1087f7a7a5c2SJack Wang 1088f7a7a5c2SJack Wang need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm); 1089f7a7a5c2SJack Wang if (need_set) { 1090f7a7a5c2SJack Wang set_bit(cpu_q->cpu, sess->cpu_queues_bm); 1091f7a7a5c2SJack Wang /* Paired with rnbd_put_permit(). Set a bit first 1092f7a7a5c2SJack Wang * and then observe the busy counter. 1093f7a7a5c2SJack Wang */ 1094f7a7a5c2SJack Wang smp_mb__before_atomic(); 1095f7a7a5c2SJack Wang } 10961e31016bSGioh Kim if (atomic_read(&sess->busy)) { 1097f7a7a5c2SJack Wang list_add_tail(&q->requeue_list, &cpu_q->requeue_list); 1098f7a7a5c2SJack Wang } else { 1099f7a7a5c2SJack Wang /* Very unlikely, but possible: busy counter was 1100f7a7a5c2SJack Wang * observed as zero. Drop all bits and return 1101f7a7a5c2SJack Wang * false to restart the queue by ourselves. 1102f7a7a5c2SJack Wang */ 1103f7a7a5c2SJack Wang if (need_set) 1104f7a7a5c2SJack Wang clear_bit(cpu_q->cpu, sess->cpu_queues_bm); 1105f7a7a5c2SJack Wang clear_bit_unlock(0, &q->in_list); 1106f7a7a5c2SJack Wang added = false; 1107f7a7a5c2SJack Wang } 1108f7a7a5c2SJack Wang } 1109f7a7a5c2SJack Wang unlock: 1110f7a7a5c2SJack Wang spin_unlock_irqrestore(&cpu_q->requeue_lock, flags); 1111f7a7a5c2SJack Wang put_cpu_ptr(sess->cpu_queues); 1112f7a7a5c2SJack Wang 1113f7a7a5c2SJack Wang return added; 1114f7a7a5c2SJack Wang } 1115f7a7a5c2SJack Wang 1116f7a7a5c2SJack Wang static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev, 1117f7a7a5c2SJack Wang struct blk_mq_hw_ctx *hctx, 1118f7a7a5c2SJack Wang int delay) 1119f7a7a5c2SJack Wang { 1120f7a7a5c2SJack Wang struct rnbd_queue *q = hctx->driver_data; 1121f7a7a5c2SJack Wang 1122f7a7a5c2SJack Wang if (delay != RNBD_DELAY_IFBUSY) 1123f7a7a5c2SJack Wang blk_mq_delay_run_hw_queue(hctx, delay); 11241e31016bSGioh Kim else if (!rnbd_clt_dev_add_to_requeue(dev, q)) 1125f7a7a5c2SJack Wang /* 1126f7a7a5c2SJack Wang * If session is not busy we have to restart 1127f7a7a5c2SJack Wang * the queue ourselves. 1128f7a7a5c2SJack Wang */ 1129f7a7a5c2SJack Wang blk_mq_delay_run_hw_queue(hctx, 10/*ms*/); 1130f7a7a5c2SJack Wang } 1131f7a7a5c2SJack Wang 1132f7a7a5c2SJack Wang static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx, 1133f7a7a5c2SJack Wang const struct blk_mq_queue_data *bd) 1134f7a7a5c2SJack Wang { 1135f7a7a5c2SJack Wang struct request *rq = bd->rq; 1136f7a7a5c2SJack Wang struct rnbd_clt_dev *dev = rq->rq_disk->private_data; 1137f7a7a5c2SJack Wang struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq); 1138f7a7a5c2SJack Wang int err; 11395a1328d0SGioh Kim blk_status_t ret = BLK_STS_IOERR; 1140f7a7a5c2SJack Wang 11411e31016bSGioh Kim if (dev->dev_state != DEV_STATE_MAPPED) 1142f7a7a5c2SJack Wang return BLK_STS_IOERR; 1143f7a7a5c2SJack Wang 1144f7a7a5c2SJack Wang iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON, 1145f7a7a5c2SJack Wang RTRS_PERMIT_NOWAIT); 11461e31016bSGioh Kim if (!iu->permit) { 1147f7a7a5c2SJack Wang rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY); 1148f7a7a5c2SJack Wang return BLK_STS_RESOURCE; 1149f7a7a5c2SJack Wang } 1150f7a7a5c2SJack Wang 11515a1328d0SGioh Kim iu->sgt.sgl = iu->first_sgl; 11525a1328d0SGioh Kim err = sg_alloc_table_chained(&iu->sgt, 11535a1328d0SGioh Kim /* Even-if the request has no segment, 1154292660faSDima Stepanov * sglist must have one entry at least. 1155292660faSDima Stepanov */ 11565a1328d0SGioh Kim blk_rq_nr_phys_segments(rq) ? : 1, 11575a1328d0SGioh Kim iu->sgt.sgl, 11585a1328d0SGioh Kim RNBD_INLINE_SG_CNT); 11595a1328d0SGioh Kim if (err) { 11605a1328d0SGioh Kim rnbd_clt_err_rl(dev, "sg_alloc_table_chained ret=%d\n", err); 11615a1328d0SGioh Kim rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/); 11625a1328d0SGioh Kim rnbd_put_permit(dev->sess, iu->permit); 11635a1328d0SGioh Kim return BLK_STS_RESOURCE; 11645a1328d0SGioh Kim } 11655a1328d0SGioh Kim 1166f7a7a5c2SJack Wang blk_mq_start_request(rq); 1167f7a7a5c2SJack Wang err = rnbd_client_xfer_request(dev, rq, iu); 11681e31016bSGioh Kim if (err == 0) 1169f7a7a5c2SJack Wang return BLK_STS_OK; 11701e31016bSGioh Kim if (err == -EAGAIN || err == -ENOMEM) { 1171f7a7a5c2SJack Wang rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/); 11725a1328d0SGioh Kim ret = BLK_STS_RESOURCE; 1173f7a7a5c2SJack Wang } 11745a1328d0SGioh Kim sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT); 1175f7a7a5c2SJack Wang rnbd_put_permit(dev->sess, iu->permit); 11765a1328d0SGioh Kim return ret; 1177f7a7a5c2SJack Wang } 1178f7a7a5c2SJack Wang 1179*5a72e899SJens Axboe static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 11802958a995SGioh Kim { 11812958a995SGioh Kim struct rnbd_queue *q = hctx->driver_data; 11822958a995SGioh Kim struct rnbd_clt_dev *dev = q->dev; 11832958a995SGioh Kim int cnt; 11842958a995SGioh Kim 11852958a995SGioh Kim cnt = rtrs_clt_rdma_cq_direct(dev->sess->rtrs, hctx->queue_num); 11862958a995SGioh Kim return cnt; 11872958a995SGioh Kim } 11882958a995SGioh Kim 11892958a995SGioh Kim static int rnbd_rdma_map_queues(struct blk_mq_tag_set *set) 11902958a995SGioh Kim { 11912958a995SGioh Kim struct rnbd_clt_session *sess = set->driver_data; 11922958a995SGioh Kim 11932958a995SGioh Kim /* shared read/write queues */ 11942958a995SGioh Kim set->map[HCTX_TYPE_DEFAULT].nr_queues = num_online_cpus(); 11952958a995SGioh Kim set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 11962958a995SGioh Kim set->map[HCTX_TYPE_READ].nr_queues = num_online_cpus(); 11972958a995SGioh Kim set->map[HCTX_TYPE_READ].queue_offset = 0; 11982958a995SGioh Kim blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 11992958a995SGioh Kim blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 12002958a995SGioh Kim 12012958a995SGioh Kim if (sess->nr_poll_queues) { 12022958a995SGioh Kim /* dedicated queue for poll */ 12032958a995SGioh Kim set->map[HCTX_TYPE_POLL].nr_queues = sess->nr_poll_queues; 12042958a995SGioh Kim set->map[HCTX_TYPE_POLL].queue_offset = set->map[HCTX_TYPE_READ].queue_offset + 12052958a995SGioh Kim set->map[HCTX_TYPE_READ].nr_queues; 12062958a995SGioh Kim blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 12072958a995SGioh Kim pr_info("[session=%s] mapped %d/%d/%d default/read/poll queues.\n", 12082958a995SGioh Kim sess->sessname, 12092958a995SGioh Kim set->map[HCTX_TYPE_DEFAULT].nr_queues, 12102958a995SGioh Kim set->map[HCTX_TYPE_READ].nr_queues, 12112958a995SGioh Kim set->map[HCTX_TYPE_POLL].nr_queues); 12122958a995SGioh Kim } else { 12132958a995SGioh Kim pr_info("[session=%s] mapped %d/%d default/read queues.\n", 12142958a995SGioh Kim sess->sessname, 12152958a995SGioh Kim set->map[HCTX_TYPE_DEFAULT].nr_queues, 12162958a995SGioh Kim set->map[HCTX_TYPE_READ].nr_queues); 12172958a995SGioh Kim } 12182958a995SGioh Kim 12192958a995SGioh Kim return 0; 12202958a995SGioh Kim } 12212958a995SGioh Kim 1222f7a7a5c2SJack Wang static struct blk_mq_ops rnbd_mq_ops = { 1223f7a7a5c2SJack Wang .queue_rq = rnbd_queue_rq, 1224f7a7a5c2SJack Wang .complete = rnbd_softirq_done_fn, 12252958a995SGioh Kim .map_queues = rnbd_rdma_map_queues, 12262958a995SGioh Kim .poll = rnbd_rdma_poll, 1227f7a7a5c2SJack Wang }; 1228f7a7a5c2SJack Wang 1229f7a7a5c2SJack Wang static int setup_mq_tags(struct rnbd_clt_session *sess) 1230f7a7a5c2SJack Wang { 1231f7a7a5c2SJack Wang struct blk_mq_tag_set *tag_set = &sess->tag_set; 1232f7a7a5c2SJack Wang 1233f7a7a5c2SJack Wang memset(tag_set, 0, sizeof(*tag_set)); 1234f7a7a5c2SJack Wang tag_set->ops = &rnbd_mq_ops; 1235f7a7a5c2SJack Wang tag_set->queue_depth = sess->queue_depth; 1236f7a7a5c2SJack Wang tag_set->numa_node = NUMA_NO_NODE; 1237f7a7a5c2SJack Wang tag_set->flags = BLK_MQ_F_SHOULD_MERGE | 123851db1c37SMing Lei BLK_MQ_F_TAG_QUEUE_SHARED; 12395a1328d0SGioh Kim tag_set->cmd_size = sizeof(struct rnbd_iu) + RNBD_RDMA_SGL_SIZE; 12402958a995SGioh Kim 12412958a995SGioh Kim /* for HCTX_TYPE_DEFAULT, HCTX_TYPE_READ, HCTX_TYPE_POLL */ 12422958a995SGioh Kim tag_set->nr_maps = sess->nr_poll_queues ? HCTX_MAX_TYPES : 2; 12432958a995SGioh Kim /* 12442958a995SGioh Kim * HCTX_TYPE_DEFAULT and HCTX_TYPE_READ share one set of queues 12452958a995SGioh Kim * others are for HCTX_TYPE_POLL 12462958a995SGioh Kim */ 12472958a995SGioh Kim tag_set->nr_hw_queues = num_online_cpus() + sess->nr_poll_queues; 12482958a995SGioh Kim tag_set->driver_data = sess; 1249f7a7a5c2SJack Wang 1250f7a7a5c2SJack Wang return blk_mq_alloc_tag_set(tag_set); 1251f7a7a5c2SJack Wang } 1252f7a7a5c2SJack Wang 1253f7a7a5c2SJack Wang static struct rnbd_clt_session * 1254f7a7a5c2SJack Wang find_and_get_or_create_sess(const char *sessname, 1255f7a7a5c2SJack Wang const struct rtrs_addr *paths, 12562958a995SGioh Kim size_t path_cnt, u16 port_nr, u32 nr_poll_queues) 1257f7a7a5c2SJack Wang { 1258f7a7a5c2SJack Wang struct rnbd_clt_session *sess; 1259f7a7a5c2SJack Wang struct rtrs_attrs attrs; 1260f7a7a5c2SJack Wang int err; 1261ce9d2b4fSTom Rix bool first = false; 1262f7a7a5c2SJack Wang struct rtrs_clt_ops rtrs_ops; 1263f7a7a5c2SJack Wang 1264f7a7a5c2SJack Wang sess = find_or_create_sess(sessname, &first); 12652958a995SGioh Kim if (sess == ERR_PTR(-ENOMEM)) 12662958a995SGioh Kim return ERR_PTR(-ENOMEM); 12672958a995SGioh Kim else if ((nr_poll_queues && !first) || (!nr_poll_queues && sess->nr_poll_queues)) { 12682958a995SGioh Kim /* 12692958a995SGioh Kim * A device MUST have its own session to use the polling-mode. 12702958a995SGioh Kim * It must fail to map new device with the same session. 12712958a995SGioh Kim */ 12722958a995SGioh Kim err = -EINVAL; 12732958a995SGioh Kim goto put_sess; 12742958a995SGioh Kim } 12752958a995SGioh Kim 1276ce9d2b4fSTom Rix if (!first) 1277f7a7a5c2SJack Wang return sess; 1278f7a7a5c2SJack Wang 1279ce9fe18aSMd Haris Iqbal if (!path_cnt) { 1280ce9fe18aSMd Haris Iqbal pr_err("Session %s not found, and path parameter not given", sessname); 1281ce9fe18aSMd Haris Iqbal err = -ENXIO; 1282ce9fe18aSMd Haris Iqbal goto put_sess; 1283ce9fe18aSMd Haris Iqbal } 1284ce9fe18aSMd Haris Iqbal 1285f7a7a5c2SJack Wang rtrs_ops = (struct rtrs_clt_ops) { 1286f7a7a5c2SJack Wang .priv = sess, 1287f7a7a5c2SJack Wang .link_ev = rnbd_clt_link_ev, 1288f7a7a5c2SJack Wang }; 1289f7a7a5c2SJack Wang /* 1290f7a7a5c2SJack Wang * Nothing was found, establish rtrs connection and proceed further. 1291f7a7a5c2SJack Wang */ 1292f7a7a5c2SJack Wang sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname, 1293f7a7a5c2SJack Wang paths, path_cnt, port_nr, 12949aaf9a2aSGioh Kim 0, /* Do not use pdu of rtrs */ 12957404bddeSJack Wang RECONNECT_DELAY, 12962958a995SGioh Kim MAX_RECONNECTS, nr_poll_queues); 1297f7a7a5c2SJack Wang if (IS_ERR(sess->rtrs)) { 1298f7a7a5c2SJack Wang err = PTR_ERR(sess->rtrs); 1299f7a7a5c2SJack Wang goto wake_up_and_put; 1300f7a7a5c2SJack Wang } 13011056ad82SMd Haris Iqbal 13021056ad82SMd Haris Iqbal err = rtrs_clt_query(sess->rtrs, &attrs); 13031056ad82SMd Haris Iqbal if (err) 13041056ad82SMd Haris Iqbal goto close_rtrs; 13051056ad82SMd Haris Iqbal 1306f7a7a5c2SJack Wang sess->max_io_size = attrs.max_io_size; 1307f7a7a5c2SJack Wang sess->queue_depth = attrs.queue_depth; 13082958a995SGioh Kim sess->nr_poll_queues = nr_poll_queues; 13097404bddeSJack Wang sess->max_segments = attrs.max_segments; 1310f7a7a5c2SJack Wang 1311f7a7a5c2SJack Wang err = setup_mq_tags(sess); 1312f7a7a5c2SJack Wang if (err) 1313f7a7a5c2SJack Wang goto close_rtrs; 1314f7a7a5c2SJack Wang 13159f455eeaSGioh Kim err = send_msg_sess_info(sess, RTRS_PERMIT_WAIT); 1316f7a7a5c2SJack Wang if (err) 1317f7a7a5c2SJack Wang goto close_rtrs; 1318f7a7a5c2SJack Wang 1319f7a7a5c2SJack Wang wake_up_rtrs_waiters(sess); 1320f7a7a5c2SJack Wang 1321f7a7a5c2SJack Wang return sess; 1322f7a7a5c2SJack Wang 1323f7a7a5c2SJack Wang close_rtrs: 1324f7a7a5c2SJack Wang close_rtrs(sess); 1325f7a7a5c2SJack Wang put_sess: 1326f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 1327f7a7a5c2SJack Wang 1328f7a7a5c2SJack Wang return ERR_PTR(err); 1329f7a7a5c2SJack Wang 1330f7a7a5c2SJack Wang wake_up_and_put: 1331f7a7a5c2SJack Wang wake_up_rtrs_waiters(sess); 1332f7a7a5c2SJack Wang goto put_sess; 1333f7a7a5c2SJack Wang } 1334f7a7a5c2SJack Wang 1335f7a7a5c2SJack Wang static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev, 1336f7a7a5c2SJack Wang struct rnbd_queue *q, 1337f7a7a5c2SJack Wang struct blk_mq_hw_ctx *hctx) 1338f7a7a5c2SJack Wang { 1339f7a7a5c2SJack Wang INIT_LIST_HEAD(&q->requeue_list); 1340f7a7a5c2SJack Wang q->dev = dev; 1341f7a7a5c2SJack Wang q->hctx = hctx; 1342f7a7a5c2SJack Wang } 1343f7a7a5c2SJack Wang 1344f7a7a5c2SJack Wang static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev) 1345f7a7a5c2SJack Wang { 1346f7a7a5c2SJack Wang int i; 1347f7a7a5c2SJack Wang struct blk_mq_hw_ctx *hctx; 1348f7a7a5c2SJack Wang struct rnbd_queue *q; 1349f7a7a5c2SJack Wang 1350f7a7a5c2SJack Wang queue_for_each_hw_ctx(dev->queue, hctx, i) { 1351f7a7a5c2SJack Wang q = &dev->hw_queues[i]; 1352f7a7a5c2SJack Wang rnbd_init_hw_queue(dev, q, hctx); 1353f7a7a5c2SJack Wang hctx->driver_data = q; 1354f7a7a5c2SJack Wang } 1355f7a7a5c2SJack Wang } 1356f7a7a5c2SJack Wang 1357f7a7a5c2SJack Wang static void setup_request_queue(struct rnbd_clt_dev *dev) 1358f7a7a5c2SJack Wang { 1359f7a7a5c2SJack Wang blk_queue_logical_block_size(dev->queue, dev->logical_block_size); 1360f7a7a5c2SJack Wang blk_queue_physical_block_size(dev->queue, dev->physical_block_size); 1361f7a7a5c2SJack Wang blk_queue_max_hw_sectors(dev->queue, dev->max_hw_sectors); 1362f7a7a5c2SJack Wang blk_queue_max_write_same_sectors(dev->queue, 1363f7a7a5c2SJack Wang dev->max_write_same_sectors); 1364f7a7a5c2SJack Wang 1365f7a7a5c2SJack Wang /* 1366f7a7a5c2SJack Wang * we don't support discards to "discontiguous" segments 1367f7a7a5c2SJack Wang * in on request 1368f7a7a5c2SJack Wang */ 1369f7a7a5c2SJack Wang blk_queue_max_discard_segments(dev->queue, 1); 1370f7a7a5c2SJack Wang 1371f7a7a5c2SJack Wang blk_queue_max_discard_sectors(dev->queue, dev->max_discard_sectors); 1372f7a7a5c2SJack Wang dev->queue->limits.discard_granularity = dev->discard_granularity; 1373f7a7a5c2SJack Wang dev->queue->limits.discard_alignment = dev->discard_alignment; 1374f7a7a5c2SJack Wang if (dev->max_discard_sectors) 1375f7a7a5c2SJack Wang blk_queue_flag_set(QUEUE_FLAG_DISCARD, dev->queue); 1376f7a7a5c2SJack Wang if (dev->secure_discard) 1377f7a7a5c2SJack Wang blk_queue_flag_set(QUEUE_FLAG_SECERASE, dev->queue); 1378f7a7a5c2SJack Wang 1379f7a7a5c2SJack Wang blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, dev->queue); 1380f7a7a5c2SJack Wang blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, dev->queue); 1381f7a7a5c2SJack Wang blk_queue_max_segments(dev->queue, dev->max_segments); 1382f7a7a5c2SJack Wang blk_queue_io_opt(dev->queue, dev->sess->max_io_size); 1383f7a7a5c2SJack Wang blk_queue_virt_boundary(dev->queue, SZ_4K - 1); 1384512c781fSGioh Kim blk_queue_write_cache(dev->queue, dev->wc, dev->fua); 1385f7a7a5c2SJack Wang } 1386f7a7a5c2SJack Wang 1387f7a7a5c2SJack Wang static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx) 1388f7a7a5c2SJack Wang { 1389f7a7a5c2SJack Wang dev->gd->major = rnbd_client_major; 1390f7a7a5c2SJack Wang dev->gd->first_minor = idx << RNBD_PART_BITS; 13912c6ee0aeSChristoph Hellwig dev->gd->minors = 1 << RNBD_PART_BITS; 1392f7a7a5c2SJack Wang dev->gd->fops = &rnbd_client_ops; 1393f7a7a5c2SJack Wang dev->gd->queue = dev->queue; 1394f7a7a5c2SJack Wang dev->gd->private_data = dev; 1395f7a7a5c2SJack Wang snprintf(dev->gd->disk_name, sizeof(dev->gd->disk_name), "rnbd%d", 1396f7a7a5c2SJack Wang idx); 1397f7a7a5c2SJack Wang pr_debug("disk_name=%s, capacity=%zu\n", 1398f7a7a5c2SJack Wang dev->gd->disk_name, 1399f7a7a5c2SJack Wang dev->nsectors * (dev->logical_block_size / SECTOR_SIZE) 1400f7a7a5c2SJack Wang ); 1401f7a7a5c2SJack Wang 1402f7a7a5c2SJack Wang set_capacity(dev->gd, dev->nsectors); 1403f7a7a5c2SJack Wang 1404f7a7a5c2SJack Wang if (dev->access_mode == RNBD_ACCESS_RO) { 1405f7a7a5c2SJack Wang dev->read_only = true; 1406f7a7a5c2SJack Wang set_disk_ro(dev->gd, true); 1407f7a7a5c2SJack Wang } else { 1408f7a7a5c2SJack Wang dev->read_only = false; 1409f7a7a5c2SJack Wang } 1410f7a7a5c2SJack Wang 1411f7a7a5c2SJack Wang if (!dev->rotational) 1412f7a7a5c2SJack Wang blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue); 1413d0a70ab1SGuoqing Jiang add_disk(dev->gd); 1414f7a7a5c2SJack Wang } 1415f7a7a5c2SJack Wang 14168b7f0511SGuoqing Jiang static int rnbd_client_setup_device(struct rnbd_clt_dev *dev) 1417f7a7a5c2SJack Wang { 14182c6ee0aeSChristoph Hellwig int idx = dev->clt_device_id; 1419f7a7a5c2SJack Wang 1420f7a7a5c2SJack Wang dev->size = dev->nsectors * dev->logical_block_size; 1421f7a7a5c2SJack Wang 14222c6ee0aeSChristoph Hellwig dev->gd = blk_mq_alloc_disk(&dev->sess->tag_set, dev); 14232c6ee0aeSChristoph Hellwig if (IS_ERR(dev->gd)) 14242c6ee0aeSChristoph Hellwig return PTR_ERR(dev->gd); 14252c6ee0aeSChristoph Hellwig dev->queue = dev->gd->queue; 14262c6ee0aeSChristoph Hellwig rnbd_init_mq_hw_queues(dev); 1427f7a7a5c2SJack Wang 1428f7a7a5c2SJack Wang setup_request_queue(dev); 1429f7a7a5c2SJack Wang rnbd_clt_setup_gen_disk(dev, idx); 1430f7a7a5c2SJack Wang return 0; 1431f7a7a5c2SJack Wang } 1432f7a7a5c2SJack Wang 1433f7a7a5c2SJack Wang static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess, 1434f7a7a5c2SJack Wang enum rnbd_access_mode access_mode, 14352958a995SGioh Kim const char *pathname, 14362958a995SGioh Kim u32 nr_poll_queues) 1437f7a7a5c2SJack Wang { 1438f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 1439f7a7a5c2SJack Wang int ret; 1440f7a7a5c2SJack Wang 1441f7a7a5c2SJack Wang dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, NUMA_NO_NODE); 1442f7a7a5c2SJack Wang if (!dev) 1443f7a7a5c2SJack Wang return ERR_PTR(-ENOMEM); 1444f7a7a5c2SJack Wang 14452958a995SGioh Kim /* 14462958a995SGioh Kim * nr_cpu_ids: the number of softirq queues 14472958a995SGioh Kim * nr_poll_queues: the number of polling queues 14482958a995SGioh Kim */ 14492958a995SGioh Kim dev->hw_queues = kcalloc(nr_cpu_ids + nr_poll_queues, 14502958a995SGioh Kim sizeof(*dev->hw_queues), 1451f7a7a5c2SJack Wang GFP_KERNEL); 1452f7a7a5c2SJack Wang if (!dev->hw_queues) { 1453f7a7a5c2SJack Wang ret = -ENOMEM; 1454f7a7a5c2SJack Wang goto out_alloc; 1455f7a7a5c2SJack Wang } 1456f7a7a5c2SJack Wang 1457f7a7a5c2SJack Wang mutex_lock(&ida_lock); 1458f7a7a5c2SJack Wang ret = ida_simple_get(&index_ida, 0, 1 << (MINORBITS - RNBD_PART_BITS), 1459f7a7a5c2SJack Wang GFP_KERNEL); 1460f7a7a5c2SJack Wang mutex_unlock(&ida_lock); 1461f7a7a5c2SJack Wang if (ret < 0) { 1462f7a7a5c2SJack Wang pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n", 1463f7a7a5c2SJack Wang pathname, sess->sessname, ret); 1464f7a7a5c2SJack Wang goto out_queues; 1465f7a7a5c2SJack Wang } 146664e8a6ecSMd Haris Iqbal 1467e7508d48SMd Haris Iqbal dev->pathname = kstrdup(pathname, GFP_KERNEL); 146864e8a6ecSMd Haris Iqbal if (!dev->pathname) { 146964e8a6ecSMd Haris Iqbal ret = -ENOMEM; 147064e8a6ecSMd Haris Iqbal goto out_queues; 147164e8a6ecSMd Haris Iqbal } 147264e8a6ecSMd Haris Iqbal 1473f7a7a5c2SJack Wang dev->clt_device_id = ret; 1474f7a7a5c2SJack Wang dev->sess = sess; 1475f7a7a5c2SJack Wang dev->access_mode = access_mode; 14762958a995SGioh Kim dev->nr_poll_queues = nr_poll_queues; 1477f7a7a5c2SJack Wang mutex_init(&dev->lock); 1478f7a7a5c2SJack Wang refcount_set(&dev->refcount, 1); 1479f7a7a5c2SJack Wang dev->dev_state = DEV_STATE_INIT; 1480f7a7a5c2SJack Wang 1481f7a7a5c2SJack Wang /* 1482f7a7a5c2SJack Wang * Here we called from sysfs entry, thus clt-sysfs is 1483f7a7a5c2SJack Wang * responsible that session will not disappear. 1484f7a7a5c2SJack Wang */ 1485f7a7a5c2SJack Wang WARN_ON(!rnbd_clt_get_sess(sess)); 1486f7a7a5c2SJack Wang 1487f7a7a5c2SJack Wang return dev; 1488f7a7a5c2SJack Wang 1489f7a7a5c2SJack Wang out_queues: 1490f7a7a5c2SJack Wang kfree(dev->hw_queues); 1491f7a7a5c2SJack Wang out_alloc: 1492f7a7a5c2SJack Wang kfree(dev); 1493f7a7a5c2SJack Wang return ERR_PTR(ret); 1494f7a7a5c2SJack Wang } 1495f7a7a5c2SJack Wang 149691f4acb2SGuoqing Jiang static bool __exists_dev(const char *pathname, const char *sessname) 1497f7a7a5c2SJack Wang { 1498f7a7a5c2SJack Wang struct rnbd_clt_session *sess; 1499f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 1500f7a7a5c2SJack Wang bool found = false; 1501f7a7a5c2SJack Wang 1502f7a7a5c2SJack Wang list_for_each_entry(sess, &sess_list, list) { 150391f4acb2SGuoqing Jiang if (sessname && strncmp(sess->sessname, sessname, 150491f4acb2SGuoqing Jiang sizeof(sess->sessname))) 150591f4acb2SGuoqing Jiang continue; 1506f7a7a5c2SJack Wang mutex_lock(&sess->lock); 1507f7a7a5c2SJack Wang list_for_each_entry(dev, &sess->devs_list, list) { 150864e8a6ecSMd Haris Iqbal if (strlen(dev->pathname) == strlen(pathname) && 150964e8a6ecSMd Haris Iqbal !strcmp(dev->pathname, pathname)) { 1510f7a7a5c2SJack Wang found = true; 1511f7a7a5c2SJack Wang break; 1512f7a7a5c2SJack Wang } 1513f7a7a5c2SJack Wang } 1514f7a7a5c2SJack Wang mutex_unlock(&sess->lock); 1515f7a7a5c2SJack Wang if (found) 1516f7a7a5c2SJack Wang break; 1517f7a7a5c2SJack Wang } 1518f7a7a5c2SJack Wang 1519f7a7a5c2SJack Wang return found; 1520f7a7a5c2SJack Wang } 1521f7a7a5c2SJack Wang 152291f4acb2SGuoqing Jiang static bool exists_devpath(const char *pathname, const char *sessname) 1523f7a7a5c2SJack Wang { 1524f7a7a5c2SJack Wang bool found; 1525f7a7a5c2SJack Wang 1526f7a7a5c2SJack Wang mutex_lock(&sess_lock); 152791f4acb2SGuoqing Jiang found = __exists_dev(pathname, sessname); 1528f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 1529f7a7a5c2SJack Wang 1530f7a7a5c2SJack Wang return found; 1531f7a7a5c2SJack Wang } 1532f7a7a5c2SJack Wang 153302ee80f5SGuoqing Jiang static bool insert_dev_if_not_exists_devpath(struct rnbd_clt_dev *dev) 1534f7a7a5c2SJack Wang { 1535f7a7a5c2SJack Wang bool found; 153602ee80f5SGuoqing Jiang struct rnbd_clt_session *sess = dev->sess; 1537f7a7a5c2SJack Wang 1538f7a7a5c2SJack Wang mutex_lock(&sess_lock); 153902ee80f5SGuoqing Jiang found = __exists_dev(dev->pathname, sess->sessname); 1540f7a7a5c2SJack Wang if (!found) { 1541f7a7a5c2SJack Wang mutex_lock(&sess->lock); 1542f7a7a5c2SJack Wang list_add_tail(&dev->list, &sess->devs_list); 1543f7a7a5c2SJack Wang mutex_unlock(&sess->lock); 1544f7a7a5c2SJack Wang } 1545f7a7a5c2SJack Wang mutex_unlock(&sess_lock); 1546f7a7a5c2SJack Wang 1547f7a7a5c2SJack Wang return found; 1548f7a7a5c2SJack Wang } 1549f7a7a5c2SJack Wang 1550f7a7a5c2SJack Wang static void delete_dev(struct rnbd_clt_dev *dev) 1551f7a7a5c2SJack Wang { 1552f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 1553f7a7a5c2SJack Wang 1554f7a7a5c2SJack Wang mutex_lock(&sess->lock); 1555f7a7a5c2SJack Wang list_del(&dev->list); 1556f7a7a5c2SJack Wang mutex_unlock(&sess->lock); 1557f7a7a5c2SJack Wang } 1558f7a7a5c2SJack Wang 1559f7a7a5c2SJack Wang struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname, 1560f7a7a5c2SJack Wang struct rtrs_addr *paths, 1561f7a7a5c2SJack Wang size_t path_cnt, u16 port_nr, 1562f7a7a5c2SJack Wang const char *pathname, 15632958a995SGioh Kim enum rnbd_access_mode access_mode, 15642958a995SGioh Kim u32 nr_poll_queues) 1565f7a7a5c2SJack Wang { 1566f7a7a5c2SJack Wang struct rnbd_clt_session *sess; 1567f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 1568f7a7a5c2SJack Wang int ret; 1569f7a7a5c2SJack Wang 15701e31016bSGioh Kim if (exists_devpath(pathname, sessname)) 1571f7a7a5c2SJack Wang return ERR_PTR(-EEXIST); 1572f7a7a5c2SJack Wang 15732958a995SGioh Kim sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr, nr_poll_queues); 1574f7a7a5c2SJack Wang if (IS_ERR(sess)) 1575f7a7a5c2SJack Wang return ERR_CAST(sess); 1576f7a7a5c2SJack Wang 15772958a995SGioh Kim dev = init_dev(sess, access_mode, pathname, nr_poll_queues); 1578f7a7a5c2SJack Wang if (IS_ERR(dev)) { 1579f7a7a5c2SJack Wang pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n", 1580f7a7a5c2SJack Wang pathname, sess->sessname, PTR_ERR(dev)); 1581f7a7a5c2SJack Wang ret = PTR_ERR(dev); 1582f7a7a5c2SJack Wang goto put_sess; 1583f7a7a5c2SJack Wang } 158402ee80f5SGuoqing Jiang if (insert_dev_if_not_exists_devpath(dev)) { 1585f7a7a5c2SJack Wang ret = -EEXIST; 1586f7a7a5c2SJack Wang goto put_dev; 1587f7a7a5c2SJack Wang } 15889f455eeaSGioh Kim ret = send_msg_open(dev, RTRS_PERMIT_WAIT); 1589f7a7a5c2SJack Wang if (ret) { 1590f7a7a5c2SJack Wang rnbd_clt_err(dev, 1591f7a7a5c2SJack Wang "map_device: failed, can't open remote device, err: %d\n", 1592f7a7a5c2SJack Wang ret); 1593f7a7a5c2SJack Wang goto del_dev; 1594f7a7a5c2SJack Wang } 1595f7a7a5c2SJack Wang mutex_lock(&dev->lock); 1596f7a7a5c2SJack Wang pr_debug("Opened remote device: session=%s, path='%s'\n", 1597f7a7a5c2SJack Wang sess->sessname, pathname); 15988b7f0511SGuoqing Jiang ret = rnbd_client_setup_device(dev); 1599f7a7a5c2SJack Wang if (ret) { 1600f7a7a5c2SJack Wang rnbd_clt_err(dev, 1601f7a7a5c2SJack Wang "map_device: Failed to configure device, err: %d\n", 1602f7a7a5c2SJack Wang ret); 1603f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 160447be77c2SGioh Kim goto send_close; 1605f7a7a5c2SJack Wang } 1606f7a7a5c2SJack Wang 1607f7a7a5c2SJack Wang rnbd_clt_info(dev, 1608512c781fSGioh Kim "map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d, wc: %d, fua: %d)\n", 1609f7a7a5c2SJack Wang dev->gd->disk_name, dev->nsectors, 1610f7a7a5c2SJack Wang dev->logical_block_size, dev->physical_block_size, 1611f7a7a5c2SJack Wang dev->max_write_same_sectors, dev->max_discard_sectors, 1612f7a7a5c2SJack Wang dev->discard_granularity, dev->discard_alignment, 1613f7a7a5c2SJack Wang dev->secure_discard, dev->max_segments, 1614512c781fSGioh Kim dev->max_hw_sectors, dev->rotational, dev->wc, dev->fua); 1615f7a7a5c2SJack Wang 1616f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 1617f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 1618f7a7a5c2SJack Wang 1619f7a7a5c2SJack Wang return dev; 1620f7a7a5c2SJack Wang 162147be77c2SGioh Kim send_close: 16229f455eeaSGioh Kim send_msg_close(dev, dev->device_id, RTRS_PERMIT_WAIT); 1623f7a7a5c2SJack Wang del_dev: 1624f7a7a5c2SJack Wang delete_dev(dev); 1625f7a7a5c2SJack Wang put_dev: 1626f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 1627f7a7a5c2SJack Wang put_sess: 1628f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 1629f7a7a5c2SJack Wang 1630f7a7a5c2SJack Wang return ERR_PTR(ret); 1631f7a7a5c2SJack Wang } 1632f7a7a5c2SJack Wang 1633f7a7a5c2SJack Wang static void destroy_gen_disk(struct rnbd_clt_dev *dev) 1634f7a7a5c2SJack Wang { 1635f7a7a5c2SJack Wang del_gendisk(dev->gd); 16362c6ee0aeSChristoph Hellwig blk_cleanup_disk(dev->gd); 1637f7a7a5c2SJack Wang } 1638f7a7a5c2SJack Wang 1639f7a7a5c2SJack Wang static void destroy_sysfs(struct rnbd_clt_dev *dev, 1640f7a7a5c2SJack Wang const struct attribute *sysfs_self) 1641f7a7a5c2SJack Wang { 1642f7a7a5c2SJack Wang rnbd_clt_remove_dev_symlink(dev); 1643f7a7a5c2SJack Wang if (dev->kobj.state_initialized) { 1644f7a7a5c2SJack Wang if (sysfs_self) 1645f7a7a5c2SJack Wang /* To avoid deadlock firstly remove itself */ 1646f7a7a5c2SJack Wang sysfs_remove_file_self(&dev->kobj, sysfs_self); 1647f7a7a5c2SJack Wang kobject_del(&dev->kobj); 1648f7a7a5c2SJack Wang kobject_put(&dev->kobj); 1649f7a7a5c2SJack Wang } 1650f7a7a5c2SJack Wang } 1651f7a7a5c2SJack Wang 1652f7a7a5c2SJack Wang int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force, 1653f7a7a5c2SJack Wang const struct attribute *sysfs_self) 1654f7a7a5c2SJack Wang { 1655f7a7a5c2SJack Wang struct rnbd_clt_session *sess = dev->sess; 1656f7a7a5c2SJack Wang int refcount, ret = 0; 1657f7a7a5c2SJack Wang bool was_mapped; 1658f7a7a5c2SJack Wang 1659f7a7a5c2SJack Wang mutex_lock(&dev->lock); 1660f7a7a5c2SJack Wang if (dev->dev_state == DEV_STATE_UNMAPPED) { 1661f7a7a5c2SJack Wang rnbd_clt_info(dev, "Device is already being unmapped\n"); 1662f7a7a5c2SJack Wang ret = -EALREADY; 1663f7a7a5c2SJack Wang goto err; 1664f7a7a5c2SJack Wang } 1665f7a7a5c2SJack Wang refcount = refcount_read(&dev->refcount); 1666f7a7a5c2SJack Wang if (!force && refcount > 1) { 1667f7a7a5c2SJack Wang rnbd_clt_err(dev, 1668f7a7a5c2SJack Wang "Closing device failed, device is in use, (%d device users)\n", 1669f7a7a5c2SJack Wang refcount - 1); 1670f7a7a5c2SJack Wang ret = -EBUSY; 1671f7a7a5c2SJack Wang goto err; 1672f7a7a5c2SJack Wang } 1673f7a7a5c2SJack Wang was_mapped = (dev->dev_state == DEV_STATE_MAPPED); 1674f7a7a5c2SJack Wang dev->dev_state = DEV_STATE_UNMAPPED; 1675f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 1676f7a7a5c2SJack Wang 1677f7a7a5c2SJack Wang delete_dev(dev); 1678f7a7a5c2SJack Wang destroy_sysfs(dev, sysfs_self); 1679f7a7a5c2SJack Wang destroy_gen_disk(dev); 1680f7a7a5c2SJack Wang if (was_mapped && sess->rtrs) 16819f455eeaSGioh Kim send_msg_close(dev, dev->device_id, RTRS_PERMIT_WAIT); 1682f7a7a5c2SJack Wang 1683f7a7a5c2SJack Wang rnbd_clt_info(dev, "Device is unmapped\n"); 1684f7a7a5c2SJack Wang 1685f7a7a5c2SJack Wang /* Likely last reference put */ 1686f7a7a5c2SJack Wang rnbd_clt_put_dev(dev); 1687f7a7a5c2SJack Wang 1688f7a7a5c2SJack Wang /* 1689f7a7a5c2SJack Wang * Here device and session can be vanished! 1690f7a7a5c2SJack Wang */ 1691f7a7a5c2SJack Wang 1692f7a7a5c2SJack Wang return 0; 1693f7a7a5c2SJack Wang err: 1694f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 1695f7a7a5c2SJack Wang 1696f7a7a5c2SJack Wang return ret; 1697f7a7a5c2SJack Wang } 1698f7a7a5c2SJack Wang 1699f7a7a5c2SJack Wang int rnbd_clt_remap_device(struct rnbd_clt_dev *dev) 1700f7a7a5c2SJack Wang { 1701f7a7a5c2SJack Wang int err; 1702f7a7a5c2SJack Wang 1703f7a7a5c2SJack Wang mutex_lock(&dev->lock); 1704f7a7a5c2SJack Wang if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) 1705f7a7a5c2SJack Wang err = 0; 1706f7a7a5c2SJack Wang else if (dev->dev_state == DEV_STATE_UNMAPPED) 1707f7a7a5c2SJack Wang err = -ENODEV; 1708f7a7a5c2SJack Wang else if (dev->dev_state == DEV_STATE_MAPPED) 1709f7a7a5c2SJack Wang err = -EALREADY; 1710f7a7a5c2SJack Wang else 1711f7a7a5c2SJack Wang err = -EBUSY; 1712f7a7a5c2SJack Wang mutex_unlock(&dev->lock); 1713f7a7a5c2SJack Wang if (!err) { 1714f7a7a5c2SJack Wang rnbd_clt_info(dev, "Remapping device.\n"); 17159f455eeaSGioh Kim err = send_msg_open(dev, RTRS_PERMIT_WAIT); 1716f7a7a5c2SJack Wang if (err) 1717f7a7a5c2SJack Wang rnbd_clt_err(dev, "remap_device: %d\n", err); 1718f7a7a5c2SJack Wang } 1719f7a7a5c2SJack Wang 1720f7a7a5c2SJack Wang return err; 1721f7a7a5c2SJack Wang } 1722f7a7a5c2SJack Wang 1723f7a7a5c2SJack Wang static void unmap_device_work(struct work_struct *work) 1724f7a7a5c2SJack Wang { 1725f7a7a5c2SJack Wang struct rnbd_clt_dev *dev; 1726f7a7a5c2SJack Wang 1727f7a7a5c2SJack Wang dev = container_of(work, typeof(*dev), unmap_on_rmmod_work); 1728f7a7a5c2SJack Wang rnbd_clt_unmap_device(dev, true, NULL); 1729f7a7a5c2SJack Wang } 1730f7a7a5c2SJack Wang 1731f7a7a5c2SJack Wang static void rnbd_destroy_sessions(void) 1732f7a7a5c2SJack Wang { 1733f7a7a5c2SJack Wang struct rnbd_clt_session *sess, *sn; 1734f7a7a5c2SJack Wang struct rnbd_clt_dev *dev, *tn; 1735f7a7a5c2SJack Wang 1736f7a7a5c2SJack Wang /* Firstly forbid access through sysfs interface */ 1737f7a7a5c2SJack Wang rnbd_clt_destroy_sysfs_files(); 1738f7a7a5c2SJack Wang 1739f7a7a5c2SJack Wang /* 1740f7a7a5c2SJack Wang * Here at this point there is no any concurrent access to sessions 1741f7a7a5c2SJack Wang * list and devices list: 17423877ece0SJack Wang * 1. New session or device can't be created - session sysfs files 1743f7a7a5c2SJack Wang * are removed. 1744f7a7a5c2SJack Wang * 2. Device or session can't be removed - module reference is taken 1745f7a7a5c2SJack Wang * into account in unmap device sysfs callback. 1746f7a7a5c2SJack Wang * 3. No IO requests inflight - each file open of block_dev increases 1747f7a7a5c2SJack Wang * module reference in get_disk(). 1748f7a7a5c2SJack Wang * 1749f7a7a5c2SJack Wang * But still there can be user requests inflights, which are sent by 1750f7a7a5c2SJack Wang * asynchronous send_msg_*() functions, thus before unmapping devices 1751f7a7a5c2SJack Wang * RTRS session must be explicitly closed. 1752f7a7a5c2SJack Wang */ 1753f7a7a5c2SJack Wang 1754f7a7a5c2SJack Wang list_for_each_entry_safe(sess, sn, &sess_list, list) { 17553a21777cSJack Wang if (!rnbd_clt_get_sess(sess)) 17563a21777cSJack Wang continue; 1757f7a7a5c2SJack Wang close_rtrs(sess); 1758f7a7a5c2SJack Wang list_for_each_entry_safe(dev, tn, &sess->devs_list, list) { 1759f7a7a5c2SJack Wang /* 1760f7a7a5c2SJack Wang * Here unmap happens in parallel for only one reason: 1761f7a7a5c2SJack Wang * blk_cleanup_queue() takes around half a second, so 1762f7a7a5c2SJack Wang * on huge amount of devices the whole module unload 1763f7a7a5c2SJack Wang * procedure takes minutes. 1764f7a7a5c2SJack Wang */ 1765f7a7a5c2SJack Wang INIT_WORK(&dev->unmap_on_rmmod_work, unmap_device_work); 1766f7a7a5c2SJack Wang queue_work(system_long_wq, &dev->unmap_on_rmmod_work); 1767f7a7a5c2SJack Wang } 1768f7a7a5c2SJack Wang rnbd_clt_put_sess(sess); 1769f7a7a5c2SJack Wang } 1770f7a7a5c2SJack Wang /* Wait for all scheduled unmap works */ 1771f7a7a5c2SJack Wang flush_workqueue(system_long_wq); 1772f7a7a5c2SJack Wang WARN_ON(!list_empty(&sess_list)); 1773f7a7a5c2SJack Wang } 1774f7a7a5c2SJack Wang 1775f7a7a5c2SJack Wang static int __init rnbd_client_init(void) 1776f7a7a5c2SJack Wang { 1777f7a7a5c2SJack Wang int err = 0; 1778f7a7a5c2SJack Wang 1779f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4); 1780f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36); 1781f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36); 1782f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264); 1783f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8); 1784f7a7a5c2SJack Wang BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56); 1785f7a7a5c2SJack Wang rnbd_client_major = register_blkdev(rnbd_client_major, "rnbd"); 1786f7a7a5c2SJack Wang if (rnbd_client_major <= 0) { 1787f7a7a5c2SJack Wang pr_err("Failed to load module, block device registration failed\n"); 1788f7a7a5c2SJack Wang return -EBUSY; 1789f7a7a5c2SJack Wang } 1790f7a7a5c2SJack Wang 1791f7a7a5c2SJack Wang err = rnbd_clt_create_sysfs_files(); 1792f7a7a5c2SJack Wang if (err) { 1793f7a7a5c2SJack Wang pr_err("Failed to load module, creating sysfs device files failed, err: %d\n", 1794f7a7a5c2SJack Wang err); 1795f7a7a5c2SJack Wang unregister_blkdev(rnbd_client_major, "rnbd"); 1796f7a7a5c2SJack Wang } 1797f7a7a5c2SJack Wang 1798f7a7a5c2SJack Wang return err; 1799f7a7a5c2SJack Wang } 1800f7a7a5c2SJack Wang 1801f7a7a5c2SJack Wang static void __exit rnbd_client_exit(void) 1802f7a7a5c2SJack Wang { 1803f7a7a5c2SJack Wang rnbd_destroy_sessions(); 1804f7a7a5c2SJack Wang unregister_blkdev(rnbd_client_major, "rnbd"); 1805f7a7a5c2SJack Wang ida_destroy(&index_ida); 1806f7a7a5c2SJack Wang } 1807f7a7a5c2SJack Wang 1808f7a7a5c2SJack Wang module_init(rnbd_client_init); 1809f7a7a5c2SJack Wang module_exit(rnbd_client_exit); 1810