1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * virtio_pmem.c: Virtio pmem Driver
4 *
5 * Discovers persistent memory range information
6 * from host and provides a virtio based flushing
7 * interface.
8 */
9 #include "virtio_pmem.h"
10 #include "nd.h"
11
12 /* The interrupt handler */
virtio_pmem_host_ack(struct virtqueue * vq)13 void virtio_pmem_host_ack(struct virtqueue *vq)
14 {
15 struct virtio_pmem *vpmem = vq->vdev->priv;
16 struct virtio_pmem_request *req_data, *req_buf;
17 unsigned long flags;
18 unsigned int len;
19
20 spin_lock_irqsave(&vpmem->pmem_lock, flags);
21 while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) {
22 req_data->done = true;
23 wake_up(&req_data->host_acked);
24
25 if (!list_empty(&vpmem->req_list)) {
26 req_buf = list_first_entry(&vpmem->req_list,
27 struct virtio_pmem_request, list);
28 req_buf->wq_buf_avail = true;
29 wake_up(&req_buf->wq_buf);
30 list_del(&req_buf->list);
31 }
32 }
33 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
34 }
35 EXPORT_SYMBOL_GPL(virtio_pmem_host_ack);
36
37 /* The request submission function */
virtio_pmem_flush(struct nd_region * nd_region)38 static int virtio_pmem_flush(struct nd_region *nd_region)
39 {
40 struct virtio_device *vdev = nd_region->provider_data;
41 struct virtio_pmem *vpmem = vdev->priv;
42 struct virtio_pmem_request *req_data;
43 struct scatterlist *sgs[2], sg, ret;
44 unsigned long flags;
45 int err, err1;
46
47 guard(mutex)(&vpmem->flush_lock);
48
49 /*
50 * Don't bother to submit the request to the device if the device is
51 * not activated.
52 */
53 if (vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_NEEDS_RESET) {
54 dev_info(&vdev->dev, "virtio pmem device needs a reset\n");
55 return -EIO;
56 }
57
58 req_data = kmalloc_obj(*req_data);
59 if (!req_data)
60 return -ENOMEM;
61
62 req_data->done = false;
63 init_waitqueue_head(&req_data->host_acked);
64 init_waitqueue_head(&req_data->wq_buf);
65 INIT_LIST_HEAD(&req_data->list);
66 req_data->req.type = cpu_to_le32(VIRTIO_PMEM_REQ_TYPE_FLUSH);
67 sg_init_one(&sg, &req_data->req, sizeof(req_data->req));
68 sgs[0] = &sg;
69 sg_init_one(&ret, &req_data->resp.ret, sizeof(req_data->resp));
70 sgs[1] = &ret;
71
72 spin_lock_irqsave(&vpmem->pmem_lock, flags);
73 /*
74 * If virtqueue_add_sgs returns -ENOSPC then req_vq virtual
75 * queue does not have free descriptor. We add the request
76 * to req_list and wait for host_ack to wake us up when free
77 * slots are available.
78 */
79 while ((err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data,
80 GFP_ATOMIC)) == -ENOSPC) {
81
82 dev_info(&vdev->dev, "failed to send command to virtio pmem device, no free slots in the virtqueue\n");
83 req_data->wq_buf_avail = false;
84 list_add_tail(&req_data->list, &vpmem->req_list);
85 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
86
87 /* A host response results in "host_ack" getting called */
88 wait_event(req_data->wq_buf, req_data->wq_buf_avail);
89 spin_lock_irqsave(&vpmem->pmem_lock, flags);
90 }
91 err1 = virtqueue_kick(vpmem->req_vq);
92 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
93 /*
94 * virtqueue_add_sgs failed with error different than -ENOSPC, we can't
95 * do anything about that.
96 */
97 if (err || !err1) {
98 dev_info(&vdev->dev, "failed to send command to virtio pmem device\n");
99 err = -EIO;
100 } else {
101 /* A host response results in "host_ack" getting called */
102 wait_event(req_data->host_acked, req_data->done);
103 err = le32_to_cpu(req_data->resp.ret);
104 }
105
106 kfree(req_data);
107 return err;
108 };
109
110 /* The asynchronous flush callback function */
async_pmem_flush(struct nd_region * nd_region,struct bio * bio)111 int async_pmem_flush(struct nd_region *nd_region, struct bio *bio)
112 {
113 /*
114 * Create child bio for asynchronous flush and chain with
115 * parent bio. Otherwise directly call nd_region flush.
116 */
117 if (bio && bio->bi_iter.bi_sector != -1) {
118 struct bio *child = bio_alloc(bio->bi_bdev, 0,
119 REQ_OP_WRITE | REQ_PREFLUSH,
120 GFP_ATOMIC);
121
122 if (!child)
123 return -ENOMEM;
124 bio_clone_blkg_association(child, bio);
125 child->bi_iter.bi_sector = -1;
126 bio_chain(child, bio);
127 submit_bio(child);
128 return 0;
129 }
130 if (virtio_pmem_flush(nd_region))
131 return -EIO;
132
133 return 0;
134 };
135 EXPORT_SYMBOL_GPL(async_pmem_flush);
136 MODULE_DESCRIPTION("Virtio Persistent Memory Driver");
137 MODULE_LICENSE("GPL");
138