1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * virtio_pmem.c: Virtio pmem Driver
4 *
5 * Discovers persistent memory range information
6 * from host and provides a virtio based flushing
7 * interface.
8 */
9 #include "virtio_pmem.h"
10 #include "nd.h"
11
12 struct virtio_pmem_flush_work {
13 struct work_struct work;
14 struct nd_region *nd_region;
15 struct bio *bio;
16 };
17
virtio_pmem_req_release(struct kref * kref)18 static void virtio_pmem_req_release(struct kref *kref)
19 {
20 struct virtio_pmem_request *req;
21
22 req = container_of(kref, struct virtio_pmem_request, kref);
23 kfree(req);
24 }
25
virtio_pmem_signal_done(struct virtio_pmem_request * req)26 static void virtio_pmem_signal_done(struct virtio_pmem_request *req)
27 {
28 /* Pairs with smp_load_acquire() in virtio_pmem_req_done(). */
29 smp_store_release(&req->done, true);
30 wake_up(&req->host_acked);
31 }
32
virtio_pmem_req_done(struct virtio_pmem_request * req)33 static bool virtio_pmem_req_done(struct virtio_pmem_request *req)
34 {
35 /* Pairs with smp_store_release() in virtio_pmem_signal_done(). */
36 return smp_load_acquire(&req->done);
37 }
38
virtio_pmem_complete_err(struct virtio_pmem_request * req)39 static void virtio_pmem_complete_err(struct virtio_pmem_request *req)
40 {
41 req->resp.ret = cpu_to_le32(1);
42 virtio_pmem_signal_done(req);
43 }
44
virtio_pmem_wake_one_waiter(struct virtio_pmem * vpmem)45 static void virtio_pmem_wake_one_waiter(struct virtio_pmem *vpmem)
46 {
47 struct virtio_pmem_request *req_buf;
48
49 if (list_empty(&vpmem->req_list))
50 return;
51
52 req_buf = list_first_entry(&vpmem->req_list,
53 struct virtio_pmem_request, list);
54 list_del_init(&req_buf->list);
55 WRITE_ONCE(req_buf->wq_buf_avail, true);
56 wake_up(&req_buf->wq_buf);
57 }
58
virtio_pmem_wake_all_waiters(struct virtio_pmem * vpmem)59 static void virtio_pmem_wake_all_waiters(struct virtio_pmem *vpmem)
60 {
61 struct virtio_pmem_request *req, *tmp;
62
63 list_for_each_entry_safe(req, tmp, &vpmem->req_list, list) {
64 list_del_init(&req->list);
65 WRITE_ONCE(req->wq_buf_avail, true);
66 wake_up(&req->wq_buf);
67 }
68 }
69
virtio_pmem_clear_inflight(struct virtio_pmem * vpmem,struct virtio_pmem_request * req)70 static void virtio_pmem_clear_inflight(struct virtio_pmem *vpmem,
71 struct virtio_pmem_request *req)
72 {
73 if (vpmem->req_inflight == req)
74 vpmem->req_inflight = NULL;
75 }
76
virtio_pmem_wake_inflight(struct virtio_pmem * vpmem)77 static void virtio_pmem_wake_inflight(struct virtio_pmem *vpmem)
78 {
79 struct virtio_pmem_request *req = vpmem->req_inflight;
80
81 if (req)
82 wake_up(&req->host_acked);
83 }
84
virtio_pmem_mark_broken(struct virtio_pmem * vpmem)85 void virtio_pmem_mark_broken(struct virtio_pmem *vpmem)
86 {
87 if (!READ_ONCE(vpmem->broken)) {
88 WRITE_ONCE(vpmem->broken, true);
89 dev_err_once(&vpmem->vdev->dev, "virtqueue is broken\n");
90 }
91
92 virtio_pmem_wake_inflight(vpmem);
93 virtio_pmem_wake_all_waiters(vpmem);
94 }
95 EXPORT_SYMBOL_GPL(virtio_pmem_mark_broken);
96
virtio_pmem_drain(struct virtio_pmem * vpmem)97 void virtio_pmem_drain(struct virtio_pmem *vpmem)
98 {
99 struct virtio_pmem_request *req;
100 unsigned int len;
101
102 if (!vpmem->req_vq)
103 return;
104
105 while ((req = virtqueue_get_buf(vpmem->req_vq, &len)) != NULL) {
106 virtio_pmem_clear_inflight(vpmem, req);
107 virtio_pmem_complete_err(req);
108 kref_put(&req->kref, virtio_pmem_req_release);
109 }
110
111 while ((req = virtqueue_detach_unused_buf(vpmem->req_vq)) != NULL) {
112 virtio_pmem_clear_inflight(vpmem, req);
113 virtio_pmem_complete_err(req);
114 kref_put(&req->kref, virtio_pmem_req_release);
115 }
116 }
117 EXPORT_SYMBOL_GPL(virtio_pmem_drain);
118
119 /* The interrupt handler */
virtio_pmem_host_ack(struct virtqueue * vq)120 void virtio_pmem_host_ack(struct virtqueue *vq)
121 {
122 struct virtio_pmem *vpmem = vq->vdev->priv;
123 struct virtio_pmem_request *req_data;
124 unsigned long flags;
125 unsigned int len;
126
127 spin_lock_irqsave(&vpmem->pmem_lock, flags);
128 while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) {
129 virtio_pmem_clear_inflight(vpmem, req_data);
130 virtio_pmem_wake_one_waiter(vpmem);
131 if (READ_ONCE(vpmem->broken))
132 virtio_pmem_complete_err(req_data);
133 else
134 virtio_pmem_signal_done(req_data);
135 kref_put(&req_data->kref, virtio_pmem_req_release);
136 }
137 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
138 }
139 EXPORT_SYMBOL_GPL(virtio_pmem_host_ack);
140
141 /* The request submission function */
virtio_pmem_flush(struct nd_region * nd_region)142 static int virtio_pmem_flush(struct nd_region *nd_region)
143 {
144 struct virtio_device *vdev = nd_region->provider_data;
145 struct virtio_pmem *vpmem = vdev->priv;
146 struct virtio_pmem_request *req_data;
147 struct scatterlist *sgs[2], sg, ret;
148 unsigned long flags;
149 int err, err1;
150
151 guard(mutex)(&vpmem->flush_lock);
152
153 /*
154 * Don't bother to submit the request to the device if the device is
155 * not activated.
156 */
157 if (vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_NEEDS_RESET) {
158 dev_info(&vdev->dev, "virtio pmem device needs a reset\n");
159 return -EIO;
160 }
161
162 if (READ_ONCE(vpmem->broken))
163 return -EIO;
164
165 req_data = kmalloc_obj(*req_data, GFP_NOIO);
166 if (!req_data)
167 return -ENOMEM;
168
169 kref_init(&req_data->kref);
170 WRITE_ONCE(req_data->done, false);
171 init_waitqueue_head(&req_data->host_acked);
172 init_waitqueue_head(&req_data->wq_buf);
173 INIT_LIST_HEAD(&req_data->list);
174 req_data->req.type = cpu_to_le32(VIRTIO_PMEM_REQ_TYPE_FLUSH);
175 sg_init_one(&sg, &req_data->req, sizeof(req_data->req));
176 sgs[0] = &sg;
177 sg_init_one(&ret, &req_data->resp.ret, sizeof(req_data->resp));
178 sgs[1] = &ret;
179
180 spin_lock_irqsave(&vpmem->pmem_lock, flags);
181 /*
182 * If virtqueue_add_sgs returns -ENOSPC then req_vq virtual
183 * queue does not have free descriptor. We add the request
184 * to req_list and wait for host_ack to wake us up when free
185 * slots are available.
186 */
187 for (;;) {
188 if (READ_ONCE(vpmem->broken)) {
189 err = -EIO;
190 break;
191 }
192
193 err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data,
194 GFP_ATOMIC);
195 if (!err) {
196 /*
197 * Take the virtqueue reference while @pmem_lock is
198 * held so completion cannot run concurrently.
199 */
200 kref_get(&req_data->kref);
201 vpmem->req_inflight = req_data;
202 break;
203 }
204
205 if (err != -ENOSPC)
206 break;
207
208 dev_info_ratelimited(&vdev->dev,
209 "failed to send command to virtio pmem device, no free slots in the virtqueue\n");
210 WRITE_ONCE(req_data->wq_buf_avail, false);
211 list_add_tail(&req_data->list, &vpmem->req_list);
212 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
213
214 /* A host response results in "host_ack" getting called */
215 wait_event(req_data->wq_buf,
216 READ_ONCE(req_data->wq_buf_avail) ||
217 READ_ONCE(vpmem->broken));
218 spin_lock_irqsave(&vpmem->pmem_lock, flags);
219
220 if (READ_ONCE(vpmem->broken))
221 break;
222 }
223
224 if (READ_ONCE(vpmem->broken))
225 err = -EIO;
226 if (err == -EIO || virtqueue_is_broken(vpmem->req_vq))
227 virtio_pmem_mark_broken(vpmem);
228
229 err1 = true;
230 if (!err && !READ_ONCE(vpmem->broken)) {
231 err1 = virtqueue_kick(vpmem->req_vq);
232 if (!err1)
233 virtio_pmem_mark_broken(vpmem);
234 }
235 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
236 /*
237 * virtqueue_add_sgs failed with error different than -ENOSPC, we can't
238 * do anything about that.
239 */
240 if (READ_ONCE(vpmem->broken) || err || !err1) {
241 dev_info(&vdev->dev, "failed to send command to virtio pmem device\n");
242 err = -EIO;
243 } else {
244 /* A host response results in "host_ack" getting called */
245 wait_event(req_data->host_acked,
246 virtio_pmem_req_done(req_data) ||
247 READ_ONCE(vpmem->broken));
248 if (virtio_pmem_req_done(req_data))
249 err = le32_to_cpu(req_data->resp.ret);
250 else
251 err = -EIO;
252 }
253
254 kref_put(&req_data->kref, virtio_pmem_req_release);
255 return err;
256 };
257
virtio_pmem_flush_work(struct work_struct * work)258 static void virtio_pmem_flush_work(struct work_struct *work)
259 {
260 struct virtio_pmem_flush_work *flush;
261 int err;
262
263 flush = container_of(work, struct virtio_pmem_flush_work, work);
264 err = virtio_pmem_flush(flush->nd_region);
265 if (err > 0)
266 err = -EIO;
267 if (err)
268 flush->bio->bi_status = errno_to_blk_status(err);
269 bio_endio(flush->bio);
270 kfree(flush);
271 }
272
273 /* The asynchronous flush callback function */
async_pmem_flush(struct nd_region * nd_region,struct bio * bio)274 int async_pmem_flush(struct nd_region *nd_region, struct bio *bio)
275 {
276 struct virtio_device *vdev = nd_region->provider_data;
277 struct virtio_pmem *vpmem = vdev->priv;
278 struct virtio_pmem_flush_work *flush;
279 unsigned long flags;
280 int err;
281
282 if (bio && bio->bi_iter.bi_sector != -1) {
283 flush = kmalloc_obj(*flush, GFP_NOIO);
284 if (!flush)
285 return -ENOMEM;
286
287 INIT_WORK(&flush->work, virtio_pmem_flush_work);
288 flush->nd_region = nd_region;
289 flush->bio = bio;
290
291 spin_lock_irqsave(&vpmem->pmem_lock, flags);
292 if (READ_ONCE(vpmem->broken)) {
293 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
294 kfree(flush);
295 return -EIO;
296 }
297 queue_work(vpmem->flush_wq, &flush->work);
298 spin_unlock_irqrestore(&vpmem->pmem_lock, flags);
299 return NVDIMM_FLUSH_ASYNC;
300 }
301
302 err = virtio_pmem_flush(nd_region);
303 if (err > 0)
304 return -EIO;
305
306 return err;
307 };
308 EXPORT_SYMBOL_GPL(async_pmem_flush);
309 MODULE_DESCRIPTION("Virtio Persistent Memory Driver");
310 MODULE_LICENSE("GPL");
311