1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * virtio_pmem.c: Virtio pmem Driver 4 * 5 * Discovers persistent memory range information 6 * from host and provides a virtio based flushing 7 * interface. 8 */ 9 #include "virtio_pmem.h" 10 #include "nd.h" 11 12 struct virtio_pmem_flush_work { 13 struct work_struct work; 14 struct nd_region *nd_region; 15 struct bio *bio; 16 }; 17 18 static void virtio_pmem_req_release(struct kref *kref) 19 { 20 struct virtio_pmem_request *req; 21 22 req = container_of(kref, struct virtio_pmem_request, kref); 23 kfree(req); 24 } 25 26 static void virtio_pmem_signal_done(struct virtio_pmem_request *req) 27 { 28 /* Pairs with smp_load_acquire() in virtio_pmem_req_done(). */ 29 smp_store_release(&req->done, true); 30 wake_up(&req->host_acked); 31 } 32 33 static bool virtio_pmem_req_done(struct virtio_pmem_request *req) 34 { 35 /* Pairs with smp_store_release() in virtio_pmem_signal_done(). */ 36 return smp_load_acquire(&req->done); 37 } 38 39 static void virtio_pmem_complete_err(struct virtio_pmem_request *req) 40 { 41 req->resp.ret = cpu_to_le32(1); 42 virtio_pmem_signal_done(req); 43 } 44 45 static void virtio_pmem_wake_one_waiter(struct virtio_pmem *vpmem) 46 { 47 struct virtio_pmem_request *req_buf; 48 49 if (list_empty(&vpmem->req_list)) 50 return; 51 52 req_buf = list_first_entry(&vpmem->req_list, 53 struct virtio_pmem_request, list); 54 list_del_init(&req_buf->list); 55 WRITE_ONCE(req_buf->wq_buf_avail, true); 56 wake_up(&req_buf->wq_buf); 57 } 58 59 static void virtio_pmem_wake_all_waiters(struct virtio_pmem *vpmem) 60 { 61 struct virtio_pmem_request *req, *tmp; 62 63 list_for_each_entry_safe(req, tmp, &vpmem->req_list, list) { 64 list_del_init(&req->list); 65 WRITE_ONCE(req->wq_buf_avail, true); 66 wake_up(&req->wq_buf); 67 } 68 } 69 70 static void virtio_pmem_clear_inflight(struct virtio_pmem *vpmem, 71 struct virtio_pmem_request *req) 72 { 73 if (vpmem->req_inflight == req) 74 vpmem->req_inflight = NULL; 75 } 76 77 static void virtio_pmem_wake_inflight(struct virtio_pmem *vpmem) 78 { 79 struct virtio_pmem_request *req = vpmem->req_inflight; 80 81 if (req) 82 wake_up(&req->host_acked); 83 } 84 85 void virtio_pmem_mark_broken(struct virtio_pmem *vpmem) 86 { 87 if (!READ_ONCE(vpmem->broken)) { 88 WRITE_ONCE(vpmem->broken, true); 89 dev_err_once(&vpmem->vdev->dev, "virtqueue is broken\n"); 90 } 91 92 virtio_pmem_wake_inflight(vpmem); 93 virtio_pmem_wake_all_waiters(vpmem); 94 } 95 EXPORT_SYMBOL_GPL(virtio_pmem_mark_broken); 96 97 void virtio_pmem_drain(struct virtio_pmem *vpmem) 98 { 99 struct virtio_pmem_request *req; 100 unsigned int len; 101 102 if (!vpmem->req_vq) 103 return; 104 105 while ((req = virtqueue_get_buf(vpmem->req_vq, &len)) != NULL) { 106 virtio_pmem_clear_inflight(vpmem, req); 107 virtio_pmem_complete_err(req); 108 kref_put(&req->kref, virtio_pmem_req_release); 109 } 110 111 while ((req = virtqueue_detach_unused_buf(vpmem->req_vq)) != NULL) { 112 virtio_pmem_clear_inflight(vpmem, req); 113 virtio_pmem_complete_err(req); 114 kref_put(&req->kref, virtio_pmem_req_release); 115 } 116 } 117 EXPORT_SYMBOL_GPL(virtio_pmem_drain); 118 119 /* The interrupt handler */ 120 void virtio_pmem_host_ack(struct virtqueue *vq) 121 { 122 struct virtio_pmem *vpmem = vq->vdev->priv; 123 struct virtio_pmem_request *req_data; 124 unsigned long flags; 125 unsigned int len; 126 127 spin_lock_irqsave(&vpmem->pmem_lock, flags); 128 while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) { 129 virtio_pmem_clear_inflight(vpmem, req_data); 130 virtio_pmem_wake_one_waiter(vpmem); 131 if (READ_ONCE(vpmem->broken)) 132 virtio_pmem_complete_err(req_data); 133 else 134 virtio_pmem_signal_done(req_data); 135 kref_put(&req_data->kref, virtio_pmem_req_release); 136 } 137 spin_unlock_irqrestore(&vpmem->pmem_lock, flags); 138 } 139 EXPORT_SYMBOL_GPL(virtio_pmem_host_ack); 140 141 /* The request submission function */ 142 static int virtio_pmem_flush(struct nd_region *nd_region) 143 { 144 struct virtio_device *vdev = nd_region->provider_data; 145 struct virtio_pmem *vpmem = vdev->priv; 146 struct virtio_pmem_request *req_data; 147 struct scatterlist *sgs[2], sg, ret; 148 unsigned long flags; 149 int err, err1; 150 151 guard(mutex)(&vpmem->flush_lock); 152 153 /* 154 * Don't bother to submit the request to the device if the device is 155 * not activated. 156 */ 157 if (vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_NEEDS_RESET) { 158 dev_info(&vdev->dev, "virtio pmem device needs a reset\n"); 159 return -EIO; 160 } 161 162 if (READ_ONCE(vpmem->broken)) 163 return -EIO; 164 165 req_data = kmalloc_obj(*req_data, GFP_NOIO); 166 if (!req_data) 167 return -ENOMEM; 168 169 kref_init(&req_data->kref); 170 WRITE_ONCE(req_data->done, false); 171 init_waitqueue_head(&req_data->host_acked); 172 init_waitqueue_head(&req_data->wq_buf); 173 INIT_LIST_HEAD(&req_data->list); 174 req_data->req.type = cpu_to_le32(VIRTIO_PMEM_REQ_TYPE_FLUSH); 175 sg_init_one(&sg, &req_data->req, sizeof(req_data->req)); 176 sgs[0] = &sg; 177 sg_init_one(&ret, &req_data->resp.ret, sizeof(req_data->resp)); 178 sgs[1] = &ret; 179 180 spin_lock_irqsave(&vpmem->pmem_lock, flags); 181 /* 182 * If virtqueue_add_sgs returns -ENOSPC then req_vq virtual 183 * queue does not have free descriptor. We add the request 184 * to req_list and wait for host_ack to wake us up when free 185 * slots are available. 186 */ 187 for (;;) { 188 if (READ_ONCE(vpmem->broken)) { 189 err = -EIO; 190 break; 191 } 192 193 err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, 194 GFP_ATOMIC); 195 if (!err) { 196 /* 197 * Take the virtqueue reference while @pmem_lock is 198 * held so completion cannot run concurrently. 199 */ 200 kref_get(&req_data->kref); 201 vpmem->req_inflight = req_data; 202 break; 203 } 204 205 if (err != -ENOSPC) 206 break; 207 208 dev_info_ratelimited(&vdev->dev, 209 "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); 210 WRITE_ONCE(req_data->wq_buf_avail, false); 211 list_add_tail(&req_data->list, &vpmem->req_list); 212 spin_unlock_irqrestore(&vpmem->pmem_lock, flags); 213 214 /* A host response results in "host_ack" getting called */ 215 wait_event(req_data->wq_buf, 216 READ_ONCE(req_data->wq_buf_avail) || 217 READ_ONCE(vpmem->broken)); 218 spin_lock_irqsave(&vpmem->pmem_lock, flags); 219 220 if (READ_ONCE(vpmem->broken)) 221 break; 222 } 223 224 if (READ_ONCE(vpmem->broken)) 225 err = -EIO; 226 if (err == -EIO || virtqueue_is_broken(vpmem->req_vq)) 227 virtio_pmem_mark_broken(vpmem); 228 229 err1 = true; 230 if (!err && !READ_ONCE(vpmem->broken)) { 231 err1 = virtqueue_kick(vpmem->req_vq); 232 if (!err1) 233 virtio_pmem_mark_broken(vpmem); 234 } 235 spin_unlock_irqrestore(&vpmem->pmem_lock, flags); 236 /* 237 * virtqueue_add_sgs failed with error different than -ENOSPC, we can't 238 * do anything about that. 239 */ 240 if (READ_ONCE(vpmem->broken) || err || !err1) { 241 dev_info(&vdev->dev, "failed to send command to virtio pmem device\n"); 242 err = -EIO; 243 } else { 244 /* A host response results in "host_ack" getting called */ 245 wait_event(req_data->host_acked, 246 virtio_pmem_req_done(req_data) || 247 READ_ONCE(vpmem->broken)); 248 if (virtio_pmem_req_done(req_data)) 249 err = le32_to_cpu(req_data->resp.ret); 250 else 251 err = -EIO; 252 } 253 254 kref_put(&req_data->kref, virtio_pmem_req_release); 255 return err; 256 }; 257 258 static void virtio_pmem_flush_work(struct work_struct *work) 259 { 260 struct virtio_pmem_flush_work *flush; 261 int err; 262 263 flush = container_of(work, struct virtio_pmem_flush_work, work); 264 err = virtio_pmem_flush(flush->nd_region); 265 if (err > 0) 266 err = -EIO; 267 if (err) 268 flush->bio->bi_status = errno_to_blk_status(err); 269 bio_endio(flush->bio); 270 kfree(flush); 271 } 272 273 /* The asynchronous flush callback function */ 274 int async_pmem_flush(struct nd_region *nd_region, struct bio *bio) 275 { 276 struct virtio_device *vdev = nd_region->provider_data; 277 struct virtio_pmem *vpmem = vdev->priv; 278 struct virtio_pmem_flush_work *flush; 279 unsigned long flags; 280 int err; 281 282 if (bio && bio->bi_iter.bi_sector != -1) { 283 flush = kmalloc_obj(*flush, GFP_NOIO); 284 if (!flush) 285 return -ENOMEM; 286 287 INIT_WORK(&flush->work, virtio_pmem_flush_work); 288 flush->nd_region = nd_region; 289 flush->bio = bio; 290 291 spin_lock_irqsave(&vpmem->pmem_lock, flags); 292 if (READ_ONCE(vpmem->broken)) { 293 spin_unlock_irqrestore(&vpmem->pmem_lock, flags); 294 kfree(flush); 295 return -EIO; 296 } 297 queue_work(vpmem->flush_wq, &flush->work); 298 spin_unlock_irqrestore(&vpmem->pmem_lock, flags); 299 return NVDIMM_FLUSH_ASYNC; 300 } 301 302 err = virtio_pmem_flush(nd_region); 303 if (err > 0) 304 return -EIO; 305 306 return err; 307 }; 308 EXPORT_SYMBOL_GPL(async_pmem_flush); 309 MODULE_DESCRIPTION("Virtio Persistent Memory Driver"); 310 MODULE_LICENSE("GPL"); 311