1 /******************************************************************************
2 *
3 * Back-end of the driver for virtual block devices. This portion of the
4 * driver exports a 'unified' block-device interface that can be accessed
5 * by any operating system that implements a compatible front end. A
6 * reference front-end implementation can be found in:
7 * drivers/block/xen-blkfront.c
8 *
9 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
10 * Copyright (c) 2005, Christopher Clark
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
36
37 #define pr_fmt(fmt) "xen-blkback: " fmt
38
39 #include <linux/spinlock.h>
40 #include <linux/kthread.h>
41 #include <linux/list.h>
42 #include <linux/delay.h>
43 #include <linux/freezer.h>
44 #include <linux/bitmap.h>
45
46 #include <xen/events.h>
47 #include <xen/page.h>
48 #include <xen/xen.h>
49 #include <asm/xen/hypervisor.h>
50 #include <asm/xen/hypercall.h>
51 #include <xen/balloon.h>
52 #include <xen/grant_table.h>
53 #include "common.h"
54
55 /*
56 * Maximum number of unused free pages to keep in the internal buffer.
57 * Setting this to a value too low will reduce memory used in each backend,
58 * but can have a performance penalty.
59 *
60 * A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can
61 * be set to a lower value that might degrade performance on some intensive
62 * IO workloads.
63 */
64
65 static int max_buffer_pages = 1024;
66 module_param_named(max_buffer_pages, max_buffer_pages, int, 0644);
67 MODULE_PARM_DESC(max_buffer_pages,
68 "Maximum number of free pages to keep in each block backend buffer");
69
70 /*
71 * Maximum number of grants to map persistently in blkback. For maximum
72 * performance this should be the total numbers of grants that can be used
73 * to fill the ring, but since this might become too high, specially with
74 * the use of indirect descriptors, we set it to a value that provides good
75 * performance without using too much memory.
76 *
77 * When the list of persistent grants is full we clean it up using a LRU
78 * algorithm.
79 */
80
81 static int max_pgrants = 1056;
82 module_param_named(max_persistent_grants, max_pgrants, int, 0644);
83 MODULE_PARM_DESC(max_persistent_grants,
84 "Maximum number of grants to map persistently");
85
86 /*
87 * How long a persistent grant is allowed to remain allocated without being in
88 * use. The time is in seconds, 0 means indefinitely long.
89 */
90
91 static unsigned int pgrant_timeout = 60;
92 module_param_named(persistent_grant_unused_seconds, pgrant_timeout,
93 uint, 0644);
94 MODULE_PARM_DESC(persistent_grant_unused_seconds,
95 "Time in seconds an unused persistent grant is allowed to "
96 "remain allocated. Default is 60, 0 means unlimited.");
97
98 /*
99 * Maximum number of rings/queues blkback supports, allow as many queues as there
100 * are CPUs if user has not specified a value.
101 */
102 unsigned int xenblk_max_queues;
103 module_param_named(max_queues, xenblk_max_queues, uint, 0644);
104 MODULE_PARM_DESC(max_queues,
105 "Maximum number of hardware queues per virtual disk." \
106 "By default it is the number of online CPUs.");
107
108 /*
109 * Maximum order of pages to be used for the shared ring between front and
110 * backend, 4KB page granularity is used.
111 */
112 unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
113 module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
114 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
115 /*
116 * The LRU mechanism to clean the lists of persistent grants needs to
117 * be executed periodically. The time interval between consecutive executions
118 * of the purge mechanism is set in ms.
119 */
120 #define LRU_INTERVAL 100
121
122 /*
123 * When the persistent grants list is full we will remove unused grants
124 * from the list. The percent number of grants to be removed at each LRU
125 * execution.
126 */
127 #define LRU_PERCENT_CLEAN 5
128
129 /* Run-time switchable: /sys/module/blkback/parameters/ */
130 static unsigned int log_stats;
131 module_param(log_stats, int, 0644);
132
133 #define BLKBACK_INVALID_HANDLE (~0)
134
persistent_gnt_timeout(struct persistent_gnt * persistent_gnt)135 static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt)
136 {
137 return pgrant_timeout && (jiffies - persistent_gnt->last_used >=
138 HZ * pgrant_timeout);
139 }
140
141 #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
142
143 static int do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags);
144 static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
145 struct blkif_request *req,
146 struct pending_req *pending_req);
147 static void make_response(struct xen_blkif_ring *ring, u64 id,
148 unsigned short op, int st);
149
150 #define foreach_grant_safe(pos, n, rbtree, node) \
151 for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
152 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
153 &(pos)->node != NULL; \
154 (pos) = container_of(n, typeof(*(pos)), node), \
155 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
156
157
158 /*
159 * We don't need locking around the persistent grant helpers
160 * because blkback uses a single-thread for each backend, so we
161 * can be sure that this functions will never be called recursively.
162 *
163 * The only exception to that is put_persistent_grant, that can be called
164 * from interrupt context (by xen_blkbk_unmap), so we have to use atomic
165 * bit operations to modify the flags of a persistent grant and to count
166 * the number of used grants.
167 */
add_persistent_gnt(struct xen_blkif_ring * ring,struct persistent_gnt * persistent_gnt)168 static int add_persistent_gnt(struct xen_blkif_ring *ring,
169 struct persistent_gnt *persistent_gnt)
170 {
171 struct rb_node **new = NULL, *parent = NULL;
172 struct persistent_gnt *this;
173 struct xen_blkif *blkif = ring->blkif;
174
175 if (ring->persistent_gnt_c >= max_pgrants) {
176 if (!blkif->vbd.overflow_max_grants)
177 blkif->vbd.overflow_max_grants = 1;
178 return -EBUSY;
179 }
180 /* Figure out where to put new node */
181 new = &ring->persistent_gnts.rb_node;
182 while (*new) {
183 this = container_of(*new, struct persistent_gnt, node);
184
185 parent = *new;
186 if (persistent_gnt->gnt < this->gnt)
187 new = &((*new)->rb_left);
188 else if (persistent_gnt->gnt > this->gnt)
189 new = &((*new)->rb_right);
190 else {
191 pr_alert_ratelimited("trying to add a gref that's already in the tree\n");
192 return -EINVAL;
193 }
194 }
195
196 persistent_gnt->active = true;
197 /* Add new node and rebalance tree. */
198 rb_link_node(&(persistent_gnt->node), parent, new);
199 rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts);
200 ring->persistent_gnt_c++;
201 atomic_inc(&ring->persistent_gnt_in_use);
202 return 0;
203 }
204
get_persistent_gnt(struct xen_blkif_ring * ring,grant_ref_t gref)205 static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring,
206 grant_ref_t gref)
207 {
208 struct persistent_gnt *data;
209 struct rb_node *node = NULL;
210
211 node = ring->persistent_gnts.rb_node;
212 while (node) {
213 data = container_of(node, struct persistent_gnt, node);
214
215 if (gref < data->gnt)
216 node = node->rb_left;
217 else if (gref > data->gnt)
218 node = node->rb_right;
219 else {
220 if (data->active) {
221 pr_alert_ratelimited("requesting a grant already in use\n");
222 return NULL;
223 }
224 data->active = true;
225 atomic_inc(&ring->persistent_gnt_in_use);
226 return data;
227 }
228 }
229 return NULL;
230 }
231
put_persistent_gnt(struct xen_blkif_ring * ring,struct persistent_gnt * persistent_gnt)232 static void put_persistent_gnt(struct xen_blkif_ring *ring,
233 struct persistent_gnt *persistent_gnt)
234 {
235 if (!persistent_gnt->active)
236 pr_alert_ratelimited("freeing a grant already unused\n");
237 persistent_gnt->last_used = jiffies;
238 persistent_gnt->active = false;
239 atomic_dec(&ring->persistent_gnt_in_use);
240 }
241
free_persistent_gnts(struct xen_blkif_ring * ring)242 static void free_persistent_gnts(struct xen_blkif_ring *ring)
243 {
244 struct rb_root *root = &ring->persistent_gnts;
245 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
246 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
247 struct persistent_gnt *persistent_gnt;
248 struct rb_node *n;
249 int segs_to_unmap = 0;
250 struct gntab_unmap_queue_data unmap_data;
251
252 if (RB_EMPTY_ROOT(root))
253 return;
254
255 unmap_data.pages = pages;
256 unmap_data.unmap_ops = unmap;
257 unmap_data.kunmap_ops = NULL;
258
259 foreach_grant_safe(persistent_gnt, n, root, node) {
260 BUG_ON(persistent_gnt->handle ==
261 BLKBACK_INVALID_HANDLE);
262 gnttab_set_unmap_op(&unmap[segs_to_unmap],
263 (unsigned long) pfn_to_kaddr(page_to_pfn(
264 persistent_gnt->page)),
265 GNTMAP_host_map,
266 persistent_gnt->handle);
267
268 pages[segs_to_unmap] = persistent_gnt->page;
269
270 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
271 !rb_next(&persistent_gnt->node)) {
272
273 unmap_data.count = segs_to_unmap;
274 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
275
276 gnttab_page_cache_put(&ring->free_pages, pages,
277 segs_to_unmap);
278 segs_to_unmap = 0;
279 }
280
281 rb_erase(&persistent_gnt->node, root);
282 kfree(persistent_gnt);
283 ring->persistent_gnt_c--;
284 }
285
286 BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
287 BUG_ON(ring->persistent_gnt_c != 0);
288 }
289
xen_blkbk_unmap_purged_grants(struct work_struct * work)290 void xen_blkbk_unmap_purged_grants(struct work_struct *work)
291 {
292 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
293 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
294 struct persistent_gnt *persistent_gnt;
295 int segs_to_unmap = 0;
296 struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work);
297 struct gntab_unmap_queue_data unmap_data;
298
299 unmap_data.pages = pages;
300 unmap_data.unmap_ops = unmap;
301 unmap_data.kunmap_ops = NULL;
302
303 while(!list_empty(&ring->persistent_purge_list)) {
304 persistent_gnt = list_first_entry(&ring->persistent_purge_list,
305 struct persistent_gnt,
306 remove_node);
307 list_del(&persistent_gnt->remove_node);
308
309 gnttab_set_unmap_op(&unmap[segs_to_unmap],
310 vaddr(persistent_gnt->page),
311 GNTMAP_host_map,
312 persistent_gnt->handle);
313
314 pages[segs_to_unmap] = persistent_gnt->page;
315
316 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
317 unmap_data.count = segs_to_unmap;
318 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
319 gnttab_page_cache_put(&ring->free_pages, pages,
320 segs_to_unmap);
321 segs_to_unmap = 0;
322 }
323 kfree(persistent_gnt);
324 }
325 if (segs_to_unmap > 0) {
326 unmap_data.count = segs_to_unmap;
327 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
328 gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap);
329 }
330 }
331
purge_persistent_gnt(struct xen_blkif_ring * ring)332 static void purge_persistent_gnt(struct xen_blkif_ring *ring)
333 {
334 struct persistent_gnt *persistent_gnt;
335 struct rb_node *n;
336 unsigned int num_clean, total;
337 bool scan_used = false;
338 struct rb_root *root;
339
340 if (work_busy(&ring->persistent_purge_work)) {
341 pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n");
342 goto out;
343 }
344
345 if (ring->persistent_gnt_c < max_pgrants ||
346 (ring->persistent_gnt_c == max_pgrants &&
347 !ring->blkif->vbd.overflow_max_grants)) {
348 num_clean = 0;
349 } else {
350 num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN;
351 num_clean = ring->persistent_gnt_c - max_pgrants + num_clean;
352 num_clean = min(ring->persistent_gnt_c, num_clean);
353 pr_debug("Going to purge at least %u persistent grants\n",
354 num_clean);
355 }
356
357 /*
358 * At this point, we can assure that there will be no calls
359 * to get_persistent_grant (because we are executing this code from
360 * xen_blkif_schedule), there can only be calls to put_persistent_gnt,
361 * which means that the number of currently used grants will go down,
362 * but never up, so we will always be able to remove the requested
363 * number of grants.
364 */
365
366 total = 0;
367
368 BUG_ON(!list_empty(&ring->persistent_purge_list));
369 root = &ring->persistent_gnts;
370 purge_list:
371 foreach_grant_safe(persistent_gnt, n, root, node) {
372 BUG_ON(persistent_gnt->handle ==
373 BLKBACK_INVALID_HANDLE);
374
375 if (persistent_gnt->active)
376 continue;
377 if (!scan_used && !persistent_gnt_timeout(persistent_gnt))
378 continue;
379 if (scan_used && total >= num_clean)
380 continue;
381
382 rb_erase(&persistent_gnt->node, root);
383 list_add(&persistent_gnt->remove_node,
384 &ring->persistent_purge_list);
385 total++;
386 }
387 /*
388 * Check whether we also need to start cleaning
389 * grants that were used since last purge in order to cope
390 * with the requested num
391 */
392 if (!scan_used && total < num_clean) {
393 pr_debug("Still missing %u purged frames\n", num_clean - total);
394 scan_used = true;
395 goto purge_list;
396 }
397
398 if (total) {
399 ring->persistent_gnt_c -= total;
400 ring->blkif->vbd.overflow_max_grants = 0;
401
402 /* We can defer this work */
403 schedule_work(&ring->persistent_purge_work);
404 pr_debug("Purged %u/%u\n", num_clean, total);
405 }
406
407 out:
408 return;
409 }
410
411 /*
412 * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
413 */
alloc_req(struct xen_blkif_ring * ring)414 static struct pending_req *alloc_req(struct xen_blkif_ring *ring)
415 {
416 struct pending_req *req = NULL;
417 unsigned long flags;
418
419 spin_lock_irqsave(&ring->pending_free_lock, flags);
420 if (!list_empty(&ring->pending_free)) {
421 req = list_entry(ring->pending_free.next, struct pending_req,
422 free_list);
423 list_del(&req->free_list);
424 }
425 spin_unlock_irqrestore(&ring->pending_free_lock, flags);
426 return req;
427 }
428
429 /*
430 * Return the 'pending_req' structure back to the freepool. We also
431 * wake up the thread if it was waiting for a free page.
432 */
free_req(struct xen_blkif_ring * ring,struct pending_req * req)433 static void free_req(struct xen_blkif_ring *ring, struct pending_req *req)
434 {
435 unsigned long flags;
436 int was_empty;
437
438 spin_lock_irqsave(&ring->pending_free_lock, flags);
439 was_empty = list_empty(&ring->pending_free);
440 list_add(&req->free_list, &ring->pending_free);
441 spin_unlock_irqrestore(&ring->pending_free_lock, flags);
442 if (was_empty)
443 wake_up(&ring->pending_free_wq);
444 }
445
446 /*
447 * Routines for managing virtual block devices (vbds).
448 */
xen_vbd_translate(struct phys_req * req,struct xen_blkif * blkif,enum req_op operation)449 static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
450 enum req_op operation)
451 {
452 struct xen_vbd *vbd = &blkif->vbd;
453 int rc = -EACCES;
454
455 if ((operation != REQ_OP_READ) && vbd->readonly)
456 goto out;
457
458 if (likely(req->nr_sects)) {
459 blkif_sector_t end = req->sector_number + req->nr_sects;
460
461 if (unlikely(end < req->sector_number))
462 goto out;
463 if (unlikely(end > vbd_sz(vbd)))
464 goto out;
465 }
466
467 req->dev = vbd->pdevice;
468 req->bdev = file_bdev(vbd->bdev_file);
469 rc = 0;
470
471 out:
472 return rc;
473 }
474
xen_vbd_resize(struct xen_blkif * blkif)475 static void xen_vbd_resize(struct xen_blkif *blkif)
476 {
477 struct xen_vbd *vbd = &blkif->vbd;
478 struct xenbus_transaction xbt;
479 int err;
480 struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
481 unsigned long long new_size = vbd_sz(vbd);
482
483 pr_info("VBD Resize: Domid: %d, Device: (%d, %d)\n",
484 blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
485 pr_info("VBD Resize: new size %llu\n", new_size);
486 vbd->size = new_size;
487 again:
488 err = xenbus_transaction_start(&xbt);
489 if (err) {
490 pr_warn("Error starting transaction\n");
491 return;
492 }
493 err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
494 (unsigned long long)vbd_sz(vbd));
495 if (err) {
496 pr_warn("Error writing new size\n");
497 goto abort;
498 }
499 /*
500 * Write the current state; we will use this to synchronize
501 * the front-end. If the current state is "connected" the
502 * front-end will get the new size information online.
503 */
504 err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
505 if (err) {
506 pr_warn("Error writing the state\n");
507 goto abort;
508 }
509
510 err = xenbus_transaction_end(xbt, 0);
511 if (err == -EAGAIN)
512 goto again;
513 if (err)
514 pr_warn("Error ending transaction\n");
515 return;
516 abort:
517 xenbus_transaction_end(xbt, 1);
518 }
519
520 /*
521 * Notification from the guest OS.
522 */
blkif_notify_work(struct xen_blkif_ring * ring)523 static void blkif_notify_work(struct xen_blkif_ring *ring)
524 {
525 ring->waiting_reqs = 1;
526 wake_up(&ring->wq);
527 }
528
xen_blkif_be_int(int irq,void * dev_id)529 irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
530 {
531 blkif_notify_work(dev_id);
532 return IRQ_HANDLED;
533 }
534
535 /*
536 * SCHEDULER FUNCTIONS
537 */
538
print_stats(struct xen_blkif_ring * ring)539 static void print_stats(struct xen_blkif_ring *ring)
540 {
541 pr_info("(%s): oo %3llu | rd %4llu | wr %4llu | f %4llu"
542 " | ds %4llu | pg: %4u/%4d\n",
543 current->comm, ring->st_oo_req,
544 ring->st_rd_req, ring->st_wr_req,
545 ring->st_f_req, ring->st_ds_req,
546 ring->persistent_gnt_c, max_pgrants);
547 ring->st_print = jiffies + secs_to_jiffies(10);
548 ring->st_rd_req = 0;
549 ring->st_wr_req = 0;
550 ring->st_oo_req = 0;
551 ring->st_ds_req = 0;
552 }
553
xen_blkif_schedule(void * arg)554 int xen_blkif_schedule(void *arg)
555 {
556 struct xen_blkif_ring *ring = arg;
557 struct xen_blkif *blkif = ring->blkif;
558 struct xen_vbd *vbd = &blkif->vbd;
559 unsigned long timeout;
560 int ret;
561 bool do_eoi;
562 unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS;
563
564 set_freezable();
565 while (!kthread_should_stop()) {
566 if (try_to_freeze())
567 continue;
568 if (unlikely(vbd->size != vbd_sz(vbd)))
569 xen_vbd_resize(blkif);
570
571 timeout = msecs_to_jiffies(LRU_INTERVAL);
572
573 timeout = wait_event_interruptible_timeout(
574 ring->wq,
575 ring->waiting_reqs || kthread_should_stop(),
576 timeout);
577 if (timeout == 0)
578 goto purge_gnt_list;
579 timeout = wait_event_interruptible_timeout(
580 ring->pending_free_wq,
581 !list_empty(&ring->pending_free) ||
582 kthread_should_stop(),
583 timeout);
584 if (timeout == 0)
585 goto purge_gnt_list;
586
587 do_eoi = ring->waiting_reqs;
588
589 ring->waiting_reqs = 0;
590 smp_mb(); /* clear flag *before* checking for work */
591
592 ret = do_block_io_op(ring, &eoi_flags);
593 if (ret > 0)
594 ring->waiting_reqs = 1;
595 if (ret == -EACCES)
596 wait_event_interruptible(ring->shutdown_wq,
597 kthread_should_stop());
598
599 if (do_eoi && !ring->waiting_reqs) {
600 xen_irq_lateeoi(ring->irq, eoi_flags);
601 eoi_flags |= XEN_EOI_FLAG_SPURIOUS;
602 }
603
604 purge_gnt_list:
605 if (blkif->vbd.feature_gnt_persistent &&
606 time_after(jiffies, ring->next_lru)) {
607 purge_persistent_gnt(ring);
608 ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
609 }
610
611 /* Shrink the free pages pool if it is too large. */
612 if (time_before(jiffies, blkif->buffer_squeeze_end))
613 gnttab_page_cache_shrink(&ring->free_pages, 0);
614 else
615 gnttab_page_cache_shrink(&ring->free_pages,
616 max_buffer_pages);
617
618 if (log_stats && time_after(jiffies, ring->st_print))
619 print_stats(ring);
620 }
621
622 /* Drain pending purge work */
623 flush_work(&ring->persistent_purge_work);
624
625 if (log_stats)
626 print_stats(ring);
627
628 ring->xenblkd = NULL;
629
630 return 0;
631 }
632
633 /*
634 * Remove persistent grants and empty the pool of free pages
635 */
xen_blkbk_free_caches(struct xen_blkif_ring * ring)636 void xen_blkbk_free_caches(struct xen_blkif_ring *ring)
637 {
638 /* Free all persistent grant pages */
639 free_persistent_gnts(ring);
640
641 /* Since we are shutting down remove all pages from the buffer */
642 gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */);
643 }
644
xen_blkbk_unmap_prepare(struct xen_blkif_ring * ring,struct grant_page ** pages,unsigned int num,struct gnttab_unmap_grant_ref * unmap_ops,struct page ** unmap_pages)645 static unsigned int xen_blkbk_unmap_prepare(
646 struct xen_blkif_ring *ring,
647 struct grant_page **pages,
648 unsigned int num,
649 struct gnttab_unmap_grant_ref *unmap_ops,
650 struct page **unmap_pages)
651 {
652 unsigned int i, invcount = 0;
653
654 for (i = 0; i < num; i++) {
655 if (pages[i]->persistent_gnt != NULL) {
656 put_persistent_gnt(ring, pages[i]->persistent_gnt);
657 continue;
658 }
659 if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
660 continue;
661 unmap_pages[invcount] = pages[i]->page;
662 gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
663 GNTMAP_host_map, pages[i]->handle);
664 pages[i]->handle = BLKBACK_INVALID_HANDLE;
665 invcount++;
666 }
667
668 return invcount;
669 }
670
xen_blkbk_unmap_and_respond_callback(int result,struct gntab_unmap_queue_data * data)671 static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
672 {
673 struct pending_req *pending_req = (struct pending_req *)(data->data);
674 struct xen_blkif_ring *ring = pending_req->ring;
675 struct xen_blkif *blkif = ring->blkif;
676
677 /* BUG_ON used to reproduce existing behaviour,
678 but is this the best way to deal with this? */
679 BUG_ON(result);
680
681 gnttab_page_cache_put(&ring->free_pages, data->pages, data->count);
682 make_response(ring, pending_req->id,
683 pending_req->operation, pending_req->status);
684 free_req(ring, pending_req);
685 /*
686 * Make sure the request is freed before releasing blkif,
687 * or there could be a race between free_req and the
688 * cleanup done in xen_blkif_free during shutdown.
689 *
690 * NB: The fact that we might try to wake up pending_free_wq
691 * before drain_complete (in case there's a drain going on)
692 * it's not a problem with our current implementation
693 * because we can assure there's no thread waiting on
694 * pending_free_wq if there's a drain going on, but it has
695 * to be taken into account if the current model is changed.
696 */
697 if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) {
698 complete(&blkif->drain_complete);
699 }
700 xen_blkif_put(blkif);
701 }
702
xen_blkbk_unmap_and_respond(struct pending_req * req)703 static void xen_blkbk_unmap_and_respond(struct pending_req *req)
704 {
705 struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
706 struct xen_blkif_ring *ring = req->ring;
707 struct grant_page **pages = req->segments;
708 unsigned int invcount;
709
710 invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs,
711 req->unmap, req->unmap_pages);
712
713 work->data = req;
714 work->done = xen_blkbk_unmap_and_respond_callback;
715 work->unmap_ops = req->unmap;
716 work->kunmap_ops = NULL;
717 work->pages = req->unmap_pages;
718 work->count = invcount;
719
720 gnttab_unmap_refs_async(&req->gnttab_unmap_data);
721 }
722
723
724 /*
725 * Unmap the grant references.
726 *
727 * This could accumulate ops up to the batch size to reduce the number
728 * of hypercalls, but since this is only used in error paths there's
729 * no real need.
730 */
xen_blkbk_unmap(struct xen_blkif_ring * ring,struct grant_page * pages[],int num)731 static void xen_blkbk_unmap(struct xen_blkif_ring *ring,
732 struct grant_page *pages[],
733 int num)
734 {
735 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
736 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
737 unsigned int invcount = 0;
738 int ret;
739
740 while (num) {
741 unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
742
743 invcount = xen_blkbk_unmap_prepare(ring, pages, batch,
744 unmap, unmap_pages);
745 if (invcount) {
746 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
747 BUG_ON(ret);
748 gnttab_page_cache_put(&ring->free_pages, unmap_pages,
749 invcount);
750 }
751 pages += batch;
752 num -= batch;
753 }
754 }
755
xen_blkbk_map(struct xen_blkif_ring * ring,struct grant_page * pages[],int num,bool ro)756 static int xen_blkbk_map(struct xen_blkif_ring *ring,
757 struct grant_page *pages[],
758 int num, bool ro)
759 {
760 struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
761 struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
762 struct persistent_gnt *persistent_gnt = NULL;
763 phys_addr_t addr = 0;
764 int i, seg_idx, new_map_idx;
765 int segs_to_map = 0;
766 int ret = 0;
767 int last_map = 0, map_until = 0;
768 int use_persistent_gnts;
769 struct xen_blkif *blkif = ring->blkif;
770
771 use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
772
773 /*
774 * Fill out preq.nr_sects with proper amount of sectors, and setup
775 * assign map[..] with the PFN of the page in our domain with the
776 * corresponding grant reference for each page.
777 */
778 again:
779 for (i = map_until; i < num; i++) {
780 uint32_t flags;
781
782 if (use_persistent_gnts) {
783 persistent_gnt = get_persistent_gnt(
784 ring,
785 pages[i]->gref);
786 }
787
788 if (persistent_gnt) {
789 /*
790 * We are using persistent grants and
791 * the grant is already mapped
792 */
793 pages[i]->page = persistent_gnt->page;
794 pages[i]->persistent_gnt = persistent_gnt;
795 } else {
796 if (gnttab_page_cache_get(&ring->free_pages,
797 &pages[i]->page)) {
798 gnttab_page_cache_put(&ring->free_pages,
799 pages_to_gnt,
800 segs_to_map);
801 ret = -ENOMEM;
802 goto out;
803 }
804 addr = vaddr(pages[i]->page);
805 pages_to_gnt[segs_to_map] = pages[i]->page;
806 pages[i]->persistent_gnt = NULL;
807 flags = GNTMAP_host_map;
808 if (!use_persistent_gnts && ro)
809 flags |= GNTMAP_readonly;
810 gnttab_set_map_op(&map[segs_to_map++], addr,
811 flags, pages[i]->gref,
812 blkif->domid);
813 }
814 map_until = i + 1;
815 if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST)
816 break;
817 }
818
819 if (segs_to_map)
820 ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
821
822 /*
823 * Now swizzle the MFN in our domain with the MFN from the other domain
824 * so that when we access vaddr(pending_req,i) it has the contents of
825 * the page from the other domain.
826 */
827 for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) {
828 if (!pages[seg_idx]->persistent_gnt) {
829 /* This is a newly mapped grant */
830 BUG_ON(new_map_idx >= segs_to_map);
831 if (unlikely(map[new_map_idx].status != 0)) {
832 pr_debug("invalid buffer -- could not remap it\n");
833 gnttab_page_cache_put(&ring->free_pages,
834 &pages[seg_idx]->page, 1);
835 pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
836 ret |= !ret;
837 goto next;
838 }
839 pages[seg_idx]->handle = map[new_map_idx].handle;
840 } else {
841 continue;
842 }
843 if (use_persistent_gnts &&
844 ring->persistent_gnt_c < max_pgrants) {
845 /*
846 * We are using persistent grants, the grant is
847 * not mapped but we might have room for it.
848 */
849 persistent_gnt = kmalloc_obj(struct persistent_gnt);
850 if (!persistent_gnt) {
851 /*
852 * If we don't have enough memory to
853 * allocate the persistent_gnt struct
854 * map this grant non-persistenly
855 */
856 goto next;
857 }
858 persistent_gnt->gnt = map[new_map_idx].ref;
859 persistent_gnt->handle = map[new_map_idx].handle;
860 persistent_gnt->page = pages[seg_idx]->page;
861 if (add_persistent_gnt(ring,
862 persistent_gnt)) {
863 kfree(persistent_gnt);
864 persistent_gnt = NULL;
865 goto next;
866 }
867 pages[seg_idx]->persistent_gnt = persistent_gnt;
868 pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n",
869 persistent_gnt->gnt, ring->persistent_gnt_c,
870 max_pgrants);
871 goto next;
872 }
873 if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) {
874 blkif->vbd.overflow_max_grants = 1;
875 pr_debug("domain %u, device %#x is using maximum number of persistent grants\n",
876 blkif->domid, blkif->vbd.handle);
877 }
878 /*
879 * We could not map this grant persistently, so use it as
880 * a non-persistent grant.
881 */
882 next:
883 new_map_idx++;
884 }
885 segs_to_map = 0;
886 last_map = map_until;
887 if (!ret && map_until != num)
888 goto again;
889
890 out:
891 for (i = last_map; i < num; i++) {
892 /* Don't zap current batch's valid persistent grants. */
893 if (i >= map_until)
894 pages[i]->persistent_gnt = NULL;
895 pages[i]->handle = BLKBACK_INVALID_HANDLE;
896 }
897
898 return ret;
899 }
900
xen_blkbk_map_seg(struct pending_req * pending_req)901 static int xen_blkbk_map_seg(struct pending_req *pending_req)
902 {
903 int rc;
904
905 rc = xen_blkbk_map(pending_req->ring, pending_req->segments,
906 pending_req->nr_segs,
907 (pending_req->operation != BLKIF_OP_READ));
908
909 return rc;
910 }
911
xen_blkbk_parse_indirect(struct blkif_request * req,struct pending_req * pending_req,struct seg_buf seg[],struct phys_req * preq)912 static int xen_blkbk_parse_indirect(struct blkif_request *req,
913 struct pending_req *pending_req,
914 struct seg_buf seg[],
915 struct phys_req *preq)
916 {
917 struct grant_page **pages = pending_req->indirect_pages;
918 struct xen_blkif_ring *ring = pending_req->ring;
919 int indirect_grefs, rc, n, nseg, i;
920 struct blkif_request_segment *segments = NULL;
921
922 nseg = pending_req->nr_segs;
923 indirect_grefs = INDIRECT_PAGES(nseg);
924 BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
925
926 for (i = 0; i < indirect_grefs; i++)
927 pages[i]->gref = req->u.indirect.indirect_grefs[i];
928
929 rc = xen_blkbk_map(ring, pages, indirect_grefs, true);
930 if (rc)
931 goto unmap;
932
933 for (n = 0; n < nseg; n++) {
934 uint8_t first_sect, last_sect;
935
936 if ((n % SEGS_PER_INDIRECT_FRAME) == 0) {
937 /* Map indirect segments */
938 if (segments)
939 kunmap_atomic(segments);
940 segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page);
941 }
942 i = n % SEGS_PER_INDIRECT_FRAME;
943
944 pending_req->segments[n]->gref = segments[i].gref;
945
946 first_sect = READ_ONCE(segments[i].first_sect);
947 last_sect = READ_ONCE(segments[i].last_sect);
948 if (last_sect >= (XEN_PAGE_SIZE >> 9) || last_sect < first_sect) {
949 rc = -EINVAL;
950 goto unmap;
951 }
952
953 seg[n].nsec = last_sect - first_sect + 1;
954 seg[n].offset = first_sect << 9;
955 preq->nr_sects += seg[n].nsec;
956 }
957
958 unmap:
959 if (segments)
960 kunmap_atomic(segments);
961 xen_blkbk_unmap(ring, pages, indirect_grefs);
962 return rc;
963 }
964
dispatch_discard_io(struct xen_blkif_ring * ring,struct blkif_request * req)965 static int dispatch_discard_io(struct xen_blkif_ring *ring,
966 struct blkif_request *req)
967 {
968 int err = 0;
969 int status = BLKIF_RSP_OKAY;
970 struct xen_blkif *blkif = ring->blkif;
971 struct block_device *bdev = file_bdev(blkif->vbd.bdev_file);
972 struct phys_req preq;
973
974 xen_blkif_get(blkif);
975
976 preq.sector_number = req->u.discard.sector_number;
977 preq.nr_sects = req->u.discard.nr_sectors;
978
979 err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE);
980 if (err) {
981 pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n",
982 preq.sector_number,
983 preq.sector_number + preq.nr_sects, blkif->vbd.pdevice);
984 goto fail_response;
985 }
986 ring->st_ds_req++;
987
988 if (blkif->vbd.discard_secure &&
989 (req->u.discard.flag & BLKIF_DISCARD_SECURE))
990 err = blkdev_issue_secure_erase(bdev,
991 req->u.discard.sector_number,
992 req->u.discard.nr_sectors, GFP_KERNEL);
993 else
994 err = blkdev_issue_discard(bdev, req->u.discard.sector_number,
995 req->u.discard.nr_sectors, GFP_KERNEL);
996
997 fail_response:
998 if (err == -EOPNOTSUPP) {
999 pr_debug("discard op failed, not supported\n");
1000 status = BLKIF_RSP_EOPNOTSUPP;
1001 } else if (err)
1002 status = BLKIF_RSP_ERROR;
1003
1004 make_response(ring, req->u.discard.id, req->operation, status);
1005 xen_blkif_put(blkif);
1006 return err;
1007 }
1008
dispatch_other_io(struct xen_blkif_ring * ring,struct blkif_request * req,struct pending_req * pending_req)1009 static int dispatch_other_io(struct xen_blkif_ring *ring,
1010 struct blkif_request *req,
1011 struct pending_req *pending_req)
1012 {
1013 free_req(ring, pending_req);
1014 make_response(ring, req->u.other.id, req->operation,
1015 BLKIF_RSP_EOPNOTSUPP);
1016 return -EIO;
1017 }
1018
xen_blk_drain_io(struct xen_blkif_ring * ring)1019 static void xen_blk_drain_io(struct xen_blkif_ring *ring)
1020 {
1021 struct xen_blkif *blkif = ring->blkif;
1022
1023 atomic_set(&blkif->drain, 1);
1024 /*
1025 * Publish drain before checking inflight. Otherwise,
1026 * xen_blkbk_unmap_and_respond_callback() can decrement inflight with
1027 * atomic_dec_and_test() and still see drain == 0 after this path saw
1028 * inflight > 0, missing the completion.
1029 */
1030 smp_mb();
1031 do {
1032 if (atomic_read(&ring->inflight) == 0)
1033 break;
1034 wait_for_completion_interruptible_timeout(
1035 &blkif->drain_complete, HZ);
1036
1037 if (!atomic_read(&blkif->drain))
1038 break;
1039 } while (!kthread_should_stop());
1040 atomic_set(&blkif->drain, 0);
1041 }
1042
__end_block_io_op(struct pending_req * pending_req,blk_status_t error)1043 static void __end_block_io_op(struct pending_req *pending_req,
1044 blk_status_t error)
1045 {
1046 /* An error fails the entire request. */
1047 if (pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE &&
1048 error == BLK_STS_NOTSUPP) {
1049 pr_debug("flush diskcache op failed, not supported\n");
1050 xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0);
1051 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
1052 } else if (pending_req->operation == BLKIF_OP_WRITE_BARRIER &&
1053 error == BLK_STS_NOTSUPP) {
1054 pr_debug("write barrier op failed, not supported\n");
1055 xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0);
1056 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
1057 } else if (error) {
1058 pr_debug("Buffer not up-to-date at end of operation,"
1059 " error=%d\n", error);
1060 pending_req->status = BLKIF_RSP_ERROR;
1061 }
1062
1063 /*
1064 * If all of the bio's have completed it is time to unmap
1065 * the grant references associated with 'request' and provide
1066 * the proper response on the ring.
1067 */
1068 if (atomic_dec_and_test(&pending_req->pendcnt))
1069 xen_blkbk_unmap_and_respond(pending_req);
1070 }
1071
1072 /*
1073 * bio callback.
1074 */
end_block_io_op(struct bio * bio)1075 static void end_block_io_op(struct bio *bio)
1076 {
1077 __end_block_io_op(bio->bi_private, bio->bi_status);
1078 bio_put(bio);
1079 }
1080
blkif_get_x86_32_req(struct blkif_request * dst,const struct blkif_x86_32_request * src)1081 static void blkif_get_x86_32_req(struct blkif_request *dst,
1082 const struct blkif_x86_32_request *src)
1083 {
1084 unsigned int i, n;
1085
1086 dst->operation = READ_ONCE(src->operation);
1087
1088 switch (dst->operation) {
1089 case BLKIF_OP_READ:
1090 case BLKIF_OP_WRITE:
1091 case BLKIF_OP_WRITE_BARRIER:
1092 case BLKIF_OP_FLUSH_DISKCACHE:
1093 dst->u.rw.nr_segments = READ_ONCE(src->u.rw.nr_segments);
1094 dst->u.rw.handle = src->u.rw.handle;
1095 dst->u.rw.id = src->u.rw.id;
1096 dst->u.rw.sector_number = src->u.rw.sector_number;
1097 n = min_t(unsigned int, BLKIF_MAX_SEGMENTS_PER_REQUEST,
1098 dst->u.rw.nr_segments);
1099 for (i = 0; i < n; i++)
1100 dst->u.rw.seg[i] = src->u.rw.seg[i];
1101 break;
1102
1103 case BLKIF_OP_DISCARD:
1104 dst->u.discard.flag = src->u.discard.flag;
1105 dst->u.discard.id = src->u.discard.id;
1106 dst->u.discard.sector_number = src->u.discard.sector_number;
1107 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
1108 break;
1109
1110 case BLKIF_OP_INDIRECT:
1111 dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
1112 dst->u.indirect.nr_segments =
1113 READ_ONCE(src->u.indirect.nr_segments);
1114 dst->u.indirect.handle = src->u.indirect.handle;
1115 dst->u.indirect.id = src->u.indirect.id;
1116 dst->u.indirect.sector_number = src->u.indirect.sector_number;
1117 n = min(MAX_INDIRECT_PAGES,
1118 INDIRECT_PAGES(dst->u.indirect.nr_segments));
1119 for (i = 0; i < n; i++)
1120 dst->u.indirect.indirect_grefs[i] =
1121 src->u.indirect.indirect_grefs[i];
1122 break;
1123
1124 default:
1125 /*
1126 * Don't know how to translate this op. Only get the
1127 * ID so failure can be reported to the frontend.
1128 */
1129 dst->u.other.id = src->u.other.id;
1130 break;
1131 }
1132 }
1133
blkif_get_x86_64_req(struct blkif_request * dst,const struct blkif_x86_64_request * src)1134 static void blkif_get_x86_64_req(struct blkif_request *dst,
1135 const struct blkif_x86_64_request *src)
1136 {
1137 unsigned int i, n;
1138
1139 dst->operation = READ_ONCE(src->operation);
1140
1141 switch (dst->operation) {
1142 case BLKIF_OP_READ:
1143 case BLKIF_OP_WRITE:
1144 case BLKIF_OP_WRITE_BARRIER:
1145 case BLKIF_OP_FLUSH_DISKCACHE:
1146 dst->u.rw.nr_segments = READ_ONCE(src->u.rw.nr_segments);
1147 dst->u.rw.handle = src->u.rw.handle;
1148 dst->u.rw.id = src->u.rw.id;
1149 dst->u.rw.sector_number = src->u.rw.sector_number;
1150 n = min_t(unsigned int, BLKIF_MAX_SEGMENTS_PER_REQUEST,
1151 dst->u.rw.nr_segments);
1152 for (i = 0; i < n; i++)
1153 dst->u.rw.seg[i] = src->u.rw.seg[i];
1154 break;
1155
1156 case BLKIF_OP_DISCARD:
1157 dst->u.discard.flag = src->u.discard.flag;
1158 dst->u.discard.id = src->u.discard.id;
1159 dst->u.discard.sector_number = src->u.discard.sector_number;
1160 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
1161 break;
1162
1163 case BLKIF_OP_INDIRECT:
1164 dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
1165 dst->u.indirect.nr_segments =
1166 READ_ONCE(src->u.indirect.nr_segments);
1167 dst->u.indirect.handle = src->u.indirect.handle;
1168 dst->u.indirect.id = src->u.indirect.id;
1169 dst->u.indirect.sector_number = src->u.indirect.sector_number;
1170 n = min(MAX_INDIRECT_PAGES,
1171 INDIRECT_PAGES(dst->u.indirect.nr_segments));
1172 for (i = 0; i < n; i++)
1173 dst->u.indirect.indirect_grefs[i] =
1174 src->u.indirect.indirect_grefs[i];
1175 break;
1176
1177 default:
1178 /*
1179 * Don't know how to translate this op. Only get the
1180 * ID so failure can be reported to the frontend.
1181 */
1182 dst->u.other.id = src->u.other.id;
1183 break;
1184 }
1185 }
1186
1187 /*
1188 * Function to copy the from the ring buffer the 'struct blkif_request'
1189 * (which has the sectors we want, number of them, grant references, etc),
1190 * and transmute it to the block API to hand it over to the proper block disk.
1191 */
1192 static int
__do_block_io_op(struct xen_blkif_ring * ring,unsigned int * eoi_flags)1193 __do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
1194 {
1195 union blkif_back_rings *blk_rings = &ring->blk_rings;
1196 struct blkif_request req;
1197 struct pending_req *pending_req;
1198 RING_IDX rc, rp;
1199 int more_to_do = 0;
1200
1201 rc = blk_rings->common.req_cons;
1202 rp = blk_rings->common.sring->req_prod;
1203 rmb(); /* Ensure we see queued requests up to 'rp'. */
1204
1205 if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) {
1206 rc = blk_rings->common.rsp_prod_pvt;
1207 pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n",
1208 rp, rc, rp - rc, ring->blkif->vbd.pdevice);
1209 return -EACCES;
1210 }
1211 while (rc != rp) {
1212
1213 if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
1214 break;
1215
1216 /* We've seen a request, so clear spurious eoi flag. */
1217 *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS;
1218
1219 if (kthread_should_stop()) {
1220 more_to_do = 1;
1221 break;
1222 }
1223
1224 pending_req = alloc_req(ring);
1225 if (NULL == pending_req) {
1226 ring->st_oo_req++;
1227 more_to_do = 1;
1228 break;
1229 }
1230
1231 switch (ring->blkif->blk_protocol) {
1232 case BLKIF_PROTOCOL_NATIVE:
1233 memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
1234 break;
1235 case BLKIF_PROTOCOL_X86_32:
1236 blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
1237 break;
1238 case BLKIF_PROTOCOL_X86_64:
1239 blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
1240 break;
1241 default:
1242 BUG();
1243 }
1244 blk_rings->common.req_cons = ++rc; /* before make_response() */
1245
1246 /* Apply all sanity checks to /private copy/ of request. */
1247 barrier();
1248
1249 switch (req.operation) {
1250 case BLKIF_OP_READ:
1251 case BLKIF_OP_WRITE:
1252 case BLKIF_OP_WRITE_BARRIER:
1253 case BLKIF_OP_FLUSH_DISKCACHE:
1254 case BLKIF_OP_INDIRECT:
1255 if (dispatch_rw_block_io(ring, &req, pending_req))
1256 goto done;
1257 break;
1258 case BLKIF_OP_DISCARD:
1259 free_req(ring, pending_req);
1260 if (dispatch_discard_io(ring, &req))
1261 goto done;
1262 break;
1263 default:
1264 if (dispatch_other_io(ring, &req, pending_req))
1265 goto done;
1266 break;
1267 }
1268
1269 /* Yield point for this unbounded loop. */
1270 cond_resched();
1271 }
1272 done:
1273 return more_to_do;
1274 }
1275
1276 static int
do_block_io_op(struct xen_blkif_ring * ring,unsigned int * eoi_flags)1277 do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
1278 {
1279 union blkif_back_rings *blk_rings = &ring->blk_rings;
1280 int more_to_do;
1281
1282 do {
1283 more_to_do = __do_block_io_op(ring, eoi_flags);
1284 if (more_to_do)
1285 break;
1286
1287 RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
1288 } while (more_to_do);
1289
1290 return more_to_do;
1291 }
1292 /*
1293 * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
1294 * and call the 'submit_bio' to pass it to the underlying storage.
1295 */
dispatch_rw_block_io(struct xen_blkif_ring * ring,struct blkif_request * req,struct pending_req * pending_req)1296 static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
1297 struct blkif_request *req,
1298 struct pending_req *pending_req)
1299 {
1300 struct phys_req preq;
1301 struct seg_buf *seg = pending_req->seg;
1302 unsigned int nseg;
1303 struct bio *bio = NULL;
1304 struct bio **biolist = pending_req->biolist;
1305 int i, nbio = 0;
1306 enum req_op operation;
1307 blk_opf_t operation_flags = 0;
1308 struct blk_plug plug;
1309 bool drain = false;
1310 struct grant_page **pages = pending_req->segments;
1311 unsigned short req_operation;
1312
1313 req_operation = req->operation == BLKIF_OP_INDIRECT ?
1314 req->u.indirect.indirect_op : req->operation;
1315
1316 if ((req->operation == BLKIF_OP_INDIRECT) &&
1317 (req_operation != BLKIF_OP_READ) &&
1318 (req_operation != BLKIF_OP_WRITE)) {
1319 pr_debug("Invalid indirect operation (%u)\n", req_operation);
1320 goto fail_response;
1321 }
1322
1323 switch (req_operation) {
1324 case BLKIF_OP_READ:
1325 ring->st_rd_req++;
1326 operation = REQ_OP_READ;
1327 break;
1328 case BLKIF_OP_WRITE:
1329 ring->st_wr_req++;
1330 operation = REQ_OP_WRITE;
1331 operation_flags = REQ_SYNC | REQ_IDLE;
1332 break;
1333 case BLKIF_OP_WRITE_BARRIER:
1334 drain = true;
1335 fallthrough;
1336 case BLKIF_OP_FLUSH_DISKCACHE:
1337 ring->st_f_req++;
1338 operation = REQ_OP_WRITE;
1339 operation_flags = REQ_PREFLUSH;
1340 break;
1341 default:
1342 operation = 0; /* make gcc happy */
1343 goto fail_response;
1344 break;
1345 }
1346
1347 /* Check that the number of segments is sane. */
1348 nseg = req->operation == BLKIF_OP_INDIRECT ?
1349 req->u.indirect.nr_segments : req->u.rw.nr_segments;
1350
1351 if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) ||
1352 unlikely((req->operation != BLKIF_OP_INDIRECT) &&
1353 (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
1354 unlikely((req->operation == BLKIF_OP_INDIRECT) &&
1355 (nseg > MAX_INDIRECT_SEGMENTS))) {
1356 pr_debug("Bad number of segments in request (%d)\n", nseg);
1357 /* Haven't submitted any bio's yet. */
1358 goto fail_response;
1359 }
1360
1361 preq.nr_sects = 0;
1362
1363 pending_req->ring = ring;
1364 pending_req->id = req->u.rw.id;
1365 pending_req->operation = req_operation;
1366 pending_req->status = BLKIF_RSP_OKAY;
1367 pending_req->nr_segs = nseg;
1368
1369 if (req->operation != BLKIF_OP_INDIRECT) {
1370 preq.dev = req->u.rw.handle;
1371 preq.sector_number = req->u.rw.sector_number;
1372 for (i = 0; i < nseg; i++) {
1373 pages[i]->gref = req->u.rw.seg[i].gref;
1374 seg[i].nsec = req->u.rw.seg[i].last_sect -
1375 req->u.rw.seg[i].first_sect + 1;
1376 seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
1377 if ((req->u.rw.seg[i].last_sect >= (XEN_PAGE_SIZE >> 9)) ||
1378 (req->u.rw.seg[i].last_sect <
1379 req->u.rw.seg[i].first_sect))
1380 goto fail_response;
1381 preq.nr_sects += seg[i].nsec;
1382 }
1383 } else {
1384 preq.dev = req->u.indirect.handle;
1385 preq.sector_number = req->u.indirect.sector_number;
1386 if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq))
1387 goto fail_response;
1388 }
1389
1390 if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) {
1391 pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n",
1392 operation == REQ_OP_READ ? "read" : "write",
1393 preq.sector_number,
1394 preq.sector_number + preq.nr_sects,
1395 ring->blkif->vbd.pdevice);
1396 goto fail_response;
1397 }
1398
1399 /*
1400 * This check _MUST_ be done after xen_vbd_translate as the preq.bdev
1401 * is set there.
1402 */
1403 for (i = 0; i < nseg; i++) {
1404 if (((int)preq.sector_number|(int)seg[i].nsec) &
1405 ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
1406 pr_debug("Misaligned I/O request from domain %d\n",
1407 ring->blkif->domid);
1408 goto fail_response;
1409 }
1410 }
1411
1412 /* Wait on all outstanding I/O's and once that has been completed
1413 * issue the flush.
1414 */
1415 if (drain)
1416 xen_blk_drain_io(pending_req->ring);
1417
1418 /*
1419 * If we have failed at this point, we need to undo the M2P override,
1420 * set gnttab_set_unmap_op on all of the grant references and perform
1421 * the hypercall to unmap the grants - that is all done in
1422 * xen_blkbk_unmap.
1423 */
1424 if (xen_blkbk_map_seg(pending_req))
1425 goto fail_flush;
1426
1427 /*
1428 * This corresponding xen_blkif_put is done in __end_block_io_op, or
1429 * below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
1430 */
1431 xen_blkif_get(ring->blkif);
1432 atomic_inc(&ring->inflight);
1433
1434 for (i = 0; i < nseg; i++) {
1435 while ((bio == NULL) ||
1436 (bio_add_page(bio,
1437 pages[i]->page,
1438 seg[i].nsec << 9,
1439 seg[i].offset) == 0)) {
1440 bio = bio_alloc(preq.bdev, bio_max_segs(nseg - i),
1441 operation | operation_flags,
1442 GFP_KERNEL);
1443 biolist[nbio++] = bio;
1444 bio->bi_private = pending_req;
1445 bio->bi_end_io = end_block_io_op;
1446 bio->bi_iter.bi_sector = preq.sector_number;
1447 }
1448
1449 preq.sector_number += seg[i].nsec;
1450 }
1451
1452 /* This will be hit if the operation was a flush or discard. */
1453 if (!bio) {
1454 BUG_ON(operation_flags != REQ_PREFLUSH);
1455
1456 bio = bio_alloc(preq.bdev, 0, operation | operation_flags,
1457 GFP_KERNEL);
1458 biolist[nbio++] = bio;
1459 bio->bi_private = pending_req;
1460 bio->bi_end_io = end_block_io_op;
1461 }
1462
1463 atomic_set(&pending_req->pendcnt, nbio);
1464 blk_start_plug(&plug);
1465
1466 for (i = 0; i < nbio; i++)
1467 submit_bio(biolist[i]);
1468
1469 /* Let the I/Os go.. */
1470 blk_finish_plug(&plug);
1471
1472 if (operation == REQ_OP_READ)
1473 ring->st_rd_sect += preq.nr_sects;
1474 else if (operation == REQ_OP_WRITE)
1475 ring->st_wr_sect += preq.nr_sects;
1476
1477 return 0;
1478
1479 fail_flush:
1480 xen_blkbk_unmap(ring, pending_req->segments,
1481 pending_req->nr_segs);
1482 fail_response:
1483 /* Haven't submitted any bio's yet. */
1484 make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
1485 free_req(ring, pending_req);
1486 msleep(1); /* back off a bit */
1487 return -EIO;
1488 }
1489
1490
1491
1492 /*
1493 * Put a response on the ring on how the operation fared.
1494 */
make_response(struct xen_blkif_ring * ring,u64 id,unsigned short op,int st)1495 static void make_response(struct xen_blkif_ring *ring, u64 id,
1496 unsigned short op, int st)
1497 {
1498 struct blkif_response *resp;
1499 unsigned long flags;
1500 union blkif_back_rings *blk_rings;
1501 int notify;
1502
1503 spin_lock_irqsave(&ring->blk_ring_lock, flags);
1504 blk_rings = &ring->blk_rings;
1505 /* Place on the response ring for the relevant domain. */
1506 switch (ring->blkif->blk_protocol) {
1507 case BLKIF_PROTOCOL_NATIVE:
1508 resp = RING_GET_RESPONSE(&blk_rings->native,
1509 blk_rings->native.rsp_prod_pvt);
1510 break;
1511 case BLKIF_PROTOCOL_X86_32:
1512 resp = RING_GET_RESPONSE(&blk_rings->x86_32,
1513 blk_rings->x86_32.rsp_prod_pvt);
1514 break;
1515 case BLKIF_PROTOCOL_X86_64:
1516 resp = RING_GET_RESPONSE(&blk_rings->x86_64,
1517 blk_rings->x86_64.rsp_prod_pvt);
1518 break;
1519 default:
1520 BUG();
1521 }
1522
1523 resp->id = id;
1524 resp->operation = op;
1525 resp->status = st;
1526
1527 blk_rings->common.rsp_prod_pvt++;
1528 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
1529 spin_unlock_irqrestore(&ring->blk_ring_lock, flags);
1530 if (notify)
1531 notify_remote_via_irq(ring->irq);
1532 }
1533
xen_blkif_init(void)1534 static int __init xen_blkif_init(void)
1535 {
1536 int rc = 0;
1537
1538 if (!xen_domain())
1539 return -ENODEV;
1540
1541 if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
1542 pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
1543 xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
1544 xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
1545 }
1546
1547 if (xenblk_max_queues == 0)
1548 xenblk_max_queues = num_online_cpus();
1549
1550 rc = xen_blkif_interface_init();
1551 if (rc)
1552 goto failed_init;
1553
1554 rc = xen_blkif_xenbus_init();
1555 if (rc)
1556 goto failed_init;
1557
1558 failed_init:
1559 return rc;
1560 }
1561
1562 module_init(xen_blkif_init);
1563
xen_blkif_fini(void)1564 static void __exit xen_blkif_fini(void)
1565 {
1566 xen_blkif_xenbus_fini();
1567 xen_blkif_interface_fini();
1568 }
1569
1570 module_exit(xen_blkif_fini);
1571
1572 MODULE_DESCRIPTION("Virtual block device back-end driver");
1573 MODULE_LICENSE("Dual BSD/GPL");
1574 MODULE_ALIAS("xen-backend:vbd");
1575