xref: /linux/drivers/misc/vmw_vmci/vmci_queue_pair.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VMware VMCI Driver
4  *
5  * Copyright (C) 2012 VMware, Inc. All rights reserved.
6  */
7 
8 #include <linux/vmw_vmci_defs.h>
9 #include <linux/vmw_vmci_api.h>
10 #include <linux/highmem.h>
11 #include <linux/kernel.h>
12 #include <linux/mm.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/pagemap.h>
16 #include <linux/pci.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/uio.h>
20 #include <linux/wait.h>
21 #include <linux/vmalloc.h>
22 #include <linux/skbuff.h>
23 
24 #include "vmci_handle_array.h"
25 #include "vmci_queue_pair.h"
26 #include "vmci_datagram.h"
27 #include "vmci_resource.h"
28 #include "vmci_context.h"
29 #include "vmci_driver.h"
30 #include "vmci_event.h"
31 #include "vmci_route.h"
32 
33 /*
34  * In the following, we will distinguish between two kinds of VMX processes -
35  * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36  * VMCI page files in the VMX and supporting VM to VM communication and the
37  * newer ones that use the guest memory directly. We will in the following
38  * refer to the older VMX versions as old-style VMX'en, and the newer ones as
39  * new-style VMX'en.
40  *
41  * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42  * removed for readability) - see below for more details on the transtions:
43  *
44  *            --------------  NEW  -------------
45  *            |                                |
46  *           \_/                              \_/
47  *     CREATED_NO_MEM <-----------------> CREATED_MEM
48  *            |    |                           |
49  *            |    o-----------------------o   |
50  *            |                            |   |
51  *           \_/                          \_/ \_/
52  *     ATTACHED_NO_MEM <----------------> ATTACHED_MEM
53  *            |                            |   |
54  *            |     o----------------------o   |
55  *            |     |                          |
56  *           \_/   \_/                        \_/
57  *     SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
58  *            |                                |
59  *            |                                |
60  *            -------------> gone <-------------
61  *
62  * In more detail. When a VMCI queue pair is first created, it will be in the
63  * VMCIQPB_NEW state. It will then move into one of the following states:
64  *
65  * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
66  *
67  *     - the created was performed by a host endpoint, in which case there is
68  *       no backing memory yet.
69  *
70  *     - the create was initiated by an old-style VMX, that uses
71  *       vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72  *       a later point in time. This state can be distinguished from the one
73  *       above by the context ID of the creator. A host side is not allowed to
74  *       attach until the page store has been set.
75  *
76  * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77  *     is created by a VMX using the queue pair device backend that
78  *     sets the UVAs of the queue pair immediately and stores the
79  *     information for later attachers. At this point, it is ready for
80  *     the host side to attach to it.
81  *
82  * Once the queue pair is in one of the created states (with the exception of
83  * the case mentioned for older VMX'en above), it is possible to attach to the
84  * queue pair. Again we have two new states possible:
85  *
86  * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
87  *   paths:
88  *
89  *     - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90  *       pair, and attaches to a queue pair previously created by the host side.
91  *
92  *     - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93  *       already created by a guest.
94  *
95  *     - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96  *       vmci_qp_broker_set_page_store (see below).
97  *
98  * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99  *     VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100  *     bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101  *     is called to register the user memory, the VMCIQPB_ATTACH_MEM state
102  *     will be entered.
103  *
104  * From the attached queue pair, the queue pair can enter the shutdown states
105  * when either side of the queue pair detaches. If the guest side detaches
106  * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107  * the content of the queue pair will no longer be available. If the host
108  * side detaches first, the queue pair will either enter the
109  * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110  * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111  * (e.g., the host detaches while a guest is stunned).
112  *
113  * New-style VMX'en will also unmap guest memory, if the guest is
114  * quiesced, e.g., during a snapshot operation. In that case, the guest
115  * memory will no longer be available, and the queue pair will transition from
116  * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117  * in which case the queue pair will transition from the *_NO_MEM state at that
118  * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119  * since the peer may have either attached or detached in the meantime. The
120  * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121  * *_MEM state, and vice versa.
122  */
123 
124 /* The Kernel specific component of the struct vmci_queue structure. */
125 struct vmci_queue_kern_if {
126 	struct mutex __mutex;	/* Protects the queue. */
127 	struct mutex *mutex;	/* Shared by producer and consumer queues. */
128 	size_t num_pages;	/* Number of pages incl. header. */
129 	bool host;		/* Host or guest? */
130 	union {
131 		struct {
132 			dma_addr_t *pas;
133 			void **vas;
134 		} g;		/* Used by the guest. */
135 		struct {
136 			struct page **page;
137 			struct page **header_page;
138 		} h;		/* Used by the host. */
139 	} u;
140 };
141 
142 /*
143  * This structure is opaque to the clients.
144  */
145 struct vmci_qp {
146 	struct vmci_handle handle;
147 	struct vmci_queue *produce_q;
148 	struct vmci_queue *consume_q;
149 	u64 produce_q_size;
150 	u64 consume_q_size;
151 	u32 peer;
152 	u32 flags;
153 	u32 priv_flags;
154 	bool guest_endpoint;
155 	unsigned int blocked;
156 	unsigned int generation;
157 	wait_queue_head_t event;
158 };
159 
160 enum qp_broker_state {
161 	VMCIQPB_NEW,
162 	VMCIQPB_CREATED_NO_MEM,
163 	VMCIQPB_CREATED_MEM,
164 	VMCIQPB_ATTACHED_NO_MEM,
165 	VMCIQPB_ATTACHED_MEM,
166 	VMCIQPB_SHUTDOWN_NO_MEM,
167 	VMCIQPB_SHUTDOWN_MEM,
168 	VMCIQPB_GONE
169 };
170 
171 #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 				     _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 				     _qpb->state == VMCIQPB_SHUTDOWN_MEM)
174 
175 /*
176  * In the queue pair broker, we always use the guest point of view for
177  * the produce and consume queue values and references, e.g., the
178  * produce queue size stored is the guests produce queue size. The
179  * host endpoint will need to swap these around. The only exception is
180  * the local queue pairs on the host, in which case the host endpoint
181  * that creates the queue pair will have the right orientation, and
182  * the attaching host endpoint will need to swap.
183  */
184 struct qp_entry {
185 	struct list_head list_item;
186 	struct vmci_handle handle;
187 	u32 peer;
188 	u32 flags;
189 	u64 produce_size;
190 	u64 consume_size;
191 	u32 ref_count;
192 };
193 
194 struct qp_broker_entry {
195 	struct vmci_resource resource;
196 	struct qp_entry qp;
197 	u32 create_id;
198 	u32 attach_id;
199 	enum qp_broker_state state;
200 	bool require_trusted_attach;
201 	bool created_by_trusted;
202 	bool vmci_page_files;	/* Created by VMX using VMCI page files */
203 	struct vmci_queue *produce_q;
204 	struct vmci_queue *consume_q;
205 	struct vmci_queue_header saved_produce_q;
206 	struct vmci_queue_header saved_consume_q;
207 	vmci_event_release_cb wakeup_cb;
208 	void *client_data;
209 	void *local_mem;	/* Kernel memory for local queue pair */
210 };
211 
212 struct qp_guest_endpoint {
213 	struct vmci_resource resource;
214 	struct qp_entry qp;
215 	u64 num_ppns;
216 	void *produce_q;
217 	void *consume_q;
218 	struct ppn_set ppn_set;
219 };
220 
221 struct qp_list {
222 	struct list_head head;
223 	struct mutex mutex;	/* Protect queue list. */
224 };
225 
226 static struct qp_list qp_broker_list = {
227 	.head = LIST_HEAD_INIT(qp_broker_list.head),
228 	.mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
229 };
230 
231 static struct qp_list qp_guest_endpoints = {
232 	.head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 	.mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
234 };
235 
236 #define INVALID_VMCI_GUEST_MEM_ID  0
237 #define QPE_NUM_PAGES(_QPE) ((u32) \
238 			     (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 			      DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240 #define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \
241 	((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \
242 	 (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY)
243 
244 /*
245  * Frees kernel VA space for a given queue and its queue header, and
246  * frees physical data pages.
247  */
248 static void qp_free_queue(void *q, u64 size)
249 {
250 	struct vmci_queue *queue = q;
251 
252 	if (queue) {
253 		u64 i;
254 
255 		/* Given size does not include header, so add in a page here. */
256 		for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
257 			dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
258 					  queue->kernel_if->u.g.vas[i],
259 					  queue->kernel_if->u.g.pas[i]);
260 		}
261 
262 		vfree(queue);
263 	}
264 }
265 
266 /*
267  * Allocates kernel queue pages of specified size with IOMMU mappings,
268  * plus space for the queue structure/kernel interface and the queue
269  * header.
270  */
271 static void *qp_alloc_queue(u64 size, u32 flags)
272 {
273 	u64 i;
274 	struct vmci_queue *queue;
275 	size_t pas_size;
276 	size_t vas_size;
277 	size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
278 	u64 num_pages;
279 
280 	if (size > SIZE_MAX - PAGE_SIZE)
281 		return NULL;
282 	num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
283 	if (num_pages >
284 		 (SIZE_MAX - queue_size) /
285 		 (sizeof(*queue->kernel_if->u.g.pas) +
286 		  sizeof(*queue->kernel_if->u.g.vas)))
287 		return NULL;
288 
289 	pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
290 	vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
291 	queue_size += pas_size + vas_size;
292 
293 	queue = vmalloc(queue_size);
294 	if (!queue)
295 		return NULL;
296 
297 	queue->q_header = NULL;
298 	queue->saved_header = NULL;
299 	queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
300 	queue->kernel_if->mutex = NULL;
301 	queue->kernel_if->num_pages = num_pages;
302 	queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
303 	queue->kernel_if->u.g.vas =
304 		(void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
305 	queue->kernel_if->host = false;
306 
307 	for (i = 0; i < num_pages; i++) {
308 		queue->kernel_if->u.g.vas[i] =
309 			dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
310 					   &queue->kernel_if->u.g.pas[i],
311 					   GFP_KERNEL);
312 		if (!queue->kernel_if->u.g.vas[i]) {
313 			/* Size excl. the header. */
314 			qp_free_queue(queue, i * PAGE_SIZE);
315 			return NULL;
316 		}
317 	}
318 
319 	/* Queue header is the first page. */
320 	queue->q_header = queue->kernel_if->u.g.vas[0];
321 
322 	return queue;
323 }
324 
325 /*
326  * Copies from a given buffer or iovector to a VMCI Queue.  Uses
327  * kmap_local_page() to dynamically map required portions of the queue
328  * by traversing the offset -> page translation structure for the queue.
329  * Assumes that offset + size does not wrap around in the queue.
330  */
331 static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
332 				  u64 queue_offset,
333 				  struct iov_iter *from,
334 				  size_t size)
335 {
336 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
337 	size_t bytes_copied = 0;
338 
339 	while (bytes_copied < size) {
340 		const u64 page_index =
341 			(queue_offset + bytes_copied) / PAGE_SIZE;
342 		const size_t page_offset =
343 		    (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
344 		void *va;
345 		size_t to_copy;
346 
347 		if (kernel_if->host)
348 			va = kmap_local_page(kernel_if->u.h.page[page_index]);
349 		else
350 			va = kernel_if->u.g.vas[page_index + 1];
351 			/* Skip header. */
352 
353 		if (size - bytes_copied > PAGE_SIZE - page_offset)
354 			/* Enough payload to fill up from this page. */
355 			to_copy = PAGE_SIZE - page_offset;
356 		else
357 			to_copy = size - bytes_copied;
358 
359 		if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
360 					 from)) {
361 			if (kernel_if->host)
362 				kunmap_local(va);
363 			return VMCI_ERROR_INVALID_ARGS;
364 		}
365 		bytes_copied += to_copy;
366 		if (kernel_if->host)
367 			kunmap_local(va);
368 	}
369 
370 	return VMCI_SUCCESS;
371 }
372 
373 /*
374  * Copies to a given buffer or iovector from a VMCI Queue.  Uses
375  * kmap_local_page() to dynamically map required portions of the queue
376  * by traversing the offset -> page translation structure for the queue.
377  * Assumes that offset + size does not wrap around in the queue.
378  */
379 static int qp_memcpy_from_queue_iter(struct iov_iter *to,
380 				    const struct vmci_queue *queue,
381 				    u64 queue_offset, size_t size)
382 {
383 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
384 	size_t bytes_copied = 0;
385 
386 	while (bytes_copied < size) {
387 		const u64 page_index =
388 			(queue_offset + bytes_copied) / PAGE_SIZE;
389 		const size_t page_offset =
390 		    (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
391 		void *va;
392 		size_t to_copy;
393 		int err;
394 
395 		if (kernel_if->host)
396 			va = kmap_local_page(kernel_if->u.h.page[page_index]);
397 		else
398 			va = kernel_if->u.g.vas[page_index + 1];
399 			/* Skip header. */
400 
401 		if (size - bytes_copied > PAGE_SIZE - page_offset)
402 			/* Enough payload to fill up this page. */
403 			to_copy = PAGE_SIZE - page_offset;
404 		else
405 			to_copy = size - bytes_copied;
406 
407 		err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
408 		if (err != to_copy) {
409 			if (kernel_if->host)
410 				kunmap_local(va);
411 			return VMCI_ERROR_INVALID_ARGS;
412 		}
413 		bytes_copied += to_copy;
414 		if (kernel_if->host)
415 			kunmap_local(va);
416 	}
417 
418 	return VMCI_SUCCESS;
419 }
420 
421 /*
422  * Allocates two list of PPNs --- one for the pages in the produce queue,
423  * and the other for the pages in the consume queue. Initializes the list
424  * of PPNs with the page frame numbers of the KVA for the two queues (and
425  * the queue headers).
426  */
427 static int qp_alloc_ppn_set(void *prod_q,
428 			    u64 num_produce_pages,
429 			    void *cons_q,
430 			    u64 num_consume_pages, struct ppn_set *ppn_set)
431 {
432 	u64 *produce_ppns;
433 	u64 *consume_ppns;
434 	struct vmci_queue *produce_q = prod_q;
435 	struct vmci_queue *consume_q = cons_q;
436 	u64 i;
437 
438 	if (!produce_q || !num_produce_pages || !consume_q ||
439 	    !num_consume_pages || !ppn_set)
440 		return VMCI_ERROR_INVALID_ARGS;
441 
442 	if (ppn_set->initialized)
443 		return VMCI_ERROR_ALREADY_EXISTS;
444 
445 	produce_ppns =
446 	    kmalloc_objs(*produce_ppns, num_produce_pages);
447 	if (!produce_ppns)
448 		return VMCI_ERROR_NO_MEM;
449 
450 	consume_ppns =
451 	    kmalloc_objs(*consume_ppns, num_consume_pages);
452 	if (!consume_ppns) {
453 		kfree(produce_ppns);
454 		return VMCI_ERROR_NO_MEM;
455 	}
456 
457 	for (i = 0; i < num_produce_pages; i++)
458 		produce_ppns[i] =
459 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
460 
461 	for (i = 0; i < num_consume_pages; i++)
462 		consume_ppns[i] =
463 			consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
464 
465 	ppn_set->num_produce_pages = num_produce_pages;
466 	ppn_set->num_consume_pages = num_consume_pages;
467 	ppn_set->produce_ppns = produce_ppns;
468 	ppn_set->consume_ppns = consume_ppns;
469 	ppn_set->initialized = true;
470 	return VMCI_SUCCESS;
471 }
472 
473 /*
474  * Frees the two list of PPNs for a queue pair.
475  */
476 static void qp_free_ppn_set(struct ppn_set *ppn_set)
477 {
478 	if (ppn_set->initialized) {
479 		/* Do not call these functions on NULL inputs. */
480 		kfree(ppn_set->produce_ppns);
481 		kfree(ppn_set->consume_ppns);
482 	}
483 	memset(ppn_set, 0, sizeof(*ppn_set));
484 }
485 
486 /*
487  * Populates the list of PPNs in the hypercall structure with the PPNS
488  * of the produce queue and the consume queue.
489  */
490 static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
491 {
492 	if (vmci_use_ppn64()) {
493 		memcpy(call_buf, ppn_set->produce_ppns,
494 		       ppn_set->num_produce_pages *
495 		       sizeof(*ppn_set->produce_ppns));
496 		memcpy(call_buf +
497 		       ppn_set->num_produce_pages *
498 		       sizeof(*ppn_set->produce_ppns),
499 		       ppn_set->consume_ppns,
500 		       ppn_set->num_consume_pages *
501 		       sizeof(*ppn_set->consume_ppns));
502 	} else {
503 		int i;
504 		u32 *ppns = (u32 *) call_buf;
505 
506 		for (i = 0; i < ppn_set->num_produce_pages; i++)
507 			ppns[i] = (u32) ppn_set->produce_ppns[i];
508 
509 		ppns = &ppns[ppn_set->num_produce_pages];
510 
511 		for (i = 0; i < ppn_set->num_consume_pages; i++)
512 			ppns[i] = (u32) ppn_set->consume_ppns[i];
513 	}
514 
515 	return VMCI_SUCCESS;
516 }
517 
518 /*
519  * Allocates kernel VA space of specified size plus space for the queue
520  * and kernel interface.  This is different from the guest queue allocator,
521  * because we do not allocate our own queue header/data pages here but
522  * share those of the guest.
523  */
524 static struct vmci_queue *qp_host_alloc_queue(u64 size)
525 {
526 	struct vmci_queue *queue;
527 	size_t queue_page_size;
528 	u64 num_pages;
529 	const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
530 
531 	if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE))
532 		return NULL;
533 	num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
534 	if (num_pages > (SIZE_MAX - queue_size) /
535 		 sizeof(*queue->kernel_if->u.h.page))
536 		return NULL;
537 
538 	queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
539 
540 	if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
541 		return NULL;
542 
543 	queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
544 	if (queue) {
545 		queue->q_header = NULL;
546 		queue->saved_header = NULL;
547 		queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
548 		queue->kernel_if->host = true;
549 		queue->kernel_if->mutex = NULL;
550 		queue->kernel_if->num_pages = num_pages;
551 		queue->kernel_if->u.h.header_page =
552 		    (struct page **)((u8 *)queue + queue_size);
553 		queue->kernel_if->u.h.page =
554 			&queue->kernel_if->u.h.header_page[1];
555 	}
556 
557 	return queue;
558 }
559 
560 /*
561  * Frees kernel memory for a given queue (header plus translation
562  * structure).
563  */
564 static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
565 {
566 	kfree(queue);
567 }
568 
569 /*
570  * Initialize the mutex for the pair of queues.  This mutex is used to
571  * protect the q_header and the buffer from changing out from under any
572  * users of either queue.  Of course, it's only any good if the mutexes
573  * are actually acquired.  Queue structure must lie on non-paged memory
574  * or we cannot guarantee access to the mutex.
575  */
576 static void qp_init_queue_mutex(struct vmci_queue *produce_q,
577 				struct vmci_queue *consume_q)
578 {
579 	/*
580 	 * Only the host queue has shared state - the guest queues do not
581 	 * need to synchronize access using a queue mutex.
582 	 */
583 
584 	if (produce_q->kernel_if->host) {
585 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
586 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
587 		mutex_init(produce_q->kernel_if->mutex);
588 	}
589 }
590 
591 /*
592  * Cleans up the mutex for the pair of queues.
593  */
594 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
595 				   struct vmci_queue *consume_q)
596 {
597 	if (produce_q->kernel_if->host) {
598 		produce_q->kernel_if->mutex = NULL;
599 		consume_q->kernel_if->mutex = NULL;
600 	}
601 }
602 
603 /*
604  * Acquire the mutex for the queue.  Note that the produce_q and
605  * the consume_q share a mutex.  So, only one of the two need to
606  * be passed in to this routine.  Either will work just fine.
607  */
608 static void qp_acquire_queue_mutex(struct vmci_queue *queue)
609 {
610 	if (queue->kernel_if->host)
611 		mutex_lock(queue->kernel_if->mutex);
612 }
613 
614 /*
615  * Release the mutex for the queue.  Note that the produce_q and
616  * the consume_q share a mutex.  So, only one of the two need to
617  * be passed in to this routine.  Either will work just fine.
618  */
619 static void qp_release_queue_mutex(struct vmci_queue *queue)
620 {
621 	if (queue->kernel_if->host)
622 		mutex_unlock(queue->kernel_if->mutex);
623 }
624 
625 /*
626  * Helper function to release pages in the PageStoreAttachInfo
627  * previously obtained using get_user_pages.
628  */
629 static void qp_release_pages(struct page **pages,
630 			     u64 num_pages, bool dirty)
631 {
632 	int i;
633 
634 	for (i = 0; i < num_pages; i++) {
635 		if (dirty)
636 			set_page_dirty_lock(pages[i]);
637 
638 		put_page(pages[i]);
639 		pages[i] = NULL;
640 	}
641 }
642 
643 /*
644  * Lock the user pages referenced by the {produce,consume}Buffer
645  * struct into memory and populate the {produce,consume}Pages
646  * arrays in the attach structure with them.
647  */
648 static int qp_host_get_user_memory(u64 produce_uva,
649 				   u64 consume_uva,
650 				   struct vmci_queue *produce_q,
651 				   struct vmci_queue *consume_q)
652 {
653 	int retval;
654 	int err = VMCI_SUCCESS;
655 
656 	retval = get_user_pages_fast((uintptr_t) produce_uva,
657 				     produce_q->kernel_if->num_pages,
658 				     FOLL_WRITE,
659 				     produce_q->kernel_if->u.h.header_page);
660 	if (retval < (int)produce_q->kernel_if->num_pages) {
661 		pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
662 			retval);
663 		if (retval > 0)
664 			qp_release_pages(produce_q->kernel_if->u.h.header_page,
665 					retval, false);
666 		err = VMCI_ERROR_NO_MEM;
667 		goto out;
668 	}
669 
670 	retval = get_user_pages_fast((uintptr_t) consume_uva,
671 				     consume_q->kernel_if->num_pages,
672 				     FOLL_WRITE,
673 				     consume_q->kernel_if->u.h.header_page);
674 	if (retval < (int)consume_q->kernel_if->num_pages) {
675 		pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
676 			retval);
677 		if (retval > 0)
678 			qp_release_pages(consume_q->kernel_if->u.h.header_page,
679 					retval, false);
680 		qp_release_pages(produce_q->kernel_if->u.h.header_page,
681 				 produce_q->kernel_if->num_pages, false);
682 		err = VMCI_ERROR_NO_MEM;
683 	}
684 
685  out:
686 	return err;
687 }
688 
689 /*
690  * Registers the specification of the user pages used for backing a queue
691  * pair. Enough information to map in pages is stored in the OS specific
692  * part of the struct vmci_queue structure.
693  */
694 static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
695 					struct vmci_queue *produce_q,
696 					struct vmci_queue *consume_q)
697 {
698 	u64 produce_uva;
699 	u64 consume_uva;
700 
701 	/*
702 	 * The new style and the old style mapping only differs in
703 	 * that we either get a single or two UVAs, so we split the
704 	 * single UVA range at the appropriate spot.
705 	 */
706 	produce_uva = page_store->pages;
707 	consume_uva = page_store->pages +
708 	    produce_q->kernel_if->num_pages * PAGE_SIZE;
709 	return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
710 				       consume_q);
711 }
712 
713 /*
714  * Releases and removes the references to user pages stored in the attach
715  * struct.  Pages are released from the page cache and may become
716  * swappable again.
717  */
718 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
719 					   struct vmci_queue *consume_q)
720 {
721 	qp_release_pages(produce_q->kernel_if->u.h.header_page,
722 			 produce_q->kernel_if->num_pages, true);
723 	memset(produce_q->kernel_if->u.h.header_page, 0,
724 	       sizeof(*produce_q->kernel_if->u.h.header_page) *
725 	       produce_q->kernel_if->num_pages);
726 	qp_release_pages(consume_q->kernel_if->u.h.header_page,
727 			 consume_q->kernel_if->num_pages, true);
728 	memset(consume_q->kernel_if->u.h.header_page, 0,
729 	       sizeof(*consume_q->kernel_if->u.h.header_page) *
730 	       consume_q->kernel_if->num_pages);
731 }
732 
733 /*
734  * Once qp_host_register_user_memory has been performed on a
735  * queue, the queue pair headers can be mapped into the
736  * kernel. Once mapped, they must be unmapped with
737  * qp_host_unmap_queues prior to calling
738  * qp_host_unregister_user_memory.
739  * Pages are pinned.
740  */
741 static int qp_host_map_queues(struct vmci_queue *produce_q,
742 			      struct vmci_queue *consume_q)
743 {
744 	int result;
745 
746 	if (!produce_q->q_header || !consume_q->q_header) {
747 		struct page *headers[2];
748 
749 		if (produce_q->q_header != consume_q->q_header)
750 			return VMCI_ERROR_QUEUEPAIR_MISMATCH;
751 
752 		if (produce_q->kernel_if->u.h.header_page == NULL ||
753 		    *produce_q->kernel_if->u.h.header_page == NULL)
754 			return VMCI_ERROR_UNAVAILABLE;
755 
756 		headers[0] = *produce_q->kernel_if->u.h.header_page;
757 		headers[1] = *consume_q->kernel_if->u.h.header_page;
758 
759 		produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
760 		if (produce_q->q_header != NULL) {
761 			consume_q->q_header =
762 			    (struct vmci_queue_header *)((u8 *)
763 							 produce_q->q_header +
764 							 PAGE_SIZE);
765 			result = VMCI_SUCCESS;
766 		} else {
767 			pr_warn("vmap failed\n");
768 			result = VMCI_ERROR_NO_MEM;
769 		}
770 	} else {
771 		result = VMCI_SUCCESS;
772 	}
773 
774 	return result;
775 }
776 
777 /*
778  * Unmaps previously mapped queue pair headers from the kernel.
779  * Pages are unpinned.
780  */
781 static int qp_host_unmap_queues(u32 gid,
782 				struct vmci_queue *produce_q,
783 				struct vmci_queue *consume_q)
784 {
785 	if (produce_q->q_header) {
786 		if (produce_q->q_header < consume_q->q_header)
787 			vunmap(produce_q->q_header);
788 		else
789 			vunmap(consume_q->q_header);
790 
791 		produce_q->q_header = NULL;
792 		consume_q->q_header = NULL;
793 	}
794 
795 	return VMCI_SUCCESS;
796 }
797 
798 /*
799  * Finds the entry in the list corresponding to a given handle. Assumes
800  * that the list is locked.
801  */
802 static struct qp_entry *qp_list_find(struct qp_list *qp_list,
803 				     struct vmci_handle handle)
804 {
805 	struct qp_entry *entry;
806 
807 	if (vmci_handle_is_invalid(handle))
808 		return NULL;
809 
810 	list_for_each_entry(entry, &qp_list->head, list_item) {
811 		if (vmci_handle_is_equal(entry->handle, handle))
812 			return entry;
813 	}
814 
815 	return NULL;
816 }
817 
818 /*
819  * Finds the entry in the list corresponding to a given handle.
820  */
821 static struct qp_guest_endpoint *
822 qp_guest_handle_to_entry(struct vmci_handle handle)
823 {
824 	struct qp_guest_endpoint *entry;
825 	struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
826 
827 	entry = qp ? container_of(
828 		qp, struct qp_guest_endpoint, qp) : NULL;
829 	return entry;
830 }
831 
832 /*
833  * Finds the entry in the list corresponding to a given handle.
834  */
835 static struct qp_broker_entry *
836 qp_broker_handle_to_entry(struct vmci_handle handle)
837 {
838 	struct qp_broker_entry *entry;
839 	struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
840 
841 	entry = qp ? container_of(
842 		qp, struct qp_broker_entry, qp) : NULL;
843 	return entry;
844 }
845 
846 /*
847  * Dispatches a queue pair event message directly into the local event
848  * queue.
849  */
850 static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
851 {
852 	u32 context_id = vmci_get_context_id();
853 	struct vmci_event_qp ev;
854 
855 	memset(&ev, 0, sizeof(ev));
856 	ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
857 	ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
858 					  VMCI_CONTEXT_RESOURCE_ID);
859 	ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
860 	ev.msg.event_data.event =
861 	    attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
862 	ev.payload.peer_id = context_id;
863 	ev.payload.handle = handle;
864 
865 	return vmci_event_dispatch(&ev.msg.hdr);
866 }
867 
868 /*
869  * Allocates and initializes a qp_guest_endpoint structure.
870  * Allocates a queue_pair rid (and handle) iff the given entry has
871  * an invalid handle.  0 through VMCI_RESERVED_RESOURCE_ID_MAX
872  * are reserved handles.  Assumes that the QP list mutex is held
873  * by the caller.
874  */
875 static struct qp_guest_endpoint *
876 qp_guest_endpoint_create(struct vmci_handle handle,
877 			 u32 peer,
878 			 u32 flags,
879 			 u64 produce_size,
880 			 u64 consume_size,
881 			 void *produce_q,
882 			 void *consume_q)
883 {
884 	int result;
885 	struct qp_guest_endpoint *entry;
886 	/* One page each for the queue headers. */
887 	const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
888 	    DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
889 
890 	if (vmci_handle_is_invalid(handle)) {
891 		u32 context_id = vmci_get_context_id();
892 
893 		handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
894 	}
895 
896 	entry = kzalloc_obj(*entry);
897 	if (entry) {
898 		entry->qp.peer = peer;
899 		entry->qp.flags = flags;
900 		entry->qp.produce_size = produce_size;
901 		entry->qp.consume_size = consume_size;
902 		entry->qp.ref_count = 0;
903 		entry->num_ppns = num_ppns;
904 		entry->produce_q = produce_q;
905 		entry->consume_q = consume_q;
906 		INIT_LIST_HEAD(&entry->qp.list_item);
907 
908 		/* Add resource obj */
909 		result = vmci_resource_add(&entry->resource,
910 					   VMCI_RESOURCE_TYPE_QPAIR_GUEST,
911 					   handle);
912 		entry->qp.handle = vmci_resource_handle(&entry->resource);
913 		if ((result != VMCI_SUCCESS) ||
914 		    qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
915 			pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
916 				handle.context, handle.resource, result);
917 			kfree(entry);
918 			entry = NULL;
919 		}
920 	}
921 	return entry;
922 }
923 
924 /*
925  * Frees a qp_guest_endpoint structure.
926  */
927 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
928 {
929 	qp_free_ppn_set(&entry->ppn_set);
930 	qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
931 	qp_free_queue(entry->produce_q, entry->qp.produce_size);
932 	qp_free_queue(entry->consume_q, entry->qp.consume_size);
933 	/* Unlink from resource hash table and free callback */
934 	vmci_resource_remove(&entry->resource);
935 
936 	kfree(entry);
937 }
938 
939 /*
940  * Helper to make a queue_pairAlloc hypercall when the driver is
941  * supporting a guest device.
942  */
943 static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
944 {
945 	struct vmci_qp_alloc_msg *alloc_msg;
946 	size_t msg_size;
947 	size_t ppn_size;
948 	int result;
949 
950 	if (!entry || entry->num_ppns <= 2)
951 		return VMCI_ERROR_INVALID_ARGS;
952 
953 	ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
954 	msg_size = sizeof(*alloc_msg) +
955 	    (size_t) entry->num_ppns * ppn_size;
956 	alloc_msg = kmalloc(msg_size, GFP_KERNEL);
957 	if (!alloc_msg)
958 		return VMCI_ERROR_NO_MEM;
959 
960 	alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
961 					      VMCI_QUEUEPAIR_ALLOC);
962 	alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
963 	alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
964 	alloc_msg->handle = entry->qp.handle;
965 	alloc_msg->peer = entry->qp.peer;
966 	alloc_msg->flags = entry->qp.flags;
967 	alloc_msg->produce_size = entry->qp.produce_size;
968 	alloc_msg->consume_size = entry->qp.consume_size;
969 	alloc_msg->num_ppns = entry->num_ppns;
970 
971 	result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
972 				     &entry->ppn_set);
973 	if (result == VMCI_SUCCESS)
974 		result = vmci_send_datagram(&alloc_msg->hdr);
975 
976 	kfree(alloc_msg);
977 
978 	return result;
979 }
980 
981 /*
982  * Helper to make a queue_pairDetach hypercall when the driver is
983  * supporting a guest device.
984  */
985 static int qp_detatch_hypercall(struct vmci_handle handle)
986 {
987 	struct vmci_qp_detach_msg detach_msg;
988 
989 	detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
990 					      VMCI_QUEUEPAIR_DETACH);
991 	detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
992 	detach_msg.hdr.payload_size = sizeof(handle);
993 	detach_msg.handle = handle;
994 
995 	return vmci_send_datagram(&detach_msg.hdr);
996 }
997 
998 /*
999  * Adds the given entry to the list. Assumes that the list is locked.
1000  */
1001 static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1002 {
1003 	if (entry)
1004 		list_add(&entry->list_item, &qp_list->head);
1005 }
1006 
1007 /*
1008  * Removes the given entry from the list. Assumes that the list is locked.
1009  */
1010 static void qp_list_remove_entry(struct qp_list *qp_list,
1011 				 struct qp_entry *entry)
1012 {
1013 	if (entry)
1014 		list_del(&entry->list_item);
1015 }
1016 
1017 /*
1018  * Helper for VMCI queue_pair detach interface. Frees the physical
1019  * pages for the queue pair.
1020  */
1021 static int qp_detatch_guest_work(struct vmci_handle handle)
1022 {
1023 	int result;
1024 	struct qp_guest_endpoint *entry;
1025 	u32 ref_count = ~0;	/* To avoid compiler warning below */
1026 
1027 	mutex_lock(&qp_guest_endpoints.mutex);
1028 
1029 	entry = qp_guest_handle_to_entry(handle);
1030 	if (!entry) {
1031 		mutex_unlock(&qp_guest_endpoints.mutex);
1032 		return VMCI_ERROR_NOT_FOUND;
1033 	}
1034 
1035 	if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1036 		result = VMCI_SUCCESS;
1037 
1038 		if (entry->qp.ref_count > 1) {
1039 			result = qp_notify_peer_local(false, handle);
1040 			/*
1041 			 * We can fail to notify a local queuepair
1042 			 * because we can't allocate.  We still want
1043 			 * to release the entry if that happens, so
1044 			 * don't bail out yet.
1045 			 */
1046 		}
1047 	} else {
1048 		result = qp_detatch_hypercall(handle);
1049 		if (result < VMCI_SUCCESS) {
1050 			/*
1051 			 * We failed to notify a non-local queuepair.
1052 			 * That other queuepair might still be
1053 			 * accessing the shared memory, so don't
1054 			 * release the entry yet.  It will get cleaned
1055 			 * up by VMCIqueue_pair_Exit() if necessary
1056 			 * (assuming we are going away, otherwise why
1057 			 * did this fail?).
1058 			 */
1059 
1060 			mutex_unlock(&qp_guest_endpoints.mutex);
1061 			return result;
1062 		}
1063 	}
1064 
1065 	/*
1066 	 * If we get here then we either failed to notify a local queuepair, or
1067 	 * we succeeded in all cases.  Release the entry if required.
1068 	 */
1069 
1070 	entry->qp.ref_count--;
1071 	if (entry->qp.ref_count == 0)
1072 		qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1073 
1074 	/* If we didn't remove the entry, this could change once we unlock. */
1075 	if (entry)
1076 		ref_count = entry->qp.ref_count;
1077 
1078 	mutex_unlock(&qp_guest_endpoints.mutex);
1079 
1080 	if (ref_count == 0)
1081 		qp_guest_endpoint_destroy(entry);
1082 
1083 	return result;
1084 }
1085 
1086 /*
1087  * This functions handles the actual allocation of a VMCI queue
1088  * pair guest endpoint. Allocates physical pages for the queue
1089  * pair. It makes OS dependent calls through generic wrappers.
1090  */
1091 static int qp_alloc_guest_work(struct vmci_handle *handle,
1092 			       struct vmci_queue **produce_q,
1093 			       u64 produce_size,
1094 			       struct vmci_queue **consume_q,
1095 			       u64 consume_size,
1096 			       u32 peer,
1097 			       u32 flags,
1098 			       u32 priv_flags)
1099 {
1100 	const u64 num_produce_pages =
1101 	    DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1102 	const u64 num_consume_pages =
1103 	    DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1104 	void *my_produce_q = NULL;
1105 	void *my_consume_q = NULL;
1106 	int result;
1107 	struct qp_guest_endpoint *queue_pair_entry = NULL;
1108 
1109 	if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1110 		return VMCI_ERROR_NO_ACCESS;
1111 
1112 	mutex_lock(&qp_guest_endpoints.mutex);
1113 
1114 	queue_pair_entry = qp_guest_handle_to_entry(*handle);
1115 	if (queue_pair_entry) {
1116 		if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1117 			/* Local attach case. */
1118 			if (queue_pair_entry->qp.ref_count > 1) {
1119 				pr_devel("Error attempting to attach more than once\n");
1120 				result = VMCI_ERROR_UNAVAILABLE;
1121 				goto error_keep_entry;
1122 			}
1123 
1124 			if (queue_pair_entry->qp.produce_size != consume_size ||
1125 			    queue_pair_entry->qp.consume_size !=
1126 			    produce_size ||
1127 			    queue_pair_entry->qp.flags !=
1128 			    (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1129 				pr_devel("Error mismatched queue pair in local attach\n");
1130 				result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1131 				goto error_keep_entry;
1132 			}
1133 
1134 			/*
1135 			 * Do a local attach.  We swap the consume and
1136 			 * produce queues for the attacher and deliver
1137 			 * an attach event.
1138 			 */
1139 			result = qp_notify_peer_local(true, *handle);
1140 			if (result < VMCI_SUCCESS)
1141 				goto error_keep_entry;
1142 
1143 			my_produce_q = queue_pair_entry->consume_q;
1144 			my_consume_q = queue_pair_entry->produce_q;
1145 			goto out;
1146 		}
1147 
1148 		result = VMCI_ERROR_ALREADY_EXISTS;
1149 		goto error_keep_entry;
1150 	}
1151 
1152 	my_produce_q = qp_alloc_queue(produce_size, flags);
1153 	if (!my_produce_q) {
1154 		pr_warn("Error allocating pages for produce queue\n");
1155 		result = VMCI_ERROR_NO_MEM;
1156 		goto error;
1157 	}
1158 
1159 	my_consume_q = qp_alloc_queue(consume_size, flags);
1160 	if (!my_consume_q) {
1161 		pr_warn("Error allocating pages for consume queue\n");
1162 		result = VMCI_ERROR_NO_MEM;
1163 		goto error;
1164 	}
1165 
1166 	queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1167 						    produce_size, consume_size,
1168 						    my_produce_q, my_consume_q);
1169 	if (!queue_pair_entry) {
1170 		pr_warn("Error allocating memory in %s\n", __func__);
1171 		result = VMCI_ERROR_NO_MEM;
1172 		goto error;
1173 	}
1174 
1175 	result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1176 				  num_consume_pages,
1177 				  &queue_pair_entry->ppn_set);
1178 	if (result < VMCI_SUCCESS) {
1179 		pr_warn("qp_alloc_ppn_set failed\n");
1180 		goto error;
1181 	}
1182 
1183 	/*
1184 	 * It's only necessary to notify the host if this queue pair will be
1185 	 * attached to from another context.
1186 	 */
1187 	if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1188 		/* Local create case. */
1189 		u32 context_id = vmci_get_context_id();
1190 
1191 		/*
1192 		 * Enforce similar checks on local queue pairs as we
1193 		 * do for regular ones.  The handle's context must
1194 		 * match the creator or attacher context id (here they
1195 		 * are both the current context id) and the
1196 		 * attach-only flag cannot exist during create.  We
1197 		 * also ensure specified peer is this context or an
1198 		 * invalid one.
1199 		 */
1200 		if (queue_pair_entry->qp.handle.context != context_id ||
1201 		    (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1202 		     queue_pair_entry->qp.peer != context_id)) {
1203 			result = VMCI_ERROR_NO_ACCESS;
1204 			goto error;
1205 		}
1206 
1207 		if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1208 			result = VMCI_ERROR_NOT_FOUND;
1209 			goto error;
1210 		}
1211 	} else {
1212 		result = qp_alloc_hypercall(queue_pair_entry);
1213 		if (result < VMCI_SUCCESS) {
1214 			pr_devel("qp_alloc_hypercall result = %d\n", result);
1215 			goto error;
1216 		}
1217 	}
1218 
1219 	qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1220 			    (struct vmci_queue *)my_consume_q);
1221 
1222 	qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1223 
1224  out:
1225 	queue_pair_entry->qp.ref_count++;
1226 	*handle = queue_pair_entry->qp.handle;
1227 	*produce_q = (struct vmci_queue *)my_produce_q;
1228 	*consume_q = (struct vmci_queue *)my_consume_q;
1229 
1230 	/*
1231 	 * We should initialize the queue pair header pages on a local
1232 	 * queue pair create.  For non-local queue pairs, the
1233 	 * hypervisor initializes the header pages in the create step.
1234 	 */
1235 	if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1236 	    queue_pair_entry->qp.ref_count == 1) {
1237 		vmci_q_header_init((*produce_q)->q_header, *handle);
1238 		vmci_q_header_init((*consume_q)->q_header, *handle);
1239 	}
1240 
1241 	mutex_unlock(&qp_guest_endpoints.mutex);
1242 
1243 	return VMCI_SUCCESS;
1244 
1245  error:
1246 	mutex_unlock(&qp_guest_endpoints.mutex);
1247 	if (queue_pair_entry) {
1248 		/* The queues will be freed inside the destroy routine. */
1249 		qp_guest_endpoint_destroy(queue_pair_entry);
1250 	} else {
1251 		qp_free_queue(my_produce_q, produce_size);
1252 		qp_free_queue(my_consume_q, consume_size);
1253 	}
1254 	return result;
1255 
1256  error_keep_entry:
1257 	/* This path should only be used when an existing entry was found. */
1258 	mutex_unlock(&qp_guest_endpoints.mutex);
1259 	return result;
1260 }
1261 
1262 /*
1263  * The first endpoint issuing a queue pair allocation will create the state
1264  * of the queue pair in the queue pair broker.
1265  *
1266  * If the creator is a guest, it will associate a VMX virtual address range
1267  * with the queue pair as specified by the page_store. For compatibility with
1268  * older VMX'en, that would use a separate step to set the VMX virtual
1269  * address range, the virtual address range can be registered later using
1270  * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1271  * used.
1272  *
1273  * If the creator is the host, a page_store of NULL should be used as well,
1274  * since the host is not able to supply a page store for the queue pair.
1275  *
1276  * For older VMX and host callers, the queue pair will be created in the
1277  * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1278  * created in VMCOQPB_CREATED_MEM state.
1279  */
1280 static int qp_broker_create(struct vmci_handle handle,
1281 			    u32 peer,
1282 			    u32 flags,
1283 			    u32 priv_flags,
1284 			    u64 produce_size,
1285 			    u64 consume_size,
1286 			    struct vmci_qp_page_store *page_store,
1287 			    struct vmci_ctx *context,
1288 			    vmci_event_release_cb wakeup_cb,
1289 			    void *client_data, struct qp_broker_entry **ent)
1290 {
1291 	struct qp_broker_entry *entry = NULL;
1292 	const u32 context_id = vmci_ctx_get_id(context);
1293 	bool is_local = flags & VMCI_QPFLAG_LOCAL;
1294 	int result;
1295 	u64 guest_produce_size;
1296 	u64 guest_consume_size;
1297 
1298 	/* Do not create if the caller asked not to. */
1299 	if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1300 		return VMCI_ERROR_NOT_FOUND;
1301 
1302 	/*
1303 	 * Creator's context ID should match handle's context ID or the creator
1304 	 * must allow the context in handle's context ID as the "peer".
1305 	 */
1306 	if (handle.context != context_id && handle.context != peer)
1307 		return VMCI_ERROR_NO_ACCESS;
1308 
1309 	if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1310 		return VMCI_ERROR_DST_UNREACHABLE;
1311 
1312 	/*
1313 	 * Creator's context ID for local queue pairs should match the
1314 	 * peer, if a peer is specified.
1315 	 */
1316 	if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1317 		return VMCI_ERROR_NO_ACCESS;
1318 
1319 	entry = kzalloc_obj(*entry, GFP_ATOMIC);
1320 	if (!entry)
1321 		return VMCI_ERROR_NO_MEM;
1322 
1323 	if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1324 		/*
1325 		 * The queue pair broker entry stores values from the guest
1326 		 * point of view, so a creating host side endpoint should swap
1327 		 * produce and consume values -- unless it is a local queue
1328 		 * pair, in which case no swapping is necessary, since the local
1329 		 * attacher will swap queues.
1330 		 */
1331 
1332 		guest_produce_size = consume_size;
1333 		guest_consume_size = produce_size;
1334 	} else {
1335 		guest_produce_size = produce_size;
1336 		guest_consume_size = consume_size;
1337 	}
1338 
1339 	entry->qp.handle = handle;
1340 	entry->qp.peer = peer;
1341 	entry->qp.flags = flags;
1342 	entry->qp.produce_size = guest_produce_size;
1343 	entry->qp.consume_size = guest_consume_size;
1344 	entry->qp.ref_count = 1;
1345 	entry->create_id = context_id;
1346 	entry->attach_id = VMCI_INVALID_ID;
1347 	entry->state = VMCIQPB_NEW;
1348 	entry->require_trusted_attach =
1349 	    !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1350 	entry->created_by_trusted =
1351 	    !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1352 	entry->vmci_page_files = false;
1353 	entry->wakeup_cb = wakeup_cb;
1354 	entry->client_data = client_data;
1355 	entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1356 	if (entry->produce_q == NULL) {
1357 		result = VMCI_ERROR_NO_MEM;
1358 		goto error;
1359 	}
1360 	entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1361 	if (entry->consume_q == NULL) {
1362 		result = VMCI_ERROR_NO_MEM;
1363 		goto error;
1364 	}
1365 
1366 	qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1367 
1368 	INIT_LIST_HEAD(&entry->qp.list_item);
1369 
1370 	if (is_local) {
1371 		u8 *tmp;
1372 
1373 		entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1374 					   PAGE_SIZE, GFP_KERNEL);
1375 		if (entry->local_mem == NULL) {
1376 			result = VMCI_ERROR_NO_MEM;
1377 			goto error;
1378 		}
1379 		entry->state = VMCIQPB_CREATED_MEM;
1380 		entry->produce_q->q_header = entry->local_mem;
1381 		tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1382 		    (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1383 		entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1384 	} else if (page_store) {
1385 		/*
1386 		 * The VMX already initialized the queue pair headers, so no
1387 		 * need for the kernel side to do that.
1388 		 */
1389 		result = qp_host_register_user_memory(page_store,
1390 						      entry->produce_q,
1391 						      entry->consume_q);
1392 		if (result < VMCI_SUCCESS)
1393 			goto error;
1394 
1395 		entry->state = VMCIQPB_CREATED_MEM;
1396 	} else {
1397 		/*
1398 		 * A create without a page_store may be either a host
1399 		 * side create (in which case we are waiting for the
1400 		 * guest side to supply the memory) or an old style
1401 		 * queue pair create (in which case we will expect a
1402 		 * set page store call as the next step).
1403 		 */
1404 		entry->state = VMCIQPB_CREATED_NO_MEM;
1405 	}
1406 
1407 	qp_list_add_entry(&qp_broker_list, &entry->qp);
1408 	if (ent != NULL)
1409 		*ent = entry;
1410 
1411 	/* Add to resource obj */
1412 	result = vmci_resource_add(&entry->resource,
1413 				   VMCI_RESOURCE_TYPE_QPAIR_HOST,
1414 				   handle);
1415 	if (result != VMCI_SUCCESS) {
1416 		pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1417 			handle.context, handle.resource, result);
1418 		goto error;
1419 	}
1420 
1421 	entry->qp.handle = vmci_resource_handle(&entry->resource);
1422 	if (is_local) {
1423 		vmci_q_header_init(entry->produce_q->q_header,
1424 				   entry->qp.handle);
1425 		vmci_q_header_init(entry->consume_q->q_header,
1426 				   entry->qp.handle);
1427 	}
1428 
1429 	vmci_ctx_qp_create(context, entry->qp.handle);
1430 
1431 	return VMCI_SUCCESS;
1432 
1433  error:
1434 	if (entry != NULL) {
1435 		qp_host_free_queue(entry->produce_q, guest_produce_size);
1436 		qp_host_free_queue(entry->consume_q, guest_consume_size);
1437 		kfree(entry);
1438 	}
1439 
1440 	return result;
1441 }
1442 
1443 /*
1444  * Enqueues an event datagram to notify the peer VM attached to
1445  * the given queue pair handle about attach/detach event by the
1446  * given VM.  Returns Payload size of datagram enqueued on
1447  * success, error code otherwise.
1448  */
1449 static int qp_notify_peer(bool attach,
1450 			  struct vmci_handle handle,
1451 			  u32 my_id,
1452 			  u32 peer_id)
1453 {
1454 	int rv;
1455 	struct vmci_event_qp ev;
1456 
1457 	if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1458 	    peer_id == VMCI_INVALID_ID)
1459 		return VMCI_ERROR_INVALID_ARGS;
1460 
1461 	/*
1462 	 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1463 	 * number of pending events from the hypervisor to a given VM
1464 	 * otherwise a rogue VM could do an arbitrary number of attach
1465 	 * and detach operations causing memory pressure in the host
1466 	 * kernel.
1467 	 */
1468 
1469 	memset(&ev, 0, sizeof(ev));
1470 	ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1471 	ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1472 					  VMCI_CONTEXT_RESOURCE_ID);
1473 	ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1474 	ev.msg.event_data.event = attach ?
1475 	    VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1476 	ev.payload.handle = handle;
1477 	ev.payload.peer_id = my_id;
1478 
1479 	rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1480 				    &ev.msg.hdr, false);
1481 	if (rv < VMCI_SUCCESS)
1482 		pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1483 			attach ? "ATTACH" : "DETACH", peer_id);
1484 
1485 	return rv;
1486 }
1487 
1488 /*
1489  * The second endpoint issuing a queue pair allocation will attach to
1490  * the queue pair registered with the queue pair broker.
1491  *
1492  * If the attacher is a guest, it will associate a VMX virtual address
1493  * range with the queue pair as specified by the page_store. At this
1494  * point, the already attach host endpoint may start using the queue
1495  * pair, and an attach event is sent to it. For compatibility with
1496  * older VMX'en, that used a separate step to set the VMX virtual
1497  * address range, the virtual address range can be registered later
1498  * using vmci_qp_broker_set_page_store. In that case, a page_store of
1499  * NULL should be used, and the attach event will be generated once
1500  * the actual page store has been set.
1501  *
1502  * If the attacher is the host, a page_store of NULL should be used as
1503  * well, since the page store information is already set by the guest.
1504  *
1505  * For new VMX and host callers, the queue pair will be moved to the
1506  * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1507  * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1508  */
1509 static int qp_broker_attach(struct qp_broker_entry *entry,
1510 			    u32 peer,
1511 			    u32 flags,
1512 			    u32 priv_flags,
1513 			    u64 produce_size,
1514 			    u64 consume_size,
1515 			    struct vmci_qp_page_store *page_store,
1516 			    struct vmci_ctx *context,
1517 			    vmci_event_release_cb wakeup_cb,
1518 			    void *client_data,
1519 			    struct qp_broker_entry **ent)
1520 {
1521 	const u32 context_id = vmci_ctx_get_id(context);
1522 	bool is_local = flags & VMCI_QPFLAG_LOCAL;
1523 	int result;
1524 
1525 	if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1526 	    entry->state != VMCIQPB_CREATED_MEM)
1527 		return VMCI_ERROR_UNAVAILABLE;
1528 
1529 	if (is_local) {
1530 		if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1531 		    context_id != entry->create_id) {
1532 			return VMCI_ERROR_INVALID_ARGS;
1533 		}
1534 	} else if (context_id == entry->create_id ||
1535 		   context_id == entry->attach_id) {
1536 		return VMCI_ERROR_ALREADY_EXISTS;
1537 	}
1538 
1539 	if (VMCI_CONTEXT_IS_VM(context_id) &&
1540 	    VMCI_CONTEXT_IS_VM(entry->create_id))
1541 		return VMCI_ERROR_DST_UNREACHABLE;
1542 
1543 	/*
1544 	 * If we are attaching from a restricted context then the queuepair
1545 	 * must have been created by a trusted endpoint.
1546 	 */
1547 	if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1548 	    !entry->created_by_trusted)
1549 		return VMCI_ERROR_NO_ACCESS;
1550 
1551 	/*
1552 	 * If we are attaching to a queuepair that was created by a restricted
1553 	 * context then we must be trusted.
1554 	 */
1555 	if (entry->require_trusted_attach &&
1556 	    (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1557 		return VMCI_ERROR_NO_ACCESS;
1558 
1559 	/*
1560 	 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1561 	 * control check is not performed.
1562 	 */
1563 	if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1564 		return VMCI_ERROR_NO_ACCESS;
1565 
1566 	if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1567 		/*
1568 		 * Do not attach if the caller doesn't support Host Queue Pairs
1569 		 * and a host created this queue pair.
1570 		 */
1571 
1572 		if (!vmci_ctx_supports_host_qp(context))
1573 			return VMCI_ERROR_INVALID_RESOURCE;
1574 
1575 	} else if (context_id == VMCI_HOST_CONTEXT_ID) {
1576 		struct vmci_ctx *create_context;
1577 		bool supports_host_qp;
1578 
1579 		/*
1580 		 * Do not attach a host to a user created queue pair if that
1581 		 * user doesn't support host queue pair end points.
1582 		 */
1583 
1584 		create_context = vmci_ctx_get(entry->create_id);
1585 		supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1586 		vmci_ctx_put(create_context);
1587 
1588 		if (!supports_host_qp)
1589 			return VMCI_ERROR_INVALID_RESOURCE;
1590 	}
1591 
1592 	if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1593 		return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1594 
1595 	if (context_id != VMCI_HOST_CONTEXT_ID) {
1596 		/*
1597 		 * The queue pair broker entry stores values from the guest
1598 		 * point of view, so an attaching guest should match the values
1599 		 * stored in the entry.
1600 		 */
1601 
1602 		if (entry->qp.produce_size != produce_size ||
1603 		    entry->qp.consume_size != consume_size) {
1604 			return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1605 		}
1606 	} else if (entry->qp.produce_size != consume_size ||
1607 		   entry->qp.consume_size != produce_size) {
1608 		return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1609 	}
1610 
1611 	if (context_id != VMCI_HOST_CONTEXT_ID) {
1612 		/*
1613 		 * If a guest attached to a queue pair, it will supply
1614 		 * the backing memory.  If this is a pre NOVMVM vmx,
1615 		 * the backing memory will be supplied by calling
1616 		 * vmci_qp_broker_set_page_store() following the
1617 		 * return of the vmci_qp_broker_alloc() call. If it is
1618 		 * a vmx of version NOVMVM or later, the page store
1619 		 * must be supplied as part of the
1620 		 * vmci_qp_broker_alloc call.  Under all circumstances
1621 		 * must the initially created queue pair not have any
1622 		 * memory associated with it already.
1623 		 */
1624 
1625 		if (entry->state != VMCIQPB_CREATED_NO_MEM)
1626 			return VMCI_ERROR_INVALID_ARGS;
1627 
1628 		if (page_store != NULL) {
1629 			/*
1630 			 * Patch up host state to point to guest
1631 			 * supplied memory. The VMX already
1632 			 * initialized the queue pair headers, so no
1633 			 * need for the kernel side to do that.
1634 			 */
1635 
1636 			result = qp_host_register_user_memory(page_store,
1637 							      entry->produce_q,
1638 							      entry->consume_q);
1639 			if (result < VMCI_SUCCESS)
1640 				return result;
1641 
1642 			entry->state = VMCIQPB_ATTACHED_MEM;
1643 		} else {
1644 			entry->state = VMCIQPB_ATTACHED_NO_MEM;
1645 		}
1646 	} else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1647 		/*
1648 		 * The host side is attempting to attach to a queue
1649 		 * pair that doesn't have any memory associated with
1650 		 * it. This must be a pre NOVMVM vmx that hasn't set
1651 		 * the page store information yet, or a quiesced VM.
1652 		 */
1653 
1654 		return VMCI_ERROR_UNAVAILABLE;
1655 	} else {
1656 		/* The host side has successfully attached to a queue pair. */
1657 		entry->state = VMCIQPB_ATTACHED_MEM;
1658 	}
1659 
1660 	if (entry->state == VMCIQPB_ATTACHED_MEM) {
1661 		result =
1662 		    qp_notify_peer(true, entry->qp.handle, context_id,
1663 				   entry->create_id);
1664 		if (result < VMCI_SUCCESS)
1665 			pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1666 				entry->create_id, entry->qp.handle.context,
1667 				entry->qp.handle.resource);
1668 	}
1669 
1670 	entry->attach_id = context_id;
1671 	entry->qp.ref_count++;
1672 	if (wakeup_cb) {
1673 		entry->wakeup_cb = wakeup_cb;
1674 		entry->client_data = client_data;
1675 	}
1676 
1677 	/*
1678 	 * When attaching to local queue pairs, the context already has
1679 	 * an entry tracking the queue pair, so don't add another one.
1680 	 */
1681 	if (!is_local)
1682 		vmci_ctx_qp_create(context, entry->qp.handle);
1683 
1684 	if (ent != NULL)
1685 		*ent = entry;
1686 
1687 	return VMCI_SUCCESS;
1688 }
1689 
1690 /*
1691  * queue_pair_Alloc for use when setting up queue pair endpoints
1692  * on the host.
1693  */
1694 static int qp_broker_alloc(struct vmci_handle handle,
1695 			   u32 peer,
1696 			   u32 flags,
1697 			   u32 priv_flags,
1698 			   u64 produce_size,
1699 			   u64 consume_size,
1700 			   struct vmci_qp_page_store *page_store,
1701 			   struct vmci_ctx *context,
1702 			   vmci_event_release_cb wakeup_cb,
1703 			   void *client_data,
1704 			   struct qp_broker_entry **ent,
1705 			   bool *swap)
1706 {
1707 	const u32 context_id = vmci_ctx_get_id(context);
1708 	bool create;
1709 	struct qp_broker_entry *entry = NULL;
1710 	bool is_local = flags & VMCI_QPFLAG_LOCAL;
1711 	int result;
1712 
1713 	if (vmci_handle_is_invalid(handle) ||
1714 	    (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1715 	    !(produce_size || consume_size) ||
1716 	    !context || context_id == VMCI_INVALID_ID ||
1717 	    handle.context == VMCI_INVALID_ID) {
1718 		return VMCI_ERROR_INVALID_ARGS;
1719 	}
1720 
1721 	if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1722 		return VMCI_ERROR_INVALID_ARGS;
1723 
1724 	/*
1725 	 * In the initial argument check, we ensure that non-vmkernel hosts
1726 	 * are not allowed to create local queue pairs.
1727 	 */
1728 
1729 	mutex_lock(&qp_broker_list.mutex);
1730 
1731 	if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1732 		pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1733 			 context_id, handle.context, handle.resource);
1734 		mutex_unlock(&qp_broker_list.mutex);
1735 		return VMCI_ERROR_ALREADY_EXISTS;
1736 	}
1737 
1738 	if (handle.resource != VMCI_INVALID_ID)
1739 		entry = qp_broker_handle_to_entry(handle);
1740 
1741 	if (!entry) {
1742 		create = true;
1743 		result =
1744 		    qp_broker_create(handle, peer, flags, priv_flags,
1745 				     produce_size, consume_size, page_store,
1746 				     context, wakeup_cb, client_data, ent);
1747 	} else {
1748 		create = false;
1749 		result =
1750 		    qp_broker_attach(entry, peer, flags, priv_flags,
1751 				     produce_size, consume_size, page_store,
1752 				     context, wakeup_cb, client_data, ent);
1753 	}
1754 
1755 	mutex_unlock(&qp_broker_list.mutex);
1756 
1757 	if (swap)
1758 		*swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1759 		    !(create && is_local);
1760 
1761 	return result;
1762 }
1763 
1764 /*
1765  * This function implements the kernel API for allocating a queue
1766  * pair.
1767  */
1768 static int qp_alloc_host_work(struct vmci_handle *handle,
1769 			      struct vmci_queue **produce_q,
1770 			      u64 produce_size,
1771 			      struct vmci_queue **consume_q,
1772 			      u64 consume_size,
1773 			      u32 peer,
1774 			      u32 flags,
1775 			      u32 priv_flags,
1776 			      vmci_event_release_cb wakeup_cb,
1777 			      void *client_data)
1778 {
1779 	struct vmci_handle new_handle;
1780 	struct vmci_ctx *context;
1781 	struct qp_broker_entry *entry;
1782 	int result;
1783 	bool swap;
1784 
1785 	if (vmci_handle_is_invalid(*handle)) {
1786 		new_handle = vmci_make_handle(
1787 			VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1788 	} else
1789 		new_handle = *handle;
1790 
1791 	context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1792 	entry = NULL;
1793 	result =
1794 	    qp_broker_alloc(new_handle, peer, flags, priv_flags,
1795 			    produce_size, consume_size, NULL, context,
1796 			    wakeup_cb, client_data, &entry, &swap);
1797 	if (result == VMCI_SUCCESS) {
1798 		if (swap) {
1799 			/*
1800 			 * If this is a local queue pair, the attacher
1801 			 * will swap around produce and consume
1802 			 * queues.
1803 			 */
1804 
1805 			*produce_q = entry->consume_q;
1806 			*consume_q = entry->produce_q;
1807 		} else {
1808 			*produce_q = entry->produce_q;
1809 			*consume_q = entry->consume_q;
1810 		}
1811 
1812 		*handle = vmci_resource_handle(&entry->resource);
1813 	} else {
1814 		*handle = VMCI_INVALID_HANDLE;
1815 		pr_devel("queue pair broker failed to alloc (result=%d)\n",
1816 			 result);
1817 	}
1818 	vmci_ctx_put(context);
1819 	return result;
1820 }
1821 
1822 /*
1823  * Allocates a VMCI queue_pair. Only checks validity of input
1824  * arguments. The real work is done in the host or guest
1825  * specific function.
1826  */
1827 int vmci_qp_alloc(struct vmci_handle *handle,
1828 		  struct vmci_queue **produce_q,
1829 		  u64 produce_size,
1830 		  struct vmci_queue **consume_q,
1831 		  u64 consume_size,
1832 		  u32 peer,
1833 		  u32 flags,
1834 		  u32 priv_flags,
1835 		  bool guest_endpoint,
1836 		  vmci_event_release_cb wakeup_cb,
1837 		  void *client_data)
1838 {
1839 	if (!handle || !produce_q || !consume_q ||
1840 	    (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1841 		return VMCI_ERROR_INVALID_ARGS;
1842 
1843 	if (guest_endpoint) {
1844 		return qp_alloc_guest_work(handle, produce_q,
1845 					   produce_size, consume_q,
1846 					   consume_size, peer,
1847 					   flags, priv_flags);
1848 	} else {
1849 		return qp_alloc_host_work(handle, produce_q,
1850 					  produce_size, consume_q,
1851 					  consume_size, peer, flags,
1852 					  priv_flags, wakeup_cb, client_data);
1853 	}
1854 }
1855 
1856 /*
1857  * This function implements the host kernel API for detaching from
1858  * a queue pair.
1859  */
1860 static int qp_detatch_host_work(struct vmci_handle handle)
1861 {
1862 	int result;
1863 	struct vmci_ctx *context;
1864 
1865 	context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1866 
1867 	result = vmci_qp_broker_detach(handle, context);
1868 
1869 	vmci_ctx_put(context);
1870 	return result;
1871 }
1872 
1873 /*
1874  * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1875  * Real work is done in the host or guest specific function.
1876  */
1877 static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1878 {
1879 	if (vmci_handle_is_invalid(handle))
1880 		return VMCI_ERROR_INVALID_ARGS;
1881 
1882 	if (guest_endpoint)
1883 		return qp_detatch_guest_work(handle);
1884 	else
1885 		return qp_detatch_host_work(handle);
1886 }
1887 
1888 /*
1889  * Returns the entry from the head of the list. Assumes that the list is
1890  * locked.
1891  */
1892 static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1893 {
1894 	if (!list_empty(&qp_list->head)) {
1895 		struct qp_entry *entry =
1896 		    list_first_entry(&qp_list->head, struct qp_entry,
1897 				     list_item);
1898 		return entry;
1899 	}
1900 
1901 	return NULL;
1902 }
1903 
1904 void vmci_qp_broker_exit(void)
1905 {
1906 	struct qp_entry *entry;
1907 	struct qp_broker_entry *be;
1908 
1909 	mutex_lock(&qp_broker_list.mutex);
1910 
1911 	while ((entry = qp_list_get_head(&qp_broker_list))) {
1912 		be = (struct qp_broker_entry *)entry;
1913 
1914 		qp_list_remove_entry(&qp_broker_list, entry);
1915 		kfree(be);
1916 	}
1917 
1918 	mutex_unlock(&qp_broker_list.mutex);
1919 }
1920 
1921 /*
1922  * Requests that a queue pair be allocated with the VMCI queue
1923  * pair broker. Allocates a queue pair entry if one does not
1924  * exist. Attaches to one if it exists, and retrieves the page
1925  * files backing that queue_pair.  Assumes that the queue pair
1926  * broker lock is held.
1927  */
1928 int vmci_qp_broker_alloc(struct vmci_handle handle,
1929 			 u32 peer,
1930 			 u32 flags,
1931 			 u32 priv_flags,
1932 			 u64 produce_size,
1933 			 u64 consume_size,
1934 			 struct vmci_qp_page_store *page_store,
1935 			 struct vmci_ctx *context)
1936 {
1937 	if (!QP_SIZES_ARE_VALID(produce_size, consume_size))
1938 		return VMCI_ERROR_NO_RESOURCES;
1939 
1940 	return qp_broker_alloc(handle, peer, flags, priv_flags,
1941 			       produce_size, consume_size,
1942 			       page_store, context, NULL, NULL, NULL, NULL);
1943 }
1944 
1945 /*
1946  * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1947  * step to add the UVAs of the VMX mapping of the queue pair. This function
1948  * provides backwards compatibility with such VMX'en, and takes care of
1949  * registering the page store for a queue pair previously allocated by the
1950  * VMX during create or attach. This function will move the queue pair state
1951  * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1952  * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1953  * attached state with memory, the queue pair is ready to be used by the
1954  * host peer, and an attached event will be generated.
1955  *
1956  * Assumes that the queue pair broker lock is held.
1957  *
1958  * This function is only used by the hosted platform, since there is no
1959  * issue with backwards compatibility for vmkernel.
1960  */
1961 int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1962 				  u64 produce_uva,
1963 				  u64 consume_uva,
1964 				  struct vmci_ctx *context)
1965 {
1966 	struct qp_broker_entry *entry;
1967 	int result;
1968 	const u32 context_id = vmci_ctx_get_id(context);
1969 
1970 	if (vmci_handle_is_invalid(handle) || !context ||
1971 	    context_id == VMCI_INVALID_ID)
1972 		return VMCI_ERROR_INVALID_ARGS;
1973 
1974 	/*
1975 	 * We only support guest to host queue pairs, so the VMX must
1976 	 * supply UVAs for the mapped page files.
1977 	 */
1978 
1979 	if (produce_uva == 0 || consume_uva == 0)
1980 		return VMCI_ERROR_INVALID_ARGS;
1981 
1982 	mutex_lock(&qp_broker_list.mutex);
1983 
1984 	if (!vmci_ctx_qp_exists(context, handle)) {
1985 		pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1986 			context_id, handle.context, handle.resource);
1987 		result = VMCI_ERROR_NOT_FOUND;
1988 		goto out;
1989 	}
1990 
1991 	entry = qp_broker_handle_to_entry(handle);
1992 	if (!entry) {
1993 		result = VMCI_ERROR_NOT_FOUND;
1994 		goto out;
1995 	}
1996 
1997 	/*
1998 	 * If I'm the owner then I can set the page store.
1999 	 *
2000 	 * Or, if a host created the queue_pair and I'm the attached peer
2001 	 * then I can set the page store.
2002 	 */
2003 	if (entry->create_id != context_id &&
2004 	    (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2005 	     entry->attach_id != context_id)) {
2006 		result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2007 		goto out;
2008 	}
2009 
2010 	if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2011 	    entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2012 		result = VMCI_ERROR_UNAVAILABLE;
2013 		goto out;
2014 	}
2015 
2016 	result = qp_host_get_user_memory(produce_uva, consume_uva,
2017 					 entry->produce_q, entry->consume_q);
2018 	if (result < VMCI_SUCCESS)
2019 		goto out;
2020 
2021 	result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2022 	if (result < VMCI_SUCCESS) {
2023 		qp_host_unregister_user_memory(entry->produce_q,
2024 					       entry->consume_q);
2025 		goto out;
2026 	}
2027 
2028 	if (entry->state == VMCIQPB_CREATED_NO_MEM)
2029 		entry->state = VMCIQPB_CREATED_MEM;
2030 	else
2031 		entry->state = VMCIQPB_ATTACHED_MEM;
2032 
2033 	entry->vmci_page_files = true;
2034 
2035 	if (entry->state == VMCIQPB_ATTACHED_MEM) {
2036 		result =
2037 		    qp_notify_peer(true, handle, context_id, entry->create_id);
2038 		if (result < VMCI_SUCCESS) {
2039 			pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2040 				entry->create_id, entry->qp.handle.context,
2041 				entry->qp.handle.resource);
2042 		}
2043 	}
2044 
2045 	result = VMCI_SUCCESS;
2046  out:
2047 	mutex_unlock(&qp_broker_list.mutex);
2048 	return result;
2049 }
2050 
2051 /*
2052  * Resets saved queue headers for the given QP broker
2053  * entry. Should be used when guest memory becomes available
2054  * again, or the guest detaches.
2055  */
2056 static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2057 {
2058 	entry->produce_q->saved_header = NULL;
2059 	entry->consume_q->saved_header = NULL;
2060 }
2061 
2062 /*
2063  * The main entry point for detaching from a queue pair registered with the
2064  * queue pair broker. If more than one endpoint is attached to the queue
2065  * pair, the first endpoint will mainly decrement a reference count and
2066  * generate a notification to its peer. The last endpoint will clean up
2067  * the queue pair state registered with the broker.
2068  *
2069  * When a guest endpoint detaches, it will unmap and unregister the guest
2070  * memory backing the queue pair. If the host is still attached, it will
2071  * no longer be able to access the queue pair content.
2072  *
2073  * If the queue pair is already in a state where there is no memory
2074  * registered for the queue pair (any *_NO_MEM state), it will transition to
2075  * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2076  * endpoint is the first of two endpoints to detach. If the host endpoint is
2077  * the first out of two to detach, the queue pair will move to the
2078  * VMCIQPB_SHUTDOWN_MEM state.
2079  */
2080 int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2081 {
2082 	struct qp_broker_entry *entry;
2083 	const u32 context_id = vmci_ctx_get_id(context);
2084 	u32 peer_id;
2085 	bool is_local = false;
2086 	int result;
2087 
2088 	if (vmci_handle_is_invalid(handle) || !context ||
2089 	    context_id == VMCI_INVALID_ID) {
2090 		return VMCI_ERROR_INVALID_ARGS;
2091 	}
2092 
2093 	mutex_lock(&qp_broker_list.mutex);
2094 
2095 	if (!vmci_ctx_qp_exists(context, handle)) {
2096 		pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2097 			 context_id, handle.context, handle.resource);
2098 		result = VMCI_ERROR_NOT_FOUND;
2099 		goto out;
2100 	}
2101 
2102 	entry = qp_broker_handle_to_entry(handle);
2103 	if (!entry) {
2104 		pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2105 			 context_id, handle.context, handle.resource);
2106 		result = VMCI_ERROR_NOT_FOUND;
2107 		goto out;
2108 	}
2109 
2110 	if (context_id != entry->create_id && context_id != entry->attach_id) {
2111 		result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2112 		goto out;
2113 	}
2114 
2115 	if (context_id == entry->create_id) {
2116 		peer_id = entry->attach_id;
2117 		entry->create_id = VMCI_INVALID_ID;
2118 	} else {
2119 		peer_id = entry->create_id;
2120 		entry->attach_id = VMCI_INVALID_ID;
2121 	}
2122 	entry->qp.ref_count--;
2123 
2124 	is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2125 
2126 	if (context_id != VMCI_HOST_CONTEXT_ID) {
2127 		bool headers_mapped;
2128 
2129 		/*
2130 		 * Pre NOVMVM vmx'en may detach from a queue pair
2131 		 * before setting the page store, and in that case
2132 		 * there is no user memory to detach from. Also, more
2133 		 * recent VMX'en may detach from a queue pair in the
2134 		 * quiesced state.
2135 		 */
2136 
2137 		qp_acquire_queue_mutex(entry->produce_q);
2138 		headers_mapped = entry->produce_q->q_header ||
2139 		    entry->consume_q->q_header;
2140 		if (QPBROKERSTATE_HAS_MEM(entry)) {
2141 			result =
2142 			    qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2143 						 entry->produce_q,
2144 						 entry->consume_q);
2145 			if (result < VMCI_SUCCESS)
2146 				pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2147 					handle.context, handle.resource,
2148 					result);
2149 
2150 			qp_host_unregister_user_memory(entry->produce_q,
2151 						       entry->consume_q);
2152 
2153 		}
2154 
2155 		if (!headers_mapped)
2156 			qp_reset_saved_headers(entry);
2157 
2158 		qp_release_queue_mutex(entry->produce_q);
2159 
2160 		if (!headers_mapped && entry->wakeup_cb)
2161 			entry->wakeup_cb(entry->client_data);
2162 
2163 	} else {
2164 		if (entry->wakeup_cb) {
2165 			entry->wakeup_cb = NULL;
2166 			entry->client_data = NULL;
2167 		}
2168 	}
2169 
2170 	if (entry->qp.ref_count == 0) {
2171 		qp_list_remove_entry(&qp_broker_list, &entry->qp);
2172 
2173 		if (is_local)
2174 			kfree(entry->local_mem);
2175 
2176 		qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2177 		qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2178 		qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2179 		/* Unlink from resource hash table and free callback */
2180 		vmci_resource_remove(&entry->resource);
2181 
2182 		kfree(entry);
2183 
2184 		vmci_ctx_qp_destroy(context, handle);
2185 	} else {
2186 		qp_notify_peer(false, handle, context_id, peer_id);
2187 		if (context_id == VMCI_HOST_CONTEXT_ID &&
2188 		    QPBROKERSTATE_HAS_MEM(entry)) {
2189 			entry->state = VMCIQPB_SHUTDOWN_MEM;
2190 		} else {
2191 			entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2192 		}
2193 
2194 		if (!is_local)
2195 			vmci_ctx_qp_destroy(context, handle);
2196 
2197 	}
2198 	result = VMCI_SUCCESS;
2199  out:
2200 	mutex_unlock(&qp_broker_list.mutex);
2201 	return result;
2202 }
2203 
2204 /*
2205  * Establishes the necessary mappings for a queue pair given a
2206  * reference to the queue pair guest memory. This is usually
2207  * called when a guest is unquiesced and the VMX is allowed to
2208  * map guest memory once again.
2209  */
2210 int vmci_qp_broker_map(struct vmci_handle handle,
2211 		       struct vmci_ctx *context,
2212 		       u64 guest_mem)
2213 {
2214 	struct qp_broker_entry *entry;
2215 	const u32 context_id = vmci_ctx_get_id(context);
2216 	int result;
2217 
2218 	if (vmci_handle_is_invalid(handle) || !context ||
2219 	    context_id == VMCI_INVALID_ID)
2220 		return VMCI_ERROR_INVALID_ARGS;
2221 
2222 	mutex_lock(&qp_broker_list.mutex);
2223 
2224 	if (!vmci_ctx_qp_exists(context, handle)) {
2225 		pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2226 			 context_id, handle.context, handle.resource);
2227 		result = VMCI_ERROR_NOT_FOUND;
2228 		goto out;
2229 	}
2230 
2231 	entry = qp_broker_handle_to_entry(handle);
2232 	if (!entry) {
2233 		pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2234 			 context_id, handle.context, handle.resource);
2235 		result = VMCI_ERROR_NOT_FOUND;
2236 		goto out;
2237 	}
2238 
2239 	if (context_id != entry->create_id && context_id != entry->attach_id) {
2240 		result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2241 		goto out;
2242 	}
2243 
2244 	result = VMCI_SUCCESS;
2245 
2246 	if (context_id != VMCI_HOST_CONTEXT_ID &&
2247 	    !QPBROKERSTATE_HAS_MEM(entry)) {
2248 		struct vmci_qp_page_store page_store;
2249 
2250 		page_store.pages = guest_mem;
2251 		page_store.len = QPE_NUM_PAGES(entry->qp);
2252 
2253 		qp_acquire_queue_mutex(entry->produce_q);
2254 		qp_reset_saved_headers(entry);
2255 		result =
2256 		    qp_host_register_user_memory(&page_store,
2257 						 entry->produce_q,
2258 						 entry->consume_q);
2259 		qp_release_queue_mutex(entry->produce_q);
2260 		if (result == VMCI_SUCCESS) {
2261 			/* Move state from *_NO_MEM to *_MEM */
2262 
2263 			entry->state++;
2264 
2265 			if (entry->wakeup_cb)
2266 				entry->wakeup_cb(entry->client_data);
2267 		}
2268 	}
2269 
2270  out:
2271 	mutex_unlock(&qp_broker_list.mutex);
2272 	return result;
2273 }
2274 
2275 /*
2276  * Saves a snapshot of the queue headers for the given QP broker
2277  * entry. Should be used when guest memory is unmapped.
2278  * Results:
2279  * VMCI_SUCCESS on success, appropriate error code if guest memory
2280  * can't be accessed..
2281  */
2282 static int qp_save_headers(struct qp_broker_entry *entry)
2283 {
2284 	int result;
2285 
2286 	if (entry->produce_q->saved_header != NULL &&
2287 	    entry->consume_q->saved_header != NULL) {
2288 		/*
2289 		 *  If the headers have already been saved, we don't need to do
2290 		 *  it again, and we don't want to map in the headers
2291 		 *  unnecessarily.
2292 		 */
2293 
2294 		return VMCI_SUCCESS;
2295 	}
2296 
2297 	if (NULL == entry->produce_q->q_header ||
2298 	    NULL == entry->consume_q->q_header) {
2299 		result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2300 		if (result < VMCI_SUCCESS)
2301 			return result;
2302 	}
2303 
2304 	memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2305 	       sizeof(entry->saved_produce_q));
2306 	entry->produce_q->saved_header = &entry->saved_produce_q;
2307 	memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2308 	       sizeof(entry->saved_consume_q));
2309 	entry->consume_q->saved_header = &entry->saved_consume_q;
2310 
2311 	return VMCI_SUCCESS;
2312 }
2313 
2314 /*
2315  * Removes all references to the guest memory of a given queue pair, and
2316  * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2317  * called when a VM is being quiesced where access to guest memory should
2318  * avoided.
2319  */
2320 int vmci_qp_broker_unmap(struct vmci_handle handle,
2321 			 struct vmci_ctx *context,
2322 			 u32 gid)
2323 {
2324 	struct qp_broker_entry *entry;
2325 	const u32 context_id = vmci_ctx_get_id(context);
2326 	int result;
2327 
2328 	if (vmci_handle_is_invalid(handle) || !context ||
2329 	    context_id == VMCI_INVALID_ID)
2330 		return VMCI_ERROR_INVALID_ARGS;
2331 
2332 	mutex_lock(&qp_broker_list.mutex);
2333 
2334 	if (!vmci_ctx_qp_exists(context, handle)) {
2335 		pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2336 			 context_id, handle.context, handle.resource);
2337 		result = VMCI_ERROR_NOT_FOUND;
2338 		goto out;
2339 	}
2340 
2341 	entry = qp_broker_handle_to_entry(handle);
2342 	if (!entry) {
2343 		pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2344 			 context_id, handle.context, handle.resource);
2345 		result = VMCI_ERROR_NOT_FOUND;
2346 		goto out;
2347 	}
2348 
2349 	if (context_id != entry->create_id && context_id != entry->attach_id) {
2350 		result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2351 		goto out;
2352 	}
2353 
2354 	if (context_id != VMCI_HOST_CONTEXT_ID &&
2355 	    QPBROKERSTATE_HAS_MEM(entry)) {
2356 		qp_acquire_queue_mutex(entry->produce_q);
2357 		result = qp_save_headers(entry);
2358 		if (result < VMCI_SUCCESS)
2359 			pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2360 				handle.context, handle.resource, result);
2361 
2362 		qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2363 
2364 		/*
2365 		 * On hosted, when we unmap queue pairs, the VMX will also
2366 		 * unmap the guest memory, so we invalidate the previously
2367 		 * registered memory. If the queue pair is mapped again at a
2368 		 * later point in time, we will need to reregister the user
2369 		 * memory with a possibly new user VA.
2370 		 */
2371 		qp_host_unregister_user_memory(entry->produce_q,
2372 					       entry->consume_q);
2373 
2374 		/*
2375 		 * Move state from *_MEM to *_NO_MEM.
2376 		 */
2377 		entry->state--;
2378 
2379 		qp_release_queue_mutex(entry->produce_q);
2380 	}
2381 
2382 	result = VMCI_SUCCESS;
2383 
2384  out:
2385 	mutex_unlock(&qp_broker_list.mutex);
2386 	return result;
2387 }
2388 
2389 /*
2390  * Destroys all guest queue pair endpoints. If active guest queue
2391  * pairs still exist, hypercalls to attempt detach from these
2392  * queue pairs will be made. Any failure to detach is silently
2393  * ignored.
2394  */
2395 void vmci_qp_guest_endpoints_exit(void)
2396 {
2397 	struct qp_entry *entry;
2398 	struct qp_guest_endpoint *ep;
2399 
2400 	mutex_lock(&qp_guest_endpoints.mutex);
2401 
2402 	while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2403 		ep = (struct qp_guest_endpoint *)entry;
2404 
2405 		/* Don't make a hypercall for local queue_pairs. */
2406 		if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2407 			qp_detatch_hypercall(entry->handle);
2408 
2409 		/* We cannot fail the exit, so let's reset ref_count. */
2410 		entry->ref_count = 0;
2411 		qp_list_remove_entry(&qp_guest_endpoints, entry);
2412 
2413 		qp_guest_endpoint_destroy(ep);
2414 	}
2415 
2416 	mutex_unlock(&qp_guest_endpoints.mutex);
2417 }
2418 
2419 /*
2420  * Helper routine that will lock the queue pair before subsequent
2421  * operations.
2422  * Note: Non-blocking on the host side is currently only implemented in ESX.
2423  * Since non-blocking isn't yet implemented on the host personality we
2424  * have no reason to acquire a spin lock.  So to avoid the use of an
2425  * unnecessary lock only acquire the mutex if we can block.
2426  */
2427 static void qp_lock(const struct vmci_qp *qpair)
2428 {
2429 	qp_acquire_queue_mutex(qpair->produce_q);
2430 }
2431 
2432 /*
2433  * Helper routine that unlocks the queue pair after calling
2434  * qp_lock.
2435  */
2436 static void qp_unlock(const struct vmci_qp *qpair)
2437 {
2438 	qp_release_queue_mutex(qpair->produce_q);
2439 }
2440 
2441 /*
2442  * The queue headers may not be mapped at all times. If a queue is
2443  * currently not mapped, it will be attempted to do so.
2444  */
2445 static int qp_map_queue_headers(struct vmci_queue *produce_q,
2446 				struct vmci_queue *consume_q)
2447 {
2448 	int result;
2449 
2450 	if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2451 		result = qp_host_map_queues(produce_q, consume_q);
2452 		if (result < VMCI_SUCCESS)
2453 			return (produce_q->saved_header &&
2454 				consume_q->saved_header) ?
2455 			    VMCI_ERROR_QUEUEPAIR_NOT_READY :
2456 			    VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2457 	}
2458 
2459 	return VMCI_SUCCESS;
2460 }
2461 
2462 /*
2463  * Helper routine that will retrieve the produce and consume
2464  * headers of a given queue pair. If the guest memory of the
2465  * queue pair is currently not available, the saved queue headers
2466  * will be returned, if these are available.
2467  */
2468 static int qp_get_queue_headers(const struct vmci_qp *qpair,
2469 				struct vmci_queue_header **produce_q_header,
2470 				struct vmci_queue_header **consume_q_header)
2471 {
2472 	int result;
2473 
2474 	result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2475 	if (result == VMCI_SUCCESS) {
2476 		*produce_q_header = qpair->produce_q->q_header;
2477 		*consume_q_header = qpair->consume_q->q_header;
2478 	} else if (qpair->produce_q->saved_header &&
2479 		   qpair->consume_q->saved_header) {
2480 		*produce_q_header = qpair->produce_q->saved_header;
2481 		*consume_q_header = qpair->consume_q->saved_header;
2482 		result = VMCI_SUCCESS;
2483 	}
2484 
2485 	return result;
2486 }
2487 
2488 /*
2489  * Callback from VMCI queue pair broker indicating that a queue
2490  * pair that was previously not ready, now either is ready or
2491  * gone forever.
2492  */
2493 static int qp_wakeup_cb(void *client_data)
2494 {
2495 	struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2496 
2497 	qp_lock(qpair);
2498 	while (qpair->blocked > 0) {
2499 		qpair->blocked--;
2500 		qpair->generation++;
2501 		wake_up(&qpair->event);
2502 	}
2503 	qp_unlock(qpair);
2504 
2505 	return VMCI_SUCCESS;
2506 }
2507 
2508 /*
2509  * Makes the calling thread wait for the queue pair to become
2510  * ready for host side access.  Returns true when thread is
2511  * woken up after queue pair state change, false otherwise.
2512  */
2513 static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2514 {
2515 	unsigned int generation;
2516 
2517 	qpair->blocked++;
2518 	generation = qpair->generation;
2519 	qp_unlock(qpair);
2520 	wait_event(qpair->event, generation != qpair->generation);
2521 	qp_lock(qpair);
2522 
2523 	return true;
2524 }
2525 
2526 /*
2527  * Enqueues a given buffer to the produce queue using the provided
2528  * function. As many bytes as possible (space available in the queue)
2529  * are enqueued.  Assumes the queue->mutex has been acquired.  Returns
2530  * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2531  * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2532  * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2533  * an error occurred when accessing the buffer,
2534  * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2535  * available.  Otherwise, the number of bytes written to the queue is
2536  * returned.  Updates the tail pointer of the produce queue.
2537  */
2538 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2539 				 struct vmci_queue *consume_q,
2540 				 const u64 produce_q_size,
2541 				 struct iov_iter *from)
2542 {
2543 	s64 free_space;
2544 	u64 tail;
2545 	size_t buf_size = iov_iter_count(from);
2546 	size_t written;
2547 	ssize_t result;
2548 
2549 	result = qp_map_queue_headers(produce_q, consume_q);
2550 	if (unlikely(result != VMCI_SUCCESS))
2551 		return result;
2552 
2553 	free_space = vmci_q_header_free_space(produce_q->q_header,
2554 					      consume_q->q_header,
2555 					      produce_q_size);
2556 	if (free_space == 0)
2557 		return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2558 
2559 	if (free_space < VMCI_SUCCESS)
2560 		return (ssize_t) free_space;
2561 
2562 	written = (size_t) (free_space > buf_size ? buf_size : free_space);
2563 	tail = vmci_q_header_producer_tail(produce_q->q_header);
2564 	if (likely(tail + written < produce_q_size)) {
2565 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2566 	} else {
2567 		/* Tail pointer wraps around. */
2568 
2569 		const size_t tmp = (size_t) (produce_q_size - tail);
2570 
2571 		result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2572 		if (result >= VMCI_SUCCESS)
2573 			result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2574 						 written - tmp);
2575 	}
2576 
2577 	if (result < VMCI_SUCCESS)
2578 		return result;
2579 
2580 	/*
2581 	 * This virt_wmb() ensures that data written to the queue
2582 	 * is observable before the new producer_tail is.
2583 	 */
2584 	virt_wmb();
2585 
2586 	vmci_q_header_add_producer_tail(produce_q->q_header, written,
2587 					produce_q_size);
2588 	return written;
2589 }
2590 
2591 /*
2592  * Dequeues data (if available) from the given consume queue. Writes data
2593  * to the user provided buffer using the provided function.
2594  * Assumes the queue->mutex has been acquired.
2595  * Results:
2596  * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2597  * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2598  * (as defined by the queue size).
2599  * VMCI_ERROR_INVALID_ARGS, if an error occurred when accessing the buffer.
2600  * Otherwise the number of bytes dequeued is returned.
2601  * Side effects:
2602  * Updates the head pointer of the consume queue.
2603  */
2604 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2605 				 struct vmci_queue *consume_q,
2606 				 const u64 consume_q_size,
2607 				 struct iov_iter *to,
2608 				 bool update_consumer)
2609 {
2610 	size_t buf_size = iov_iter_count(to);
2611 	s64 buf_ready;
2612 	u64 head;
2613 	size_t read;
2614 	ssize_t result;
2615 
2616 	result = qp_map_queue_headers(produce_q, consume_q);
2617 	if (unlikely(result != VMCI_SUCCESS))
2618 		return result;
2619 
2620 	buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2621 					    produce_q->q_header,
2622 					    consume_q_size);
2623 	if (buf_ready == 0)
2624 		return VMCI_ERROR_QUEUEPAIR_NODATA;
2625 
2626 	if (buf_ready < VMCI_SUCCESS)
2627 		return (ssize_t) buf_ready;
2628 
2629 	/*
2630 	 * This virt_rmb() ensures that data from the queue will be read
2631 	 * after we have determined how much is ready to be consumed.
2632 	 */
2633 	virt_rmb();
2634 
2635 	read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2636 	head = vmci_q_header_consumer_head(produce_q->q_header);
2637 	if (likely(head + read < consume_q_size)) {
2638 		result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2639 	} else {
2640 		/* Head pointer wraps around. */
2641 
2642 		const size_t tmp = (size_t) (consume_q_size - head);
2643 
2644 		result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2645 		if (result >= VMCI_SUCCESS)
2646 			result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2647 						   read - tmp);
2648 
2649 	}
2650 
2651 	if (result < VMCI_SUCCESS)
2652 		return result;
2653 
2654 	if (update_consumer)
2655 		vmci_q_header_add_consumer_head(produce_q->q_header,
2656 						read, consume_q_size);
2657 
2658 	return read;
2659 }
2660 
2661 /*
2662  * vmci_qpair_alloc() - Allocates a queue pair.
2663  * @qpair:      Pointer for the new vmci_qp struct.
2664  * @handle:     Handle to track the resource.
2665  * @produce_qsize:      Desired size of the producer queue.
2666  * @consume_qsize:      Desired size of the consumer queue.
2667  * @peer:       ContextID of the peer.
2668  * @flags:      VMCI flags.
2669  * @priv_flags: VMCI privilege flags.
2670  *
2671  * This is the client interface for allocating the memory for a
2672  * vmci_qp structure and then attaching to the underlying
2673  * queue.  If an error occurs allocating the memory for the
2674  * vmci_qp structure no attempt is made to attach.  If an
2675  * error occurs attaching, then the structure is freed.
2676  */
2677 int vmci_qpair_alloc(struct vmci_qp **qpair,
2678 		     struct vmci_handle *handle,
2679 		     u64 produce_qsize,
2680 		     u64 consume_qsize,
2681 		     u32 peer,
2682 		     u32 flags,
2683 		     u32 priv_flags)
2684 {
2685 	struct vmci_qp *my_qpair;
2686 	int retval;
2687 	struct vmci_handle src = VMCI_INVALID_HANDLE;
2688 	struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2689 	enum vmci_route route;
2690 	vmci_event_release_cb wakeup_cb;
2691 	void *client_data;
2692 
2693 	/*
2694 	 * Restrict the size of a queuepair.  The device already
2695 	 * enforces a limit on the total amount of memory that can be
2696 	 * allocated to queuepairs for a guest.  However, we try to
2697 	 * allocate this memory before we make the queuepair
2698 	 * allocation hypercall.  On Linux, we allocate each page
2699 	 * separately, which means rather than fail, the guest will
2700 	 * thrash while it tries to allocate, and will become
2701 	 * increasingly unresponsive to the point where it appears to
2702 	 * be hung.  So we place a limit on the size of an individual
2703 	 * queuepair here, and leave the device to enforce the
2704 	 * restriction on total queuepair memory.  (Note that this
2705 	 * doesn't prevent all cases; a user with only this much
2706 	 * physical memory could still get into trouble.)  The error
2707 	 * used by the device is NO_RESOURCES, so use that here too.
2708 	 */
2709 
2710 	if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize))
2711 		return VMCI_ERROR_NO_RESOURCES;
2712 
2713 	retval = vmci_route(&src, &dst, false, &route);
2714 	if (retval < VMCI_SUCCESS)
2715 		route = vmci_guest_code_active() ?
2716 		    VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2717 
2718 	if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2719 		pr_devel("NONBLOCK OR PINNED set");
2720 		return VMCI_ERROR_INVALID_ARGS;
2721 	}
2722 
2723 	my_qpair = kzalloc_obj(*my_qpair);
2724 	if (!my_qpair)
2725 		return VMCI_ERROR_NO_MEM;
2726 
2727 	my_qpair->produce_q_size = produce_qsize;
2728 	my_qpair->consume_q_size = consume_qsize;
2729 	my_qpair->peer = peer;
2730 	my_qpair->flags = flags;
2731 	my_qpair->priv_flags = priv_flags;
2732 
2733 	wakeup_cb = NULL;
2734 	client_data = NULL;
2735 
2736 	if (VMCI_ROUTE_AS_HOST == route) {
2737 		my_qpair->guest_endpoint = false;
2738 		if (!(flags & VMCI_QPFLAG_LOCAL)) {
2739 			my_qpair->blocked = 0;
2740 			my_qpair->generation = 0;
2741 			init_waitqueue_head(&my_qpair->event);
2742 			wakeup_cb = qp_wakeup_cb;
2743 			client_data = (void *)my_qpair;
2744 		}
2745 	} else {
2746 		my_qpair->guest_endpoint = true;
2747 	}
2748 
2749 	retval = vmci_qp_alloc(handle,
2750 			       &my_qpair->produce_q,
2751 			       my_qpair->produce_q_size,
2752 			       &my_qpair->consume_q,
2753 			       my_qpair->consume_q_size,
2754 			       my_qpair->peer,
2755 			       my_qpair->flags,
2756 			       my_qpair->priv_flags,
2757 			       my_qpair->guest_endpoint,
2758 			       wakeup_cb, client_data);
2759 
2760 	if (retval < VMCI_SUCCESS) {
2761 		kfree(my_qpair);
2762 		return retval;
2763 	}
2764 
2765 	*qpair = my_qpair;
2766 	my_qpair->handle = *handle;
2767 
2768 	return retval;
2769 }
2770 EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2771 
2772 /*
2773  * vmci_qpair_detach() - Detatches the client from a queue pair.
2774  * @qpair:      Reference of a pointer to the qpair struct.
2775  *
2776  * This is the client interface for detaching from a VMCIQPair.
2777  * Note that this routine will free the memory allocated for the
2778  * vmci_qp structure too.
2779  */
2780 int vmci_qpair_detach(struct vmci_qp **qpair)
2781 {
2782 	int result;
2783 	struct vmci_qp *old_qpair;
2784 
2785 	if (!qpair || !(*qpair))
2786 		return VMCI_ERROR_INVALID_ARGS;
2787 
2788 	old_qpair = *qpair;
2789 	result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2790 
2791 	/*
2792 	 * The guest can fail to detach for a number of reasons, and
2793 	 * if it does so, it will cleanup the entry (if there is one).
2794 	 * The host can fail too, but it won't cleanup the entry
2795 	 * immediately, it will do that later when the context is
2796 	 * freed.  Either way, we need to release the qpair struct
2797 	 * here; there isn't much the caller can do, and we don't want
2798 	 * to leak.
2799 	 */
2800 
2801 	memset(old_qpair, 0, sizeof(*old_qpair));
2802 	old_qpair->handle = VMCI_INVALID_HANDLE;
2803 	old_qpair->peer = VMCI_INVALID_ID;
2804 	kfree(old_qpair);
2805 	*qpair = NULL;
2806 
2807 	return result;
2808 }
2809 EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2810 
2811 /*
2812  * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2813  * @qpair:      Pointer to the queue pair struct.
2814  * @producer_tail:      Reference used for storing producer tail index.
2815  * @consumer_head:      Reference used for storing the consumer head index.
2816  *
2817  * This is the client interface for getting the current indexes of the
2818  * QPair from the point of the view of the caller as the producer.
2819  */
2820 int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2821 				   u64 *producer_tail,
2822 				   u64 *consumer_head)
2823 {
2824 	struct vmci_queue_header *produce_q_header;
2825 	struct vmci_queue_header *consume_q_header;
2826 	int result;
2827 
2828 	if (!qpair)
2829 		return VMCI_ERROR_INVALID_ARGS;
2830 
2831 	qp_lock(qpair);
2832 	result =
2833 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2834 	if (result == VMCI_SUCCESS)
2835 		vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2836 					   producer_tail, consumer_head);
2837 	qp_unlock(qpair);
2838 
2839 	if (result == VMCI_SUCCESS &&
2840 	    ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2841 	     (consumer_head && *consumer_head >= qpair->produce_q_size)))
2842 		return VMCI_ERROR_INVALID_SIZE;
2843 
2844 	return result;
2845 }
2846 EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2847 
2848 /*
2849  * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2850  * @qpair:      Pointer to the queue pair struct.
2851  * @consumer_tail:      Reference used for storing consumer tail index.
2852  * @producer_head:      Reference used for storing the producer head index.
2853  *
2854  * This is the client interface for getting the current indexes of the
2855  * QPair from the point of the view of the caller as the consumer.
2856  */
2857 int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2858 				   u64 *consumer_tail,
2859 				   u64 *producer_head)
2860 {
2861 	struct vmci_queue_header *produce_q_header;
2862 	struct vmci_queue_header *consume_q_header;
2863 	int result;
2864 
2865 	if (!qpair)
2866 		return VMCI_ERROR_INVALID_ARGS;
2867 
2868 	qp_lock(qpair);
2869 	result =
2870 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2871 	if (result == VMCI_SUCCESS)
2872 		vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2873 					   consumer_tail, producer_head);
2874 	qp_unlock(qpair);
2875 
2876 	if (result == VMCI_SUCCESS &&
2877 	    ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2878 	     (producer_head && *producer_head >= qpair->consume_q_size)))
2879 		return VMCI_ERROR_INVALID_SIZE;
2880 
2881 	return result;
2882 }
2883 EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2884 
2885 /*
2886  * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2887  * @qpair:      Pointer to the queue pair struct.
2888  *
2889  * This is the client interface for getting the amount of free
2890  * space in the QPair from the point of the view of the caller as
2891  * the producer which is the common case.  Returns < 0 if err, else
2892  * available bytes into which data can be enqueued if > 0.
2893  */
2894 s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2895 {
2896 	struct vmci_queue_header *produce_q_header;
2897 	struct vmci_queue_header *consume_q_header;
2898 	s64 result;
2899 
2900 	if (!qpair)
2901 		return VMCI_ERROR_INVALID_ARGS;
2902 
2903 	qp_lock(qpair);
2904 	result =
2905 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2906 	if (result == VMCI_SUCCESS)
2907 		result = vmci_q_header_free_space(produce_q_header,
2908 						  consume_q_header,
2909 						  qpair->produce_q_size);
2910 	else
2911 		result = 0;
2912 
2913 	qp_unlock(qpair);
2914 
2915 	return result;
2916 }
2917 EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2918 
2919 /*
2920  * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2921  * @qpair:      Pointer to the queue pair struct.
2922  *
2923  * This is the client interface for getting the amount of free
2924  * space in the QPair from the point of the view of the caller as
2925  * the consumer which is not the common case.  Returns < 0 if err, else
2926  * available bytes into which data can be enqueued if > 0.
2927  */
2928 s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2929 {
2930 	struct vmci_queue_header *produce_q_header;
2931 	struct vmci_queue_header *consume_q_header;
2932 	s64 result;
2933 
2934 	if (!qpair)
2935 		return VMCI_ERROR_INVALID_ARGS;
2936 
2937 	qp_lock(qpair);
2938 	result =
2939 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2940 	if (result == VMCI_SUCCESS)
2941 		result = vmci_q_header_free_space(consume_q_header,
2942 						  produce_q_header,
2943 						  qpair->consume_q_size);
2944 	else
2945 		result = 0;
2946 
2947 	qp_unlock(qpair);
2948 
2949 	return result;
2950 }
2951 EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2952 
2953 /*
2954  * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2955  * producer queue.
2956  * @qpair:      Pointer to the queue pair struct.
2957  *
2958  * This is the client interface for getting the amount of
2959  * enqueued data in the QPair from the point of the view of the
2960  * caller as the producer which is not the common case.  Returns < 0 if err,
2961  * else available bytes that may be read.
2962  */
2963 s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2964 {
2965 	struct vmci_queue_header *produce_q_header;
2966 	struct vmci_queue_header *consume_q_header;
2967 	s64 result;
2968 
2969 	if (!qpair)
2970 		return VMCI_ERROR_INVALID_ARGS;
2971 
2972 	qp_lock(qpair);
2973 	result =
2974 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2975 	if (result == VMCI_SUCCESS)
2976 		result = vmci_q_header_buf_ready(produce_q_header,
2977 						 consume_q_header,
2978 						 qpair->produce_q_size);
2979 	else
2980 		result = 0;
2981 
2982 	qp_unlock(qpair);
2983 
2984 	return result;
2985 }
2986 EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2987 
2988 /*
2989  * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2990  * consumer queue.
2991  * @qpair:      Pointer to the queue pair struct.
2992  *
2993  * This is the client interface for getting the amount of
2994  * enqueued data in the QPair from the point of the view of the
2995  * caller as the consumer which is the normal case.  Returns < 0 if err,
2996  * else available bytes that may be read.
2997  */
2998 s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
2999 {
3000 	struct vmci_queue_header *produce_q_header;
3001 	struct vmci_queue_header *consume_q_header;
3002 	s64 result;
3003 
3004 	if (!qpair)
3005 		return VMCI_ERROR_INVALID_ARGS;
3006 
3007 	qp_lock(qpair);
3008 	result =
3009 	    qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3010 	if (result == VMCI_SUCCESS)
3011 		result = vmci_q_header_buf_ready(consume_q_header,
3012 						 produce_q_header,
3013 						 qpair->consume_q_size);
3014 	else
3015 		result = 0;
3016 
3017 	qp_unlock(qpair);
3018 
3019 	return result;
3020 }
3021 EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3022 
3023 /*
3024  * vmci_qpair_enquev() - Throw data on the queue using iov.
3025  * @qpair:      Pointer to the queue pair struct.
3026  * @iov:        Pointer to buffer containing data
3027  * @iov_size:   Length of buffer.
3028  * @buf_type:   Buffer type (Unused).
3029  *
3030  * This is the client interface for enqueueing data into the queue.
3031  * This function uses IO vectors to handle the work. Returns number
3032  * of bytes enqueued or < 0 on error.
3033  */
3034 ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3035 			  struct msghdr *msg,
3036 			  size_t iov_size,
3037 			  int buf_type)
3038 {
3039 	ssize_t result;
3040 
3041 	if (!qpair)
3042 		return VMCI_ERROR_INVALID_ARGS;
3043 
3044 	qp_lock(qpair);
3045 
3046 	do {
3047 		result = qp_enqueue_locked(qpair->produce_q,
3048 					   qpair->consume_q,
3049 					   qpair->produce_q_size,
3050 					   &msg->msg_iter);
3051 
3052 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3053 		    !qp_wait_for_ready_queue(qpair))
3054 			result = VMCI_ERROR_WOULD_BLOCK;
3055 
3056 	} while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3057 
3058 	qp_unlock(qpair);
3059 
3060 	return result;
3061 }
3062 EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3063 
3064 /*
3065  * vmci_qpair_dequev() - Get data from the queue using iov.
3066  * @qpair:      Pointer to the queue pair struct.
3067  * @iov:        Pointer to buffer for the data
3068  * @iov_size:   Length of buffer.
3069  * @buf_type:   Buffer type (Unused).
3070  *
3071  * This is the client interface for dequeueing data from the queue.
3072  * This function uses IO vectors to handle the work. Returns number
3073  * of bytes dequeued or < 0 on error.
3074  */
3075 ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3076 			  struct msghdr *msg,
3077 			  size_t iov_size,
3078 			  int buf_type)
3079 {
3080 	ssize_t result;
3081 
3082 	if (!qpair)
3083 		return VMCI_ERROR_INVALID_ARGS;
3084 
3085 	qp_lock(qpair);
3086 
3087 	do {
3088 		result = qp_dequeue_locked(qpair->produce_q,
3089 					   qpair->consume_q,
3090 					   qpair->consume_q_size,
3091 					   &msg->msg_iter, true);
3092 
3093 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3094 		    !qp_wait_for_ready_queue(qpair))
3095 			result = VMCI_ERROR_WOULD_BLOCK;
3096 
3097 	} while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3098 
3099 	qp_unlock(qpair);
3100 
3101 	return result;
3102 }
3103 EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3104 
3105 /*
3106  * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3107  * @qpair:      Pointer to the queue pair struct.
3108  * @iov:        Pointer to buffer for the data
3109  * @iov_size:   Length of buffer.
3110  * @buf_type:   Buffer type (Unused on Linux).
3111  *
3112  * This is the client interface for peeking into a queue.  (I.e.,
3113  * copy data from the queue without updating the head pointer.)
3114  * This function uses IO vectors to handle the work. Returns number
3115  * of bytes peeked or < 0 on error.
3116  */
3117 ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3118 			 struct msghdr *msg,
3119 			 size_t iov_size,
3120 			 int buf_type)
3121 {
3122 	ssize_t result;
3123 
3124 	if (!qpair)
3125 		return VMCI_ERROR_INVALID_ARGS;
3126 
3127 	qp_lock(qpair);
3128 
3129 	do {
3130 		result = qp_dequeue_locked(qpair->produce_q,
3131 					   qpair->consume_q,
3132 					   qpair->consume_q_size,
3133 					   &msg->msg_iter, false);
3134 
3135 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3136 		    !qp_wait_for_ready_queue(qpair))
3137 			result = VMCI_ERROR_WOULD_BLOCK;
3138 
3139 	} while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3140 
3141 	qp_unlock(qpair);
3142 	return result;
3143 }
3144 EXPORT_SYMBOL_GPL(vmci_qpair_peekv);
3145