6  * Copyright (c) 2005-2006, Christopher Clark
7  * Copyright (c) 2004-2005, K A Fraser
15  * Permission is hereby granted, free of charge, to any person obtaining a copy
50 #include <linux/dma-mapping.h>
58 #include <xen/hvc-console.h>
59 #include <xen/swiotlb-xen.h>
64 #include <xen/mem-reservation.h>
121 	 * Mapping a list of frames for storing grant entries. Frames parameter
176 /* This can be used as an l-value */
188 	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {  in get_free_entries()
194 	gnttab_free_count -= count;  in get_free_entries()
195 	while (count--) {  in get_free_entries()
221 	return gnttab_last_free - *gnttab_free_tail_ptr + 1;  in get_seq_entry_count()
227 	int ret = -ENOSPC;  in get_free_seq()
239 		if (ret < 0 && to - from >= count) {  in get_free_seq()
243 			gnttab_free_count -= count;  in get_free_seq()
260 			gnttab_free_tail_ptr = __gnttab_entry(to - 1);  in get_free_seq()
264 	if (gnttab_last_free != gnttab_size - 1)  in get_free_seq()
278 		ret = gnttab_expand(count - gnttab_free_count);  in get_free_entries_seq()
287 		ret = gnttab_expand(count - get_seq_entry_count());  in get_free_entries_seq()
293 	*gnttab_free_tail_ptr = gnttab_entry(ret + count - 1);  in get_free_entries_seq()
294 	gnttab_free_count -= count;  in get_free_entries_seq()
313 		next = callback->next;  in do_free_callbacks()
314 		if (gnttab_free_count >= callback->count) {  in do_free_callbacks()
315 			callback->next = NULL;  in do_free_callbacks()
316 			callback->fn(callback->arg);  in do_free_callbacks()
318 			callback->next = gnttab_free_callback_list;  in do_free_callbacks()
360 	for (i = start; i < start + n - 1; i++)  in gnttab_set_free()
379  *  1. Write ent->domid.
380  *  2. Write ent->frame: Frame to which access is permitted.
382  *  4. Write ent->flags, inc. valid type.
403  * Public grant-issuing interface functions
408 	gnttab_interface->update_entry(ref, domid, frame,  in gnttab_grant_foreign_access_ref()
420 		return -ENOSPC;  in gnttab_grant_foreign_access()
467 	return gnttab_interface->end_foreign_access_ref(ref);  in _gnttab_end_foreign_access_ref()
513 	while ((ignore_limit || nr--) && !list_empty(&deferred_list)) {  in gnttab_handle_deferred()
520 		list_del(&entry->list);  in gnttab_handle_deferred()
522 		if (_gnttab_end_foreign_access_ref(entry->ref)) {  in gnttab_handle_deferred()
525 			put_free_entry(entry->ref);  in gnttab_handle_deferred()
527 				 entry->ref, page_to_pfn(entry->page),  in gnttab_handle_deferred()
529 			put_page(entry->page);  in gnttab_handle_deferred()
534 			if (!--entry->warn_delay)  in gnttab_handle_deferred()
535 				pr_info("g.e. %#x still pending\n", entry->ref);  in gnttab_handle_deferred()
541 			list_add_tail(&entry->list, &deferred_list);  in gnttab_handle_deferred()
561 		unsigned long gfn = gnttab_interface->read_frame(ref);  in gnttab_add_deferred()
570 		entry->ref = ref;  in gnttab_add_deferred()
571 		entry->page = page;  in gnttab_add_deferred()
572 		entry->warn_delay = 60;  in gnttab_add_deferred()
574 		list_add_tail(&entry->list, &deferred_list);  in gnttab_add_deferred()
583 			 ref, page ? page_to_pfn(page) : -1, deferred, leaked);  in gnttab_add_deferred()
588 			ref, page ? page_to_pfn(page) : -1, deferred, leaked);  in gnttab_add_deferred()
641 	for (i = count; i > 0; i--)  in gnttab_free_grant_reference_seq()
642 		put_free_entry_locked(head + i - 1);  in gnttab_free_grant_reference_seq()
653 		return -ENOSPC;  in gnttab_alloc_grant_references()
671 		return -ENOSPC;  in gnttab_alloc_grant_reference_seq()
689 		return -ENOSPC;  in gnttab_claim_grant_reference()
716 		cb = cb->next;  in gnttab_request_free_callback()
719 	callback->fn = fn;  in gnttab_request_free_callback()
720 	callback->arg = arg;  in gnttab_request_free_callback()
721 	callback->count = count;  in gnttab_request_free_callback()
722 	callback->next = gnttab_free_callback_list;  in gnttab_request_free_callback()
736 	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {  in gnttab_cancel_free_callback()
738 			*pcb = callback->next;  in gnttab_cancel_free_callback()
748 	return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /  in gnttab_frames()
758 	grefs_per_frame = gnttab_interface->grefs_per_grant_frame;  in grow_gnttab_list()
784 	while (i-- > nr_glist_frames)  in grow_gnttab_list()
786 	return -ENOMEM;  in grow_gnttab_list()
826 		return -EINVAL;  in gnttab_setup_auto_xlat_frames()
832 		return -ENOMEM;  in gnttab_setup_auto_xlat_frames()
837 		return -ENOMEM;  in gnttab_setup_auto_xlat_frames()
873 			return -ENOMEM;  in gnttab_pages_set_private()
885  * gnttab_alloc_pages - alloc pages suitable for grant mapping into
908 	cache->pages = NULL;  in cache_init()
913 	return !cache->pages;  in cache_empty()
920 	page = cache->pages;  in cache_deq()
921 	cache->pages = page->zone_device_data;  in cache_deq()
928 	page->zone_device_data = cache->pages;  in cache_enq()
929 	cache->pages = page;  in cache_enq()
934 	INIT_LIST_HEAD(&cache->pages);  in cache_init()
939 	return list_empty(&cache->pages);  in cache_empty()
946 	page = list_first_entry(&cache->pages, struct page, lru);  in cache_deq()
947 	list_del(&page->lru);  in cache_deq()
954 	list_add(&page->lru, &cache->pages);  in cache_enq()
960 	spin_lock_init(&cache->lock);  in gnttab_page_cache_init()
962 	cache->num_pages = 0;  in gnttab_page_cache_init()
970 	spin_lock_irqsave(&cache->lock, flags);  in gnttab_page_cache_get()
973 		spin_unlock_irqrestore(&cache->lock, flags);  in gnttab_page_cache_get()
978 	cache->num_pages--;  in gnttab_page_cache_get()
980 	spin_unlock_irqrestore(&cache->lock, flags);  in gnttab_page_cache_get()
992 	spin_lock_irqsave(&cache->lock, flags);  in gnttab_page_cache_put()
996 	cache->num_pages += num;  in gnttab_page_cache_put()
998 	spin_unlock_irqrestore(&cache->lock, flags);  in gnttab_page_cache_put()
1008 	spin_lock_irqsave(&cache->lock, flags);  in gnttab_page_cache_shrink()
1010 	while (cache->num_pages > num) {  in gnttab_page_cache_shrink()
1012 		cache->num_pages--;  in gnttab_page_cache_shrink()
1014 			spin_unlock_irqrestore(&cache->lock, flags);  in gnttab_page_cache_shrink()
1017 			spin_lock_irqsave(&cache->lock, flags);  in gnttab_page_cache_shrink()
1021 	spin_unlock_irqrestore(&cache->lock, flags);  in gnttab_page_cache_shrink()
1044  * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
1057  * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into
1066 	if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT))  in gnttab_dma_alloc_pages()
1067 		return -ENOMEM;  in gnttab_dma_alloc_pages()
1069 	size = args->nr_pages << PAGE_SHIFT;  in gnttab_dma_alloc_pages()
1070 	if (args->coherent)  in gnttab_dma_alloc_pages()
1071 		args->vaddr = dma_alloc_coherent(args->dev, size,  in gnttab_dma_alloc_pages()
1072 						 &args->dev_bus_addr,  in gnttab_dma_alloc_pages()
1075 		args->vaddr = dma_alloc_wc(args->dev, size,  in gnttab_dma_alloc_pages()
1076 					   &args->dev_bus_addr,  in gnttab_dma_alloc_pages()
1078 	if (!args->vaddr) {  in gnttab_dma_alloc_pages()
1080 		return -ENOMEM;  in gnttab_dma_alloc_pages()
1083 	start_pfn = __phys_to_pfn(args->dev_bus_addr);  in gnttab_dma_alloc_pages()
1084 	for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages;  in gnttab_dma_alloc_pages()
1088 		args->pages[i] = page;  in gnttab_dma_alloc_pages()
1089 		args->frames[i] = xen_page_to_gfn(page);  in gnttab_dma_alloc_pages()
1093 	xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages);  in gnttab_dma_alloc_pages()
1095 	ret = xenmem_reservation_decrease(args->nr_pages, args->frames);  in gnttab_dma_alloc_pages()
1096 	if (ret != args->nr_pages) {  in gnttab_dma_alloc_pages()
1098 		ret = -EFAULT;  in gnttab_dma_alloc_pages()
1102 	ret = gnttab_pages_set_private(args->nr_pages, args->pages);  in gnttab_dma_alloc_pages()
1115  * gnttab_dma_free_pages - free DMAable pages
1123 	gnttab_pages_clear_private(args->nr_pages, args->pages);  in gnttab_dma_free_pages()
1125 	for (i = 0; i < args->nr_pages; i++)  in gnttab_dma_free_pages()
1126 		args->frames[i] = page_to_xen_pfn(args->pages[i]);  in gnttab_dma_free_pages()
1128 	ret = xenmem_reservation_increase(args->nr_pages, args->frames);  in gnttab_dma_free_pages()
1129 	if (ret != args->nr_pages) {  in gnttab_dma_free_pages()
1131 		ret = -EFAULT;  in gnttab_dma_free_pages()
1136 	xenmem_reservation_va_mapping_update(args->nr_pages, args->pages,  in gnttab_dma_free_pages()
1137 					     args->frames);  in gnttab_dma_free_pages()
1139 	size = args->nr_pages << PAGE_SHIFT;  in gnttab_dma_free_pages()
1140 	if (args->coherent)  in gnttab_dma_free_pages()
1141 		dma_free_coherent(args->dev, size,  in gnttab_dma_free_pages()
1142 				  args->vaddr, args->dev_bus_addr);  in gnttab_dma_free_pages()
1144 		dma_free_wc(args->dev, size,  in gnttab_dma_free_pages()
1145 			    args->vaddr, args->dev_bus_addr);  in gnttab_dma_free_pages()
1166 		pr_err("%s: %s eagain grant\n", func, current->comm);  in gnttab_retry_eagain_gop()
1178 		if (op->status == GNTST_eagain)  in gnttab_batch_map()
1180 						&op->status, __func__);  in gnttab_batch_map()
1191 		if (op->status == GNTST_eagain)  in gnttab_batch_copy()
1193 						&op->status, __func__);  in gnttab_batch_copy()
1207 	len = min_t(unsigned int, PAGE_SIZE - offset, len);  in gnttab_foreach_grant_in_range()
1213 		glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);  in gnttab_foreach_grant_in_range()
1218 		len -= glen;  in gnttab_foreach_grant_in_range()
1263 			foreign->domid = map_ops[i].dom;  in gnttab_map_refs()
1264 			foreign->gref = map_ops[i].ref;  in gnttab_map_refs()
1269 			pr_warn_ratelimited("maptrack limit reached, can't map all guest pages\n");  in gnttab_map_refs()
1278 			i--;  in gnttab_map_refs()
1318 	if (unmap_data->age != UINT_MAX)  in gnttab_unmap_work()
1319 		unmap_data->age++;  in gnttab_unmap_work()
1328 	for (pc = 0; pc < item->count; pc++) {  in __gnttab_unmap_refs_async()
1329 		if (page_count(item->pages[pc]) > 1) {  in __gnttab_unmap_refs_async()
1330 			unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);  in __gnttab_unmap_refs_async()
1331 			schedule_delayed_work(&item->gnttab_work,  in __gnttab_unmap_refs_async()
1337 	ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,  in __gnttab_unmap_refs_async()
1338 				item->pages, item->count);  in __gnttab_unmap_refs_async()
1339 	item->done(ret, item);  in __gnttab_unmap_refs_async()
1344 	INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);  in gnttab_unmap_refs_async()
1345 	item->age = 0;  in gnttab_unmap_refs_async()
1354 	struct unmap_refs_callback_data *d = data->data;  in unmap_refs_callback()
1356 	d->result = result;  in unmap_refs_callback()
1357 	complete(&d->completion);  in unmap_refs_callback()
1365 	item->data = &data;  in gnttab_unmap_refs_sync()
1366 	item->done = &unmap_refs_callback;  in gnttab_unmap_refs_sync()
1410 		return -ENOMEM;  in gnttab_map_frames_v2()
1418 	if (rc == -ENOSYS) {  in gnttab_map_frames_v2()
1420 		return -ENOSYS;  in gnttab_map_frames_v2()
1472 		} while (i-- > start_idx);  in gnttab_map()
1482 		return -ENOMEM;  in gnttab_map()
1489 	if (rc == -ENOSYS) {  in gnttab_map()
1491 		return -ENOSYS;  in gnttab_map()
1496 	rc = gnttab_interface->map_frames(frames, nr_gframes);  in gnttab_map()
1562 		gnttab_interface->version);  in gnttab_request_version()
1571 		return -ENOSYS;  in gnttab_setup()
1577 			return -ENOMEM;  in gnttab_setup()
1580 	return gnttab_map(0, nr_grant_frames - 1);  in gnttab_setup()
1592 		gnttab_interface->unmap_frames();  in gnttab_suspend()
1602 	extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /  in gnttab_expand()
1603 		 gnttab_interface->grefs_per_grant_frame);  in gnttab_expand()
1605 		pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"  in gnttab_expand()
1606 				    " cur=%u extra=%u limit=%u"  in gnttab_expand()
1610 		return -ENOSPC;  in gnttab_expand()
1613 	rc = gnttab_map(cur, cur + extra - 1);  in gnttab_expand()
1630 			gnttab_interface->grefs_per_grant_frame;  in gnttab_init()
1634 	 * grant reference free list on the current hypervisor.  in gnttab_init()
1642 		return -ENOMEM;  in gnttab_init()
1648 			ret = -ENOMEM;  in gnttab_init()
1655 		ret = -ENOMEM;  in gnttab_init()
1665 		ret = -ENODEV;  in gnttab_init()
1669 	gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame;  in gnttab_init()
1672 			gnttab_size - GNTTAB_NR_RESERVED_ENTRIES);  in gnttab_init()
1678 	for (i--; i >= 0; i--)  in gnttab_init()
1689 		return -ENODEV;  in __gnttab_init()
1691 	/* Delay grant-table initialization in the PV on HVM case */  in __gnttab_init()