1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/errno.h> 3 #include <linux/gfp.h> 4 #include <linux/kernel.h> 5 #include <linux/mm.h> 6 #include <linux/memremap.h> 7 #include <linux/slab.h> 8 9 #include <asm/page.h> 10 11 #include <xen/balloon.h> 12 #include <xen/page.h> 13 #include <xen/xen.h> 14 15 static DEFINE_MUTEX(list_lock); 16 static struct page *page_list; 17 static unsigned int list_count; 18 19 static struct resource *target_resource; 20 21 /* 22 * If arch is not happy with system "iomem_resource" being used for 23 * the region allocation it can provide it's own view by creating specific 24 * Xen resource with unused regions of guest physical address space provided 25 * by the hypervisor. 26 */ 27 int __weak __init arch_xen_unpopulated_init(struct resource **res) 28 { 29 *res = &iomem_resource; 30 31 return 0; 32 } 33 34 static int fill_list(unsigned int nr_pages) 35 { 36 struct dev_pagemap *pgmap; 37 struct resource *res, *tmp_res = NULL; 38 void *vaddr; 39 unsigned int i, alloc_pages = round_up(nr_pages, PAGES_PER_SECTION); 40 struct range mhp_range; 41 int ret; 42 43 res = kzalloc(sizeof(*res), GFP_KERNEL); 44 if (!res) 45 return -ENOMEM; 46 47 res->name = "Xen scratch"; 48 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; 49 50 mhp_range = mhp_get_pluggable_range(true); 51 52 ret = allocate_resource(target_resource, res, 53 alloc_pages * PAGE_SIZE, mhp_range.start, mhp_range.end, 54 PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL); 55 if (ret < 0) { 56 pr_err("Cannot allocate new IOMEM resource\n"); 57 goto err_resource; 58 } 59 60 /* 61 * Reserve the region previously allocated from Xen resource to avoid 62 * re-using it by someone else. 63 */ 64 if (target_resource != &iomem_resource) { 65 tmp_res = kzalloc(sizeof(*tmp_res), GFP_KERNEL); 66 if (!tmp_res) { 67 ret = -ENOMEM; 68 goto err_insert; 69 } 70 71 tmp_res->name = res->name; 72 tmp_res->start = res->start; 73 tmp_res->end = res->end; 74 tmp_res->flags = res->flags; 75 76 ret = request_resource(&iomem_resource, tmp_res); 77 if (ret < 0) { 78 pr_err("Cannot request resource %pR (%d)\n", tmp_res, ret); 79 kfree(tmp_res); 80 goto err_insert; 81 } 82 } 83 84 pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL); 85 if (!pgmap) { 86 ret = -ENOMEM; 87 goto err_pgmap; 88 } 89 90 pgmap->type = MEMORY_DEVICE_GENERIC; 91 pgmap->range = (struct range) { 92 .start = res->start, 93 .end = res->end, 94 }; 95 pgmap->nr_range = 1; 96 pgmap->owner = res; 97 98 #ifdef CONFIG_XEN_HAVE_PVMMU 99 /* 100 * memremap will build page tables for the new memory so 101 * the p2m must contain invalid entries so the correct 102 * non-present PTEs will be written. 103 * 104 * If a failure occurs, the original (identity) p2m entries 105 * are not restored since this region is now known not to 106 * conflict with any devices. 107 */ 108 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 109 xen_pfn_t pfn = PFN_DOWN(res->start); 110 111 for (i = 0; i < alloc_pages; i++) { 112 if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) { 113 pr_warn("set_phys_to_machine() failed, no memory added\n"); 114 ret = -ENOMEM; 115 goto err_memremap; 116 } 117 } 118 } 119 #endif 120 121 vaddr = memremap_pages(pgmap, NUMA_NO_NODE); 122 if (IS_ERR(vaddr)) { 123 pr_err("Cannot remap memory range\n"); 124 ret = PTR_ERR(vaddr); 125 goto err_memremap; 126 } 127 128 for (i = 0; i < alloc_pages; i++) { 129 struct page *pg = virt_to_page(vaddr + PAGE_SIZE * i); 130 131 pg->zone_device_data = page_list; 132 page_list = pg; 133 list_count++; 134 } 135 136 return 0; 137 138 err_memremap: 139 kfree(pgmap); 140 err_pgmap: 141 if (tmp_res) { 142 release_resource(tmp_res); 143 kfree(tmp_res); 144 } 145 err_insert: 146 release_resource(res); 147 err_resource: 148 kfree(res); 149 return ret; 150 } 151 152 /** 153 * xen_alloc_unpopulated_pages - alloc unpopulated pages 154 * @nr_pages: Number of pages 155 * @pages: pages returned 156 * @return 0 on success, error otherwise 157 */ 158 int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages) 159 { 160 unsigned int i; 161 int ret = 0; 162 163 /* 164 * Fallback to default behavior if we do not have any suitable resource 165 * to allocate required region from and as the result we won't be able to 166 * construct pages. 167 */ 168 if (!target_resource) 169 return xen_alloc_ballooned_pages(nr_pages, pages); 170 171 mutex_lock(&list_lock); 172 if (list_count < nr_pages) { 173 ret = fill_list(nr_pages - list_count); 174 if (ret) 175 goto out; 176 } 177 178 for (i = 0; i < nr_pages; i++) { 179 struct page *pg = page_list; 180 181 BUG_ON(!pg); 182 page_list = pg->zone_device_data; 183 list_count--; 184 pages[i] = pg; 185 186 #ifdef CONFIG_XEN_HAVE_PVMMU 187 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 188 ret = xen_alloc_p2m_entry(page_to_pfn(pg)); 189 if (ret < 0) { 190 unsigned int j; 191 192 for (j = 0; j <= i; j++) { 193 pages[j]->zone_device_data = page_list; 194 page_list = pages[j]; 195 list_count++; 196 } 197 goto out; 198 } 199 } 200 #endif 201 } 202 203 out: 204 mutex_unlock(&list_lock); 205 return ret; 206 } 207 EXPORT_SYMBOL(xen_alloc_unpopulated_pages); 208 209 /** 210 * xen_free_unpopulated_pages - return unpopulated pages 211 * @nr_pages: Number of pages 212 * @pages: pages to return 213 */ 214 void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages) 215 { 216 unsigned int i; 217 218 if (!target_resource) { 219 xen_free_ballooned_pages(nr_pages, pages); 220 return; 221 } 222 223 mutex_lock(&list_lock); 224 for (i = 0; i < nr_pages; i++) { 225 pages[i]->zone_device_data = page_list; 226 page_list = pages[i]; 227 list_count++; 228 } 229 mutex_unlock(&list_lock); 230 } 231 EXPORT_SYMBOL(xen_free_unpopulated_pages); 232 233 static int __init unpopulated_init(void) 234 { 235 int ret; 236 237 if (!xen_domain()) 238 return -ENODEV; 239 240 ret = arch_xen_unpopulated_init(&target_resource); 241 if (ret) { 242 pr_err("xen:unpopulated: Cannot initialize target resource\n"); 243 target_resource = NULL; 244 } 245 246 return ret; 247 } 248 early_initcall(unpopulated_init); 249