1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */ 3 4 #include <linux/anon_inodes.h> 5 #include <linux/file.h> 6 #include <linux/fs.h> 7 #include <linux/highmem.h> 8 #include <linux/vfio.h> 9 #include <linux/vfio_pci_core.h> 10 11 #include "vfio_dev.h" 12 #include "cmds.h" 13 14 static struct pds_vfio_lm_file * 15 pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size) 16 { 17 struct pds_vfio_lm_file *lm_file = NULL; 18 unsigned long long npages; 19 struct page **pages; 20 void *page_mem; 21 const void *p; 22 23 if (!size) 24 return NULL; 25 26 /* Alloc file structure */ 27 lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL); 28 if (!lm_file) 29 return NULL; 30 31 /* Create file */ 32 lm_file->filep = 33 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags); 34 if (IS_ERR(lm_file->filep)) 35 goto out_free_file; 36 37 stream_open(lm_file->filep->f_inode, lm_file->filep); 38 mutex_init(&lm_file->lock); 39 40 /* prevent file from being released before we are done with it */ 41 get_file(lm_file->filep); 42 43 /* Allocate memory for file pages */ 44 npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE); 45 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL); 46 if (!pages) 47 goto out_put_file; 48 49 page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL); 50 if (!page_mem) 51 goto out_free_pages_array; 52 53 p = page_mem - offset_in_page(page_mem); 54 for (unsigned long long i = 0; i < npages; i++) { 55 if (is_vmalloc_addr(p)) 56 pages[i] = vmalloc_to_page(p); 57 else 58 pages[i] = kmap_to_page((void *)p); 59 if (!pages[i]) 60 goto out_free_page_mem; 61 62 p += PAGE_SIZE; 63 } 64 65 /* Create scatterlist of file pages to use for DMA mapping later */ 66 if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0, 67 size, GFP_KERNEL)) 68 goto out_free_page_mem; 69 70 lm_file->size = size; 71 lm_file->pages = pages; 72 lm_file->npages = npages; 73 lm_file->page_mem = page_mem; 74 lm_file->alloc_size = npages * PAGE_SIZE; 75 76 return lm_file; 77 78 out_free_page_mem: 79 kvfree(page_mem); 80 out_free_pages_array: 81 kfree(pages); 82 out_put_file: 83 fput(lm_file->filep); 84 mutex_destroy(&lm_file->lock); 85 out_free_file: 86 kfree(lm_file); 87 88 return NULL; 89 } 90 91 static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file) 92 { 93 mutex_lock(&lm_file->lock); 94 95 lm_file->disabled = true; 96 lm_file->size = 0; 97 lm_file->alloc_size = 0; 98 lm_file->filep->f_pos = 0; 99 100 /* Free scatter list of file pages */ 101 sg_free_table(&lm_file->sg_table); 102 103 kvfree(lm_file->page_mem); 104 lm_file->page_mem = NULL; 105 kfree(lm_file->pages); 106 lm_file->pages = NULL; 107 108 mutex_unlock(&lm_file->lock); 109 110 /* allow file to be released since we are done with it */ 111 fput(lm_file->filep); 112 } 113 114 void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio) 115 { 116 if (!pds_vfio->save_file) 117 return; 118 119 pds_vfio_put_lm_file(pds_vfio->save_file); 120 pds_vfio->save_file = NULL; 121 } 122 123 void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio) 124 { 125 if (!pds_vfio->restore_file) 126 return; 127 128 pds_vfio_put_lm_file(pds_vfio->restore_file); 129 pds_vfio->restore_file = NULL; 130 } 131 132 static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file, 133 unsigned long offset) 134 { 135 unsigned long cur_offset = 0; 136 struct scatterlist *sg; 137 unsigned int i; 138 139 /* All accesses are sequential */ 140 if (offset < lm_file->last_offset || !lm_file->last_offset_sg) { 141 lm_file->last_offset = 0; 142 lm_file->last_offset_sg = lm_file->sg_table.sgl; 143 lm_file->sg_last_entry = 0; 144 } 145 146 cur_offset = lm_file->last_offset; 147 148 for_each_sg(lm_file->last_offset_sg, sg, 149 lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) { 150 if (offset < sg->length + cur_offset) { 151 lm_file->last_offset_sg = sg; 152 lm_file->sg_last_entry += i; 153 lm_file->last_offset = cur_offset; 154 return nth_page(sg_page(sg), 155 (offset - cur_offset) / PAGE_SIZE); 156 } 157 cur_offset += sg->length; 158 } 159 160 return NULL; 161 } 162 163 static int pds_vfio_release_file(struct inode *inode, struct file *filp) 164 { 165 struct pds_vfio_lm_file *lm_file = filp->private_data; 166 167 mutex_lock(&lm_file->lock); 168 lm_file->filep->f_pos = 0; 169 lm_file->size = 0; 170 mutex_unlock(&lm_file->lock); 171 mutex_destroy(&lm_file->lock); 172 kfree(lm_file); 173 174 return 0; 175 } 176 177 static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf, 178 size_t len, loff_t *pos) 179 { 180 struct pds_vfio_lm_file *lm_file = filp->private_data; 181 ssize_t done = 0; 182 183 if (pos) 184 return -ESPIPE; 185 pos = &filp->f_pos; 186 187 mutex_lock(&lm_file->lock); 188 189 if (lm_file->disabled) { 190 done = -ENODEV; 191 goto out_unlock; 192 } 193 194 if (*pos > lm_file->size) { 195 done = -EINVAL; 196 goto out_unlock; 197 } 198 199 len = min_t(size_t, lm_file->size - *pos, len); 200 while (len) { 201 size_t page_offset; 202 struct page *page; 203 size_t page_len; 204 u8 *from_buff; 205 int err; 206 207 page_offset = (*pos) % PAGE_SIZE; 208 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 209 if (!page) { 210 if (done == 0) 211 done = -EINVAL; 212 goto out_unlock; 213 } 214 215 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 216 from_buff = kmap_local_page(page); 217 err = copy_to_user(buf, from_buff + page_offset, page_len); 218 kunmap_local(from_buff); 219 if (err) { 220 done = -EFAULT; 221 goto out_unlock; 222 } 223 *pos += page_len; 224 len -= page_len; 225 done += page_len; 226 buf += page_len; 227 } 228 229 out_unlock: 230 mutex_unlock(&lm_file->lock); 231 return done; 232 } 233 234 static const struct file_operations pds_vfio_save_fops = { 235 .owner = THIS_MODULE, 236 .read = pds_vfio_save_read, 237 .release = pds_vfio_release_file, 238 }; 239 240 static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio) 241 { 242 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 243 struct pds_vfio_lm_file *lm_file; 244 u64 size; 245 int err; 246 247 /* Get live migration state size in this state */ 248 err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size); 249 if (err) { 250 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err)); 251 return err; 252 } 253 254 dev_dbg(dev, "save status, size = %lld\n", size); 255 256 if (!size) { 257 dev_err(dev, "invalid state size\n"); 258 return -EIO; 259 } 260 261 lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size); 262 if (!lm_file) { 263 dev_err(dev, "failed to create save file\n"); 264 return -ENOENT; 265 } 266 267 dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n", 268 lm_file->size, lm_file->alloc_size, lm_file->npages); 269 270 pds_vfio->save_file = lm_file; 271 272 return 0; 273 } 274 275 static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf, 276 size_t len, loff_t *pos) 277 { 278 struct pds_vfio_lm_file *lm_file = filp->private_data; 279 loff_t requested_length; 280 ssize_t done = 0; 281 282 if (pos) 283 return -ESPIPE; 284 285 pos = &filp->f_pos; 286 287 if (*pos < 0 || 288 check_add_overflow((loff_t)len, *pos, &requested_length)) 289 return -EINVAL; 290 291 mutex_lock(&lm_file->lock); 292 293 if (lm_file->disabled) { 294 done = -ENODEV; 295 goto out_unlock; 296 } 297 298 while (len) { 299 size_t page_offset; 300 struct page *page; 301 size_t page_len; 302 u8 *to_buff; 303 int err; 304 305 page_offset = (*pos) % PAGE_SIZE; 306 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 307 if (!page) { 308 if (done == 0) 309 done = -EINVAL; 310 goto out_unlock; 311 } 312 313 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 314 to_buff = kmap_local_page(page); 315 err = copy_from_user(to_buff + page_offset, buf, page_len); 316 kunmap_local(to_buff); 317 if (err) { 318 done = -EFAULT; 319 goto out_unlock; 320 } 321 *pos += page_len; 322 len -= page_len; 323 done += page_len; 324 buf += page_len; 325 lm_file->size += page_len; 326 } 327 out_unlock: 328 mutex_unlock(&lm_file->lock); 329 return done; 330 } 331 332 static const struct file_operations pds_vfio_restore_fops = { 333 .owner = THIS_MODULE, 334 .write = pds_vfio_restore_write, 335 .release = pds_vfio_release_file, 336 }; 337 338 static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio) 339 { 340 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 341 struct pds_vfio_lm_file *lm_file; 342 u64 size; 343 344 size = sizeof(union pds_lm_dev_state); 345 dev_dbg(dev, "restore status, size = %lld\n", size); 346 347 if (!size) { 348 dev_err(dev, "invalid state size"); 349 return -EIO; 350 } 351 352 lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size); 353 if (!lm_file) { 354 dev_err(dev, "failed to create restore file"); 355 return -ENOENT; 356 } 357 pds_vfio->restore_file = lm_file; 358 359 return 0; 360 } 361 362 struct file * 363 pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio, 364 enum vfio_device_mig_state next) 365 { 366 enum vfio_device_mig_state cur = pds_vfio->state; 367 int err; 368 369 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) { 370 err = pds_vfio_get_save_file(pds_vfio); 371 if (err) 372 return ERR_PTR(err); 373 374 err = pds_vfio_get_lm_state_cmd(pds_vfio); 375 if (err) { 376 pds_vfio_put_save_file(pds_vfio); 377 return ERR_PTR(err); 378 } 379 380 return pds_vfio->save_file->filep; 381 } 382 383 if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) { 384 pds_vfio_put_save_file(pds_vfio); 385 pds_vfio_dirty_disable(pds_vfio, true); 386 return NULL; 387 } 388 389 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) { 390 err = pds_vfio_get_restore_file(pds_vfio); 391 if (err) 392 return ERR_PTR(err); 393 394 return pds_vfio->restore_file->filep; 395 } 396 397 if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) { 398 err = pds_vfio_set_lm_state_cmd(pds_vfio); 399 if (err) 400 return ERR_PTR(err); 401 402 pds_vfio_put_restore_file(pds_vfio); 403 return NULL; 404 } 405 406 if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 407 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, 408 PDS_LM_STA_IN_PROGRESS); 409 err = pds_vfio_suspend_device_cmd(pds_vfio, 410 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 411 if (err) 412 return ERR_PTR(err); 413 414 return NULL; 415 } 416 417 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) { 418 err = pds_vfio_resume_device_cmd(pds_vfio, 419 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 420 if (err) 421 return ERR_PTR(err); 422 423 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE); 424 return NULL; 425 } 426 427 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 428 err = pds_vfio_resume_device_cmd(pds_vfio, 429 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 430 if (err) 431 return ERR_PTR(err); 432 433 return NULL; 434 } 435 436 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) { 437 err = pds_vfio_suspend_device_cmd(pds_vfio, 438 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 439 if (err) 440 return ERR_PTR(err); 441 return NULL; 442 } 443 444 return ERR_PTR(-EINVAL); 445 } 446