1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */ 3 4 #include <linux/anon_inodes.h> 5 #include <linux/file.h> 6 #include <linux/fs.h> 7 #include <linux/highmem.h> 8 #include <linux/vfio.h> 9 #include <linux/vfio_pci_core.h> 10 11 #include "vfio_dev.h" 12 #include "cmds.h" 13 14 static struct pds_vfio_lm_file * 15 pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size) 16 { 17 struct pds_vfio_lm_file *lm_file = NULL; 18 unsigned long long npages; 19 struct page **pages; 20 void *page_mem; 21 const void *p; 22 23 if (!size) 24 return NULL; 25 26 /* Alloc file structure */ 27 lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL); 28 if (!lm_file) 29 return NULL; 30 31 /* Create file */ 32 lm_file->filep = 33 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags); 34 if (IS_ERR(lm_file->filep)) 35 goto out_free_file; 36 37 stream_open(lm_file->filep->f_inode, lm_file->filep); 38 mutex_init(&lm_file->lock); 39 40 /* prevent file from being released before we are done with it */ 41 get_file(lm_file->filep); 42 43 /* Allocate memory for file pages */ 44 npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE); 45 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL); 46 if (!pages) 47 goto out_put_file; 48 49 page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL); 50 if (!page_mem) 51 goto out_free_pages_array; 52 53 p = page_mem - offset_in_page(page_mem); 54 for (unsigned long long i = 0; i < npages; i++) { 55 if (is_vmalloc_addr(p)) 56 pages[i] = vmalloc_to_page(p); 57 else 58 pages[i] = kmap_to_page((void *)p); 59 if (!pages[i]) 60 goto out_free_page_mem; 61 62 p += PAGE_SIZE; 63 } 64 65 /* Create scatterlist of file pages to use for DMA mapping later */ 66 if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0, 67 size, GFP_KERNEL)) 68 goto out_free_page_mem; 69 70 lm_file->size = size; 71 lm_file->pages = pages; 72 lm_file->npages = npages; 73 lm_file->page_mem = page_mem; 74 lm_file->alloc_size = npages * PAGE_SIZE; 75 76 return lm_file; 77 78 out_free_page_mem: 79 kvfree(page_mem); 80 out_free_pages_array: 81 kfree(pages); 82 out_put_file: 83 fput(lm_file->filep); 84 mutex_destroy(&lm_file->lock); 85 out_free_file: 86 kfree(lm_file); 87 88 return NULL; 89 } 90 91 static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file) 92 { 93 mutex_lock(&lm_file->lock); 94 95 lm_file->disabled = true; 96 lm_file->size = 0; 97 lm_file->alloc_size = 0; 98 lm_file->filep->f_pos = 0; 99 100 /* Free scatter list of file pages */ 101 sg_free_table(&lm_file->sg_table); 102 103 kvfree(lm_file->page_mem); 104 lm_file->page_mem = NULL; 105 kfree(lm_file->pages); 106 lm_file->pages = NULL; 107 108 mutex_unlock(&lm_file->lock); 109 110 /* allow file to be released since we are done with it */ 111 fput(lm_file->filep); 112 } 113 114 void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio) 115 { 116 if (!pds_vfio->save_file) 117 return; 118 119 pds_vfio_put_lm_file(pds_vfio->save_file); 120 pds_vfio->save_file = NULL; 121 } 122 123 void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio) 124 { 125 if (!pds_vfio->restore_file) 126 return; 127 128 pds_vfio_put_lm_file(pds_vfio->restore_file); 129 pds_vfio->restore_file = NULL; 130 } 131 132 static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file, 133 unsigned long offset) 134 { 135 unsigned long cur_offset = 0; 136 struct scatterlist *sg; 137 unsigned int i; 138 139 /* All accesses are sequential */ 140 if (offset < lm_file->last_offset || !lm_file->last_offset_sg) { 141 lm_file->last_offset = 0; 142 lm_file->last_offset_sg = lm_file->sg_table.sgl; 143 lm_file->sg_last_entry = 0; 144 } 145 146 cur_offset = lm_file->last_offset; 147 148 for_each_sg(lm_file->last_offset_sg, sg, 149 lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) { 150 if (offset < sg->length + cur_offset) { 151 lm_file->last_offset_sg = sg; 152 lm_file->sg_last_entry += i; 153 lm_file->last_offset = cur_offset; 154 return sg_page(sg) + (offset - cur_offset) / PAGE_SIZE; 155 } 156 cur_offset += sg->length; 157 } 158 159 return NULL; 160 } 161 162 static int pds_vfio_release_file(struct inode *inode, struct file *filp) 163 { 164 struct pds_vfio_lm_file *lm_file = filp->private_data; 165 166 mutex_lock(&lm_file->lock); 167 lm_file->filep->f_pos = 0; 168 lm_file->size = 0; 169 mutex_unlock(&lm_file->lock); 170 mutex_destroy(&lm_file->lock); 171 kfree(lm_file); 172 173 return 0; 174 } 175 176 static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf, 177 size_t len, loff_t *pos) 178 { 179 struct pds_vfio_lm_file *lm_file = filp->private_data; 180 ssize_t done = 0; 181 182 if (pos) 183 return -ESPIPE; 184 pos = &filp->f_pos; 185 186 mutex_lock(&lm_file->lock); 187 188 if (lm_file->disabled) { 189 done = -ENODEV; 190 goto out_unlock; 191 } 192 193 if (*pos > lm_file->size) { 194 done = -EINVAL; 195 goto out_unlock; 196 } 197 198 len = min_t(size_t, lm_file->size - *pos, len); 199 while (len) { 200 size_t page_offset; 201 struct page *page; 202 size_t page_len; 203 u8 *from_buff; 204 int err; 205 206 page_offset = (*pos) % PAGE_SIZE; 207 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 208 if (!page) { 209 if (done == 0) 210 done = -EINVAL; 211 goto out_unlock; 212 } 213 214 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 215 from_buff = kmap_local_page(page); 216 err = copy_to_user(buf, from_buff + page_offset, page_len); 217 kunmap_local(from_buff); 218 if (err) { 219 done = -EFAULT; 220 goto out_unlock; 221 } 222 *pos += page_len; 223 len -= page_len; 224 done += page_len; 225 buf += page_len; 226 } 227 228 out_unlock: 229 mutex_unlock(&lm_file->lock); 230 return done; 231 } 232 233 static const struct file_operations pds_vfio_save_fops = { 234 .owner = THIS_MODULE, 235 .read = pds_vfio_save_read, 236 .release = pds_vfio_release_file, 237 }; 238 239 static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio) 240 { 241 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 242 struct pds_vfio_lm_file *lm_file; 243 u64 size; 244 int err; 245 246 /* Get live migration state size in this state */ 247 err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size); 248 if (err) { 249 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err)); 250 return err; 251 } 252 253 dev_dbg(dev, "save status, size = %lld\n", size); 254 255 if (!size) { 256 dev_err(dev, "invalid state size\n"); 257 return -EIO; 258 } 259 260 lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size); 261 if (!lm_file) { 262 dev_err(dev, "failed to create save file\n"); 263 return -ENOENT; 264 } 265 266 dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n", 267 lm_file->size, lm_file->alloc_size, lm_file->npages); 268 269 pds_vfio->save_file = lm_file; 270 271 return 0; 272 } 273 274 static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf, 275 size_t len, loff_t *pos) 276 { 277 struct pds_vfio_lm_file *lm_file = filp->private_data; 278 loff_t requested_length; 279 ssize_t done = 0; 280 281 if (pos) 282 return -ESPIPE; 283 284 pos = &filp->f_pos; 285 286 if (*pos < 0 || 287 check_add_overflow((loff_t)len, *pos, &requested_length)) 288 return -EINVAL; 289 290 mutex_lock(&lm_file->lock); 291 292 if (lm_file->disabled) { 293 done = -ENODEV; 294 goto out_unlock; 295 } 296 297 while (len) { 298 size_t page_offset; 299 struct page *page; 300 size_t page_len; 301 u8 *to_buff; 302 int err; 303 304 page_offset = (*pos) % PAGE_SIZE; 305 page = pds_vfio_get_file_page(lm_file, *pos - page_offset); 306 if (!page) { 307 if (done == 0) 308 done = -EINVAL; 309 goto out_unlock; 310 } 311 312 page_len = min_t(size_t, len, PAGE_SIZE - page_offset); 313 to_buff = kmap_local_page(page); 314 err = copy_from_user(to_buff + page_offset, buf, page_len); 315 kunmap_local(to_buff); 316 if (err) { 317 done = -EFAULT; 318 goto out_unlock; 319 } 320 *pos += page_len; 321 len -= page_len; 322 done += page_len; 323 buf += page_len; 324 lm_file->size += page_len; 325 } 326 out_unlock: 327 mutex_unlock(&lm_file->lock); 328 return done; 329 } 330 331 static const struct file_operations pds_vfio_restore_fops = { 332 .owner = THIS_MODULE, 333 .write = pds_vfio_restore_write, 334 .release = pds_vfio_release_file, 335 }; 336 337 static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio) 338 { 339 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev; 340 struct pds_vfio_lm_file *lm_file; 341 u64 size; 342 343 size = sizeof(union pds_lm_dev_state); 344 dev_dbg(dev, "restore status, size = %lld\n", size); 345 346 if (!size) { 347 dev_err(dev, "invalid state size"); 348 return -EIO; 349 } 350 351 lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size); 352 if (!lm_file) { 353 dev_err(dev, "failed to create restore file"); 354 return -ENOENT; 355 } 356 pds_vfio->restore_file = lm_file; 357 358 return 0; 359 } 360 361 struct file * 362 pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio, 363 enum vfio_device_mig_state next) 364 { 365 enum vfio_device_mig_state cur = pds_vfio->state; 366 int err; 367 368 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) { 369 err = pds_vfio_get_save_file(pds_vfio); 370 if (err) 371 return ERR_PTR(err); 372 373 err = pds_vfio_get_lm_state_cmd(pds_vfio); 374 if (err) { 375 pds_vfio_put_save_file(pds_vfio); 376 return ERR_PTR(err); 377 } 378 379 return pds_vfio->save_file->filep; 380 } 381 382 if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) { 383 pds_vfio_put_save_file(pds_vfio); 384 pds_vfio_dirty_disable(pds_vfio, true); 385 return NULL; 386 } 387 388 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) { 389 err = pds_vfio_get_restore_file(pds_vfio); 390 if (err) 391 return ERR_PTR(err); 392 393 return pds_vfio->restore_file->filep; 394 } 395 396 if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) { 397 err = pds_vfio_set_lm_state_cmd(pds_vfio); 398 if (err) 399 return ERR_PTR(err); 400 401 pds_vfio_put_restore_file(pds_vfio); 402 return NULL; 403 } 404 405 if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 406 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, 407 PDS_LM_STA_IN_PROGRESS); 408 err = pds_vfio_suspend_device_cmd(pds_vfio, 409 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 410 if (err) 411 return ERR_PTR(err); 412 413 return NULL; 414 } 415 416 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) { 417 err = pds_vfio_resume_device_cmd(pds_vfio, 418 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 419 if (err) 420 return ERR_PTR(err); 421 422 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE); 423 return NULL; 424 } 425 426 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) { 427 err = pds_vfio_resume_device_cmd(pds_vfio, 428 PDS_LM_SUSPEND_RESUME_TYPE_P2P); 429 if (err) 430 return ERR_PTR(err); 431 432 return NULL; 433 } 434 435 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) { 436 err = pds_vfio_suspend_device_cmd(pds_vfio, 437 PDS_LM_SUSPEND_RESUME_TYPE_FULL); 438 if (err) 439 return ERR_PTR(err); 440 return NULL; 441 } 442 443 return ERR_PTR(-EINVAL); 444 } 445