Lines Matching refs:migf
68 static void mlx5vf_disable_fd(struct mlx5_vf_migration_file *migf) in mlx5vf_disable_fd() argument
70 mutex_lock(&migf->lock); in mlx5vf_disable_fd()
71 migf->state = MLX5_MIGF_STATE_ERROR; in mlx5vf_disable_fd()
72 migf->filp->f_pos = 0; in mlx5vf_disable_fd()
73 mutex_unlock(&migf->lock); in mlx5vf_disable_fd()
78 struct mlx5_vf_migration_file *migf = filp->private_data; in mlx5vf_release_file() local
80 mlx5vf_disable_fd(migf); in mlx5vf_release_file()
81 mutex_destroy(&migf->lock); in mlx5vf_release_file()
82 kfree(migf); in mlx5vf_release_file()
87 mlx5vf_get_data_buff_from_pos(struct mlx5_vf_migration_file *migf, loff_t pos, in mlx5vf_get_data_buff_from_pos() argument
94 spin_lock_irq(&migf->list_lock); in mlx5vf_get_data_buff_from_pos()
95 if (list_empty(&migf->buf_list)) { in mlx5vf_get_data_buff_from_pos()
100 buf = list_first_entry(&migf->buf_list, struct mlx5_vhca_data_buffer, in mlx5vf_get_data_buff_from_pos()
112 migf->state = MLX5_MIGF_STATE_ERROR; in mlx5vf_get_data_buff_from_pos()
115 spin_unlock_irq(&migf->list_lock); in mlx5vf_get_data_buff_from_pos()
121 struct mlx5_vf_migration_file *migf = vhca_buf->migf; in mlx5vf_buf_read_done() local
129 migf->buf_header[chunk_num - 1] = vhca_buf; in mlx5vf_buf_read_done()
131 migf->buf[chunk_num - 1] = vhca_buf; in mlx5vf_buf_read_done()
133 spin_lock_irq(&migf->list_lock); in mlx5vf_buf_read_done()
137 migf->next_required_umem_size; in mlx5vf_buf_read_done()
138 migf->next_required_umem_size = 0; in mlx5vf_buf_read_done()
139 migf->num_ready_chunks--; in mlx5vf_buf_read_done()
141 spin_unlock_irq(&migf->list_lock); in mlx5vf_buf_read_done()
143 mlx5vf_mig_file_set_save_work(migf, chunk_num, in mlx5vf_buf_read_done()
148 spin_lock_irq(&migf->list_lock); in mlx5vf_buf_read_done()
150 list_add_tail(&vhca_buf->buf_elm, &vhca_buf->migf->avail_list); in mlx5vf_buf_read_done()
151 spin_unlock_irq(&migf->list_lock); in mlx5vf_buf_read_done()
198 struct mlx5_vf_migration_file *migf = filp->private_data; in mlx5vf_save_read() local
209 if (wait_event_interruptible(migf->poll_wait, in mlx5vf_save_read()
210 !list_empty(&migf->buf_list) || in mlx5vf_save_read()
211 migf->state == MLX5_MIGF_STATE_ERROR || in mlx5vf_save_read()
212 migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR || in mlx5vf_save_read()
213 migf->state == MLX5_MIGF_STATE_PRE_COPY || in mlx5vf_save_read()
214 migf->state == MLX5_MIGF_STATE_COMPLETE)) in mlx5vf_save_read()
218 mutex_lock(&migf->lock); in mlx5vf_save_read()
219 if (migf->state == MLX5_MIGF_STATE_ERROR) { in mlx5vf_save_read()
227 vhca_buf = mlx5vf_get_data_buff_from_pos(migf, *pos, in mlx5vf_save_read()
232 if (end_of_data && (migf->state == MLX5_MIGF_STATE_PRE_COPY || in mlx5vf_save_read()
233 migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR)) { in mlx5vf_save_read()
238 if (end_of_data && migf->state != MLX5_MIGF_STATE_COMPLETE) { in mlx5vf_save_read()
263 mutex_unlock(&migf->lock); in mlx5vf_save_read()
270 struct mlx5_vf_migration_file *migf = filp->private_data; in mlx5vf_save_poll() local
273 poll_wait(filp, &migf->poll_wait, wait); in mlx5vf_save_poll()
275 mutex_lock(&migf->lock); in mlx5vf_save_poll()
276 if (migf->state == MLX5_MIGF_STATE_ERROR) in mlx5vf_save_poll()
278 else if (!list_empty(&migf->buf_list) || in mlx5vf_save_poll()
279 migf->state == MLX5_MIGF_STATE_COMPLETE) in mlx5vf_save_poll()
281 mutex_unlock(&migf->lock); in mlx5vf_save_poll()
290 static void mlx5vf_mark_err(struct mlx5_vf_migration_file *migf) in mlx5vf_mark_err() argument
292 migf->state = MLX5_MIGF_STATE_ERROR; in mlx5vf_mark_err()
293 wake_up_interruptible(&migf->poll_wait); in mlx5vf_mark_err()
296 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, in mlx5vf_mig_file_set_save_work() argument
299 migf->save_data[chunk_num - 1].next_required_umem_size = in mlx5vf_mig_file_set_save_work()
301 migf->save_data[chunk_num - 1].migf = migf; in mlx5vf_mig_file_set_save_work()
302 get_file(migf->filp); in mlx5vf_mig_file_set_save_work()
303 queue_work(migf->mvdev->cb_wq, in mlx5vf_mig_file_set_save_work()
304 &migf->save_data[chunk_num - 1].work); in mlx5vf_mig_file_set_save_work()
308 mlx5vf_mig_file_get_stop_copy_buf(struct mlx5_vf_migration_file *migf, in mlx5vf_mig_file_get_stop_copy_buf() argument
311 struct mlx5_vhca_data_buffer *buf = migf->buf[index]; in mlx5vf_mig_file_get_stop_copy_buf()
316 buf->migf->buf[index] = NULL; in mlx5vf_mig_file_get_stop_copy_buf()
322 buf = mlx5vf_get_data_buffer(buf->migf, required_length, in mlx5vf_mig_file_get_stop_copy_buf()
335 struct mlx5_vf_migration_file *migf = save_data->migf; in mlx5vf_mig_file_save_work() local
336 struct mlx5vf_pci_core_device *mvdev = migf->mvdev; in mlx5vf_mig_file_save_work()
340 if (migf->state == MLX5_MIGF_STATE_ERROR) in mlx5vf_mig_file_save_work()
343 buf = mlx5vf_mig_file_get_stop_copy_buf(migf, in mlx5vf_mig_file_save_work()
349 if (mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false)) in mlx5vf_mig_file_save_work()
357 mlx5vf_mark_err(migf); in mlx5vf_mig_file_save_work()
360 fput(migf->filp); in mlx5vf_mig_file_save_work()
363 static int mlx5vf_add_stop_copy_header(struct mlx5_vf_migration_file *migf, in mlx5vf_add_stop_copy_header() argument
376 header_buf = mlx5vf_get_data_buffer(migf, size, DMA_NONE); in mlx5vf_add_stop_copy_header()
391 data.stop_copy_size = cpu_to_le64(migf->buf[0]->allocated_length); in mlx5vf_add_stop_copy_header()
395 header_buf->start_pos = header_buf->migf->max_pos; in mlx5vf_add_stop_copy_header()
396 migf->max_pos += header_buf->length; in mlx5vf_add_stop_copy_header()
397 spin_lock_irqsave(&migf->list_lock, flags); in mlx5vf_add_stop_copy_header()
398 list_add_tail(&header_buf->buf_elm, &migf->buf_list); in mlx5vf_add_stop_copy_header()
399 spin_unlock_irqrestore(&migf->list_lock, flags); in mlx5vf_add_stop_copy_header()
401 migf->pre_copy_initial_bytes = size; in mlx5vf_add_stop_copy_header()
409 struct mlx5_vf_migration_file *migf, in mlx5vf_prep_stop_copy() argument
440 buf = mlx5vf_get_data_buffer(migf, inc_state_size, DMA_FROM_DEVICE); in mlx5vf_prep_stop_copy()
446 migf->buf[i] = buf; in mlx5vf_prep_stop_copy()
447 buf = mlx5vf_get_data_buffer(migf, in mlx5vf_prep_stop_copy()
453 migf->buf_header[i] = buf; in mlx5vf_prep_stop_copy()
455 migf->buf[i]->stop_copy_chunk_num = i + 1; in mlx5vf_prep_stop_copy()
456 migf->buf_header[i]->stop_copy_chunk_num = i + 1; in mlx5vf_prep_stop_copy()
457 INIT_WORK(&migf->save_data[i].work, in mlx5vf_prep_stop_copy()
459 migf->save_data[i].chunk_num = i + 1; in mlx5vf_prep_stop_copy()
463 ret = mlx5vf_add_stop_copy_header(migf, track); in mlx5vf_prep_stop_copy()
470 if (migf->buf[i]) { in mlx5vf_prep_stop_copy()
471 mlx5vf_put_data_buffer(migf->buf[i]); in mlx5vf_prep_stop_copy()
472 migf->buf[i] = NULL; in mlx5vf_prep_stop_copy()
474 if (migf->buf_header[i]) { in mlx5vf_prep_stop_copy()
475 mlx5vf_put_data_buffer(migf->buf_header[i]); in mlx5vf_prep_stop_copy()
476 migf->buf_header[i] = NULL; in mlx5vf_prep_stop_copy()
486 struct mlx5_vf_migration_file *migf = filp->private_data; in mlx5vf_precopy_ioctl() local
487 struct mlx5vf_pci_core_device *mvdev = migf->mvdev; in mlx5vf_precopy_ioctl()
531 mutex_lock(&migf->lock); in mlx5vf_precopy_ioctl()
532 if (migf->state == MLX5_MIGF_STATE_ERROR) { in mlx5vf_precopy_ioctl()
537 if (migf->pre_copy_initial_bytes > *pos) { in mlx5vf_precopy_ioctl()
538 info.initial_bytes = migf->pre_copy_initial_bytes - *pos; in mlx5vf_precopy_ioctl()
540 info.dirty_bytes = migf->max_pos - *pos; in mlx5vf_precopy_ioctl()
547 mutex_unlock(&migf->lock); in mlx5vf_precopy_ioctl()
551 mutex_unlock(&migf->lock); in mlx5vf_precopy_ioctl()
556 buf = mlx5vf_get_data_buffer(migf, inc_length, DMA_FROM_DEVICE); in mlx5vf_precopy_ioctl()
559 mlx5vf_mark_err(migf); in mlx5vf_precopy_ioctl()
563 ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, true); in mlx5vf_precopy_ioctl()
565 mlx5vf_mark_err(migf); in mlx5vf_precopy_ioctl()
577 mutex_unlock(&migf->lock); in mlx5vf_precopy_ioctl()
594 struct mlx5_vf_migration_file *migf = mvdev->saving_migf; in mlx5vf_pci_save_device_inc_data() local
599 if (migf->state == MLX5_MIGF_STATE_ERROR) in mlx5vf_pci_save_device_inc_data()
607 buf = mlx5vf_mig_file_get_stop_copy_buf(migf, 0, length); in mlx5vf_pci_save_device_inc_data()
613 ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false); in mlx5vf_pci_save_device_inc_data()
622 mlx5vf_mark_err(migf); in mlx5vf_pci_save_device_inc_data()
629 struct mlx5_vf_migration_file *migf; in mlx5vf_pci_save_device_data() local
635 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); in mlx5vf_pci_save_device_data()
636 if (!migf) in mlx5vf_pci_save_device_data()
639 migf->filp = anon_inode_getfile("mlx5vf_mig", &mlx5vf_save_fops, migf, in mlx5vf_pci_save_device_data()
641 if (IS_ERR(migf->filp)) { in mlx5vf_pci_save_device_data()
642 ret = PTR_ERR(migf->filp); in mlx5vf_pci_save_device_data()
643 kfree(migf); in mlx5vf_pci_save_device_data()
647 migf->mvdev = mvdev; in mlx5vf_pci_save_device_data()
648 stream_open(migf->filp->f_inode, migf->filp); in mlx5vf_pci_save_device_data()
649 mutex_init(&migf->lock); in mlx5vf_pci_save_device_data()
650 init_waitqueue_head(&migf->poll_wait); in mlx5vf_pci_save_device_data()
651 init_completion(&migf->save_comp); in mlx5vf_pci_save_device_data()
657 complete(&migf->save_comp); in mlx5vf_pci_save_device_data()
658 mlx5_cmd_init_async_ctx(mvdev->mdev, &migf->async_ctx); in mlx5vf_pci_save_device_data()
659 INIT_WORK(&migf->async_data.work, mlx5vf_mig_file_cleanup_cb); in mlx5vf_pci_save_device_data()
660 INIT_LIST_HEAD(&migf->buf_list); in mlx5vf_pci_save_device_data()
661 INIT_LIST_HEAD(&migf->avail_list); in mlx5vf_pci_save_device_data()
662 spin_lock_init(&migf->list_lock); in mlx5vf_pci_save_device_data()
664 ret = mlx5vf_cmd_alloc_pd(migf); in mlx5vf_pci_save_device_data()
672 ret = mlx5vf_prep_stop_copy(mvdev, migf, length, full_size, track); in mlx5vf_pci_save_device_data()
678 buf = mlx5vf_alloc_data_buffer(migf, in mlx5vf_pci_save_device_data()
679 migf->buf[0]->allocated_length, DMA_FROM_DEVICE); in mlx5vf_pci_save_device_data()
685 buf = migf->buf[0]; in mlx5vf_pci_save_device_data()
686 migf->buf[0] = NULL; in mlx5vf_pci_save_device_data()
689 ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, false, track); in mlx5vf_pci_save_device_data()
692 return migf; in mlx5vf_pci_save_device_data()
696 mlx5fv_cmd_clean_migf_resources(migf); in mlx5vf_pci_save_device_data()
698 fput(migf->filp); in mlx5vf_pci_save_device_data()
736 mlx5vf_resume_read_image(struct mlx5_vf_migration_file *migf, in mlx5vf_resume_read_image() argument
756 migf->load_state = MLX5_VF_LOAD_STATE_LOAD_IMAGE; in mlx5vf_resume_read_image()
757 migf->max_pos += image_size; in mlx5vf_resume_read_image()
765 mlx5vf_resume_read_header_data(struct mlx5_vf_migration_file *migf, in mlx5vf_resume_read_header_data() argument
775 required_data = migf->record_size - vhca_buf->length; in mlx5vf_resume_read_header_data()
786 if (vhca_buf->length == migf->record_size) { in mlx5vf_resume_read_header_data()
787 switch (migf->record_tag) { in mlx5vf_resume_read_header_data()
796 migf->stop_copy_prep_size = min_t(u64, in mlx5vf_resume_read_header_data()
806 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER; in mlx5vf_resume_read_header_data()
807 migf->max_pos += migf->record_size; in mlx5vf_resume_read_header_data()
815 mlx5vf_resume_read_header(struct mlx5_vf_migration_file *migf, in mlx5vf_resume_read_header() argument
853 migf->record_size = record_size; in mlx5vf_resume_read_header()
856 migf->record_tag = le32_to_cpup((__le32 *)(to_buff + in mlx5vf_resume_read_header()
858 switch (migf->record_tag) { in mlx5vf_resume_read_header()
860 migf->load_state = MLX5_VF_LOAD_STATE_PREP_IMAGE; in mlx5vf_resume_read_header()
863 migf->load_state = MLX5_VF_LOAD_STATE_PREP_HEADER_DATA; in mlx5vf_resume_read_header()
871 migf->load_state = MLX5_VF_LOAD_STATE_PREP_HEADER_DATA; in mlx5vf_resume_read_header()
874 migf->max_pos += vhca_buf->length; in mlx5vf_resume_read_header()
886 struct mlx5_vf_migration_file *migf = filp->private_data; in mlx5vf_resume_write() local
887 struct mlx5_vhca_data_buffer *vhca_buf = migf->buf[0]; in mlx5vf_resume_write()
888 struct mlx5_vhca_data_buffer *vhca_buf_header = migf->buf_header[0]; in mlx5vf_resume_write()
902 mutex_lock(&migf->mvdev->state_mutex); in mlx5vf_resume_write()
903 mutex_lock(&migf->lock); in mlx5vf_resume_write()
904 if (migf->state == MLX5_MIGF_STATE_ERROR) { in mlx5vf_resume_write()
911 switch (migf->load_state) { in mlx5vf_resume_write()
913 ret = mlx5vf_resume_read_header(migf, vhca_buf_header, in mlx5vf_resume_write()
920 if (vhca_buf_header->allocated_length < migf->record_size) { in mlx5vf_resume_write()
923 migf->buf_header[0] = mlx5vf_alloc_data_buffer(migf, in mlx5vf_resume_write()
924 migf->record_size, DMA_NONE); in mlx5vf_resume_write()
925 if (IS_ERR(migf->buf_header[0])) { in mlx5vf_resume_write()
926 ret = PTR_ERR(migf->buf_header[0]); in mlx5vf_resume_write()
927 migf->buf_header[0] = NULL; in mlx5vf_resume_write()
931 vhca_buf_header = migf->buf_header[0]; in mlx5vf_resume_write()
934 vhca_buf_header->start_pos = migf->max_pos; in mlx5vf_resume_write()
935 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER_DATA; in mlx5vf_resume_write()
938 ret = mlx5vf_resume_read_header_data(migf, vhca_buf_header, in mlx5vf_resume_write()
945 u64 size = max(migf->record_size, in mlx5vf_resume_write()
946 migf->stop_copy_prep_size); in mlx5vf_resume_write()
951 migf->buf[0] = mlx5vf_alloc_data_buffer(migf, in mlx5vf_resume_write()
953 if (IS_ERR(migf->buf[0])) { in mlx5vf_resume_write()
954 ret = PTR_ERR(migf->buf[0]); in mlx5vf_resume_write()
955 migf->buf[0] = NULL; in mlx5vf_resume_write()
959 vhca_buf = migf->buf[0]; in mlx5vf_resume_write()
962 vhca_buf->start_pos = migf->max_pos; in mlx5vf_resume_write()
963 migf->load_state = MLX5_VF_LOAD_STATE_READ_IMAGE; in mlx5vf_resume_write()
967 ret = mlx5vf_resume_read_image(migf, vhca_buf, in mlx5vf_resume_write()
968 migf->record_size, in mlx5vf_resume_write()
974 ret = mlx5vf_cmd_load_vhca_state(migf->mvdev, migf, vhca_buf); in mlx5vf_resume_write()
977 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER; in mlx5vf_resume_write()
992 migf->state = MLX5_MIGF_STATE_ERROR; in mlx5vf_resume_write()
993 mutex_unlock(&migf->lock); in mlx5vf_resume_write()
994 mlx5vf_state_mutex_unlock(migf->mvdev); in mlx5vf_resume_write()
1007 struct mlx5_vf_migration_file *migf; in mlx5vf_pci_resume_device_data() local
1011 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); in mlx5vf_pci_resume_device_data()
1012 if (!migf) in mlx5vf_pci_resume_device_data()
1015 migf->filp = anon_inode_getfile("mlx5vf_mig", &mlx5vf_resume_fops, migf, in mlx5vf_pci_resume_device_data()
1017 if (IS_ERR(migf->filp)) { in mlx5vf_pci_resume_device_data()
1018 ret = PTR_ERR(migf->filp); in mlx5vf_pci_resume_device_data()
1019 kfree(migf); in mlx5vf_pci_resume_device_data()
1023 stream_open(migf->filp->f_inode, migf->filp); in mlx5vf_pci_resume_device_data()
1024 mutex_init(&migf->lock); in mlx5vf_pci_resume_device_data()
1025 INIT_LIST_HEAD(&migf->buf_list); in mlx5vf_pci_resume_device_data()
1026 INIT_LIST_HEAD(&migf->avail_list); in mlx5vf_pci_resume_device_data()
1027 spin_lock_init(&migf->list_lock); in mlx5vf_pci_resume_device_data()
1028 migf->mvdev = mvdev; in mlx5vf_pci_resume_device_data()
1029 ret = mlx5vf_cmd_alloc_pd(migf); in mlx5vf_pci_resume_device_data()
1033 buf = mlx5vf_alloc_data_buffer(migf, 0, DMA_TO_DEVICE); in mlx5vf_pci_resume_device_data()
1039 migf->buf[0] = buf; in mlx5vf_pci_resume_device_data()
1040 buf = mlx5vf_alloc_data_buffer(migf, in mlx5vf_pci_resume_device_data()
1047 migf->buf_header[0] = buf; in mlx5vf_pci_resume_device_data()
1048 migf->load_state = MLX5_VF_LOAD_STATE_READ_HEADER; in mlx5vf_pci_resume_device_data()
1050 return migf; in mlx5vf_pci_resume_device_data()
1052 mlx5vf_free_data_buffer(migf->buf[0]); in mlx5vf_pci_resume_device_data()
1054 mlx5vf_cmd_dealloc_pd(migf); in mlx5vf_pci_resume_device_data()
1056 fput(migf->filp); in mlx5vf_pci_resume_device_data()
1124 struct mlx5_vf_migration_file *migf; in mlx5vf_pci_step_device_state_locked() local
1126 migf = mlx5vf_pci_save_device_data(mvdev, false); in mlx5vf_pci_step_device_state_locked()
1127 if (IS_ERR(migf)) in mlx5vf_pci_step_device_state_locked()
1128 return ERR_CAST(migf); in mlx5vf_pci_step_device_state_locked()
1129 get_file(migf->filp); in mlx5vf_pci_step_device_state_locked()
1130 mvdev->saving_migf = migf; in mlx5vf_pci_step_device_state_locked()
1131 return migf->filp; in mlx5vf_pci_step_device_state_locked()
1142 struct mlx5_vf_migration_file *migf = mvdev->saving_migf; in mlx5vf_pci_step_device_state_locked() local
1151 buf = mlx5vf_get_data_buffer(migf, size, DMA_FROM_DEVICE); in mlx5vf_pci_step_device_state_locked()
1155 ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, false, false); in mlx5vf_pci_step_device_state_locked()
1165 struct mlx5_vf_migration_file *migf; in mlx5vf_pci_step_device_state_locked() local
1167 migf = mlx5vf_pci_resume_device_data(mvdev); in mlx5vf_pci_step_device_state_locked()
1168 if (IS_ERR(migf)) in mlx5vf_pci_step_device_state_locked()
1169 return ERR_CAST(migf); in mlx5vf_pci_step_device_state_locked()
1170 get_file(migf->filp); in mlx5vf_pci_step_device_state_locked()
1171 mvdev->resuming_migf = migf; in mlx5vf_pci_step_device_state_locked()
1172 return migf->filp; in mlx5vf_pci_step_device_state_locked()
1183 struct mlx5_vf_migration_file *migf; in mlx5vf_pci_step_device_state_locked() local
1185 migf = mlx5vf_pci_save_device_data(mvdev, true); in mlx5vf_pci_step_device_state_locked()
1186 if (IS_ERR(migf)) in mlx5vf_pci_step_device_state_locked()
1187 return ERR_CAST(migf); in mlx5vf_pci_step_device_state_locked()
1188 get_file(migf->filp); in mlx5vf_pci_step_device_state_locked()
1189 mvdev->saving_migf = migf; in mlx5vf_pci_step_device_state_locked()
1190 return migf->filp; in mlx5vf_pci_step_device_state_locked()