40 	struct mlx5_vf_migration_file *migf = mvdev->saving_migf;  in mlx5vf_cmd_suspend_vhca()  local
55 	if (migf) {  in mlx5vf_cmd_suspend_vhca()
56 		err = wait_for_completion_interruptible(&migf->save_comp);  in mlx5vf_cmd_suspend_vhca()
66 	if (migf)  in mlx5vf_cmd_suspend_vhca()
67 		complete(&migf->save_comp);  in mlx5vf_cmd_suspend_vhca()
370 	struct mlx5vf_pci_core_device *mvdev = buf->migf->mvdev;  in mlx5vf_dma_data_buffer()
385 	ret = _create_mkey(mdev, buf->migf->pdn, buf, NULL, &buf->mkey);  in mlx5vf_dma_data_buffer()
399 	struct mlx5_vf_migration_file *migf = buf->migf;  in mlx5vf_free_data_buffer()  local
402 	lockdep_assert_held(&migf->mvdev->state_mutex);  in mlx5vf_free_data_buffer()
403 	WARN_ON(migf->mvdev->mdev_detach);  in mlx5vf_free_data_buffer()
406 		mlx5_core_destroy_mkey(migf->mvdev->mdev, buf->mkey);  in mlx5vf_free_data_buffer()
407 		dma_unmap_sgtable(migf->mvdev->mdev->device, &buf->table.sgt,  in mlx5vf_free_data_buffer()
463 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf,  in mlx5vf_alloc_data_buffer()  argument
475 	buf->migf = migf;  in mlx5vf_alloc_data_buffer()
497 	spin_lock_irq(&buf->migf->list_lock);  in mlx5vf_put_data_buffer()
499 	list_add_tail(&buf->buf_elm, &buf->migf->avail_list);  in mlx5vf_put_data_buffer()
500 	spin_unlock_irq(&buf->migf->list_lock);  in mlx5vf_put_data_buffer()
504 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf,  in mlx5vf_get_data_buffer()  argument
510 	lockdep_assert_held(&migf->mvdev->state_mutex);  in mlx5vf_get_data_buffer()
511 	if (migf->mvdev->mdev_detach)  in mlx5vf_get_data_buffer()
516 	spin_lock_irq(&migf->list_lock);  in mlx5vf_get_data_buffer()
517 	list_for_each_entry_safe(buf, temp_buf, &migf->avail_list, buf_elm) {  in mlx5vf_get_data_buffer()
521 				spin_unlock_irq(&migf->list_lock);  in mlx5vf_get_data_buffer()
533 	spin_unlock_irq(&migf->list_lock);  in mlx5vf_get_data_buffer()
534 	buf = mlx5vf_alloc_data_buffer(migf, length, dma_dir);  in mlx5vf_get_data_buffer()
547 mlx5vf_save_callback_complete(struct mlx5_vf_migration_file *migf,  in mlx5vf_save_callback_complete()  argument
551 	complete(&migf->save_comp);  in mlx5vf_save_callback_complete()
552 	fput(migf->filp);  in mlx5vf_save_callback_complete()
559 	struct mlx5_vf_migration_file *migf = container_of(async_data,  in mlx5vf_mig_file_cleanup_cb()  local
562 	mutex_lock(&migf->lock);  in mlx5vf_mig_file_cleanup_cb()
569 			migf->state = MLX5_MIGF_STATE_PRE_COPY_ERROR;  in mlx5vf_mig_file_cleanup_cb()
571 			migf->state = MLX5_MIGF_STATE_ERROR;  in mlx5vf_mig_file_cleanup_cb()
572 		wake_up_interruptible(&migf->poll_wait);  in mlx5vf_mig_file_cleanup_cb()
574 	mutex_unlock(&migf->lock);  in mlx5vf_mig_file_cleanup_cb()
575 	mlx5vf_save_callback_complete(migf, async_data);  in mlx5vf_mig_file_cleanup_cb()
581 	struct mlx5_vf_migration_file *migf = header_buf->migf;  in add_buf_header()  local
597 	header_buf->start_pos = header_buf->migf->max_pos;  in add_buf_header()
598 	migf->max_pos += header_buf->length;  in add_buf_header()
599 	spin_lock_irqsave(&migf->list_lock, flags);  in add_buf_header()
600 	list_add_tail(&header_buf->buf_elm, &migf->buf_list);  in add_buf_header()
601 	spin_unlock_irqrestore(&migf->list_lock, flags);  in add_buf_header()
603 		migf->pre_copy_initial_bytes += sizeof(header);  in add_buf_header()
611 	struct mlx5_vf_migration_file *migf = container_of(async_data,  in mlx5vf_save_callback()  local
619 		bool initial_pre_copy = migf->state != MLX5_MIGF_STATE_PRE_COPY &&  in mlx5vf_save_callback()
636 		async_data->buf->start_pos = migf->max_pos;  in mlx5vf_save_callback()
637 		migf->max_pos += async_data->buf->length;  in mlx5vf_save_callback()
638 		spin_lock_irqsave(&migf->list_lock, flags);  in mlx5vf_save_callback()
639 		list_add_tail(&async_data->buf->buf_elm, &migf->buf_list);  in mlx5vf_save_callback()
641 			migf->num_ready_chunks++;  in mlx5vf_save_callback()
643 			    migf->num_ready_chunks >= MAX_NUM_CHUNKS) {  in mlx5vf_save_callback()
645 				migf->next_required_umem_size = next_required_umem_size;  in mlx5vf_save_callback()
649 		spin_unlock_irqrestore(&migf->list_lock, flags);  in mlx5vf_save_callback()
651 			migf->pre_copy_initial_bytes += image_size;  in mlx5vf_save_callback()
652 			migf->state = MLX5_MIGF_STATE_PRE_COPY;  in mlx5vf_save_callback()
655 			migf->state = MLX5_MIGF_STATE_COMPLETE;  in mlx5vf_save_callback()
656 		wake_up_interruptible(&migf->poll_wait);  in mlx5vf_save_callback()
658 			mlx5vf_mig_file_set_save_work(migf,  in mlx5vf_save_callback()
662 		mlx5vf_save_callback_complete(migf, async_data);  in mlx5vf_save_callback()
671 		mlx5_cmd_out_err(migf->mvdev->mdev, MLX5_CMD_OP_SAVE_VHCA_STATE, 0,  in mlx5vf_save_callback()
676 	queue_work(migf->mvdev->cb_wq, &async_data->work);  in mlx5vf_save_callback()
680 			       struct mlx5_vf_migration_file *migf,  in mlx5vf_cmd_save_vhca_state()  argument
695 	err = wait_for_completion_interruptible(&migf->save_comp);  in mlx5vf_cmd_save_vhca_state()
699 	if ((migf->state == MLX5_MIGF_STATE_PRE_COPY ||  in mlx5vf_cmd_save_vhca_state()
700 	     migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR) && !track && !inc)  in mlx5vf_cmd_save_vhca_state()
703 	if (migf->state == MLX5_MIGF_STATE_PRE_COPY_ERROR)  in mlx5vf_cmd_save_vhca_state()
719 	async_data = &migf->async_data;  in mlx5vf_cmd_save_vhca_state()
732 		header_buf = migf->buf_header[header_idx];  in mlx5vf_cmd_save_vhca_state()
733 		migf->buf_header[header_idx] = NULL;  in mlx5vf_cmd_save_vhca_state()
737 		header_buf = mlx5vf_get_data_buffer(migf,  in mlx5vf_cmd_save_vhca_state()
746 		migf->state = MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK;  in mlx5vf_cmd_save_vhca_state()
749 	get_file(migf->filp);  in mlx5vf_cmd_save_vhca_state()
750 	err = mlx5_cmd_exec_cb(&migf->async_ctx, in, sizeof(in),  in mlx5vf_cmd_save_vhca_state()
762 	fput(migf->filp);  in mlx5vf_cmd_save_vhca_state()
766 	complete(&migf->save_comp);  in mlx5vf_cmd_save_vhca_state()
771 			       struct mlx5_vf_migration_file *migf,  in mlx5vf_cmd_load_vhca_state()  argument
797 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf)  in mlx5vf_cmd_alloc_pd()  argument
801 	lockdep_assert_held(&migf->mvdev->state_mutex);  in mlx5vf_cmd_alloc_pd()
802 	if (migf->mvdev->mdev_detach)  in mlx5vf_cmd_alloc_pd()
805 	err = mlx5_core_alloc_pd(migf->mvdev->mdev, &migf->pdn);  in mlx5vf_cmd_alloc_pd()
809 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf)  in mlx5vf_cmd_dealloc_pd()  argument
811 	lockdep_assert_held(&migf->mvdev->state_mutex);  in mlx5vf_cmd_dealloc_pd()
812 	if (migf->mvdev->mdev_detach)  in mlx5vf_cmd_dealloc_pd()
815 	mlx5_core_dealloc_pd(migf->mvdev->mdev, migf->pdn);  in mlx5vf_cmd_dealloc_pd()
818 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf)  in mlx5fv_cmd_clean_migf_resources()  argument
823 	lockdep_assert_held(&migf->mvdev->state_mutex);  in mlx5fv_cmd_clean_migf_resources()
824 	WARN_ON(migf->mvdev->mdev_detach);  in mlx5fv_cmd_clean_migf_resources()
827 		if (migf->buf[i]) {  in mlx5fv_cmd_clean_migf_resources()
828 			mlx5vf_free_data_buffer(migf->buf[i]);  in mlx5fv_cmd_clean_migf_resources()
829 			migf->buf[i] = NULL;  in mlx5fv_cmd_clean_migf_resources()
832 		if (migf->buf_header[i]) {  in mlx5fv_cmd_clean_migf_resources()
833 			mlx5vf_free_data_buffer(migf->buf_header[i]);  in mlx5fv_cmd_clean_migf_resources()
834 			migf->buf_header[i] = NULL;  in mlx5fv_cmd_clean_migf_resources()
838 	list_splice(&migf->avail_list, &migf->buf_list);  in mlx5fv_cmd_clean_migf_resources()
840 	while ((entry = list_first_entry_or_null(&migf->buf_list,  in mlx5fv_cmd_clean_migf_resources()
846 	mlx5vf_cmd_dealloc_pd(migf);  in mlx5fv_cmd_clean_migf_resources()