1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 */ 5 6 #ifndef MLX5_VFIO_CMD_H 7 #define MLX5_VFIO_CMD_H 8 9 #include <linux/kernel.h> 10 #include <linux/vfio_pci_core.h> 11 #include <linux/mlx5/driver.h> 12 #include <linux/mlx5/vport.h> 13 #include <linux/mlx5/cq.h> 14 #include <linux/mlx5/qp.h> 15 16 #define MLX5VF_PRE_COPY_SUPP(mvdev) \ 17 ((mvdev)->core_device.vdev.migration_flags & VFIO_MIGRATION_PRE_COPY) 18 19 enum mlx5_vf_migf_state { 20 MLX5_MIGF_STATE_ERROR = 1, 21 MLX5_MIGF_STATE_PRE_COPY_ERROR, 22 MLX5_MIGF_STATE_PRE_COPY, 23 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, 24 MLX5_MIGF_STATE_COMPLETE, 25 }; 26 27 enum mlx5_vf_load_state { 28 MLX5_VF_LOAD_STATE_READ_IMAGE_NO_HEADER, 29 MLX5_VF_LOAD_STATE_READ_HEADER, 30 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA, 31 MLX5_VF_LOAD_STATE_READ_HEADER_DATA, 32 MLX5_VF_LOAD_STATE_PREP_IMAGE, 33 MLX5_VF_LOAD_STATE_READ_IMAGE, 34 MLX5_VF_LOAD_STATE_LOAD_IMAGE, 35 }; 36 37 struct mlx5_vf_migration_tag_stop_copy_data { 38 __le64 stop_copy_size; 39 }; 40 41 enum mlx5_vf_migf_header_flags { 42 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, 43 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0, 44 }; 45 46 enum mlx5_vf_migf_header_tag { 47 MLX5_MIGF_HEADER_TAG_FW_DATA = 0, 48 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0, 49 }; 50 51 struct mlx5_vf_migration_header { 52 __le64 record_size; 53 /* For future use in case we may need to change the kernel protocol */ 54 __le32 flags; /* Use mlx5_vf_migf_header_flags */ 55 __le32 tag; /* Use mlx5_vf_migf_header_tag */ 56 __u8 data[]; /* Its size is given in the record_size */ 57 }; 58 59 struct mlx5_vhca_data_buffer { 60 struct sg_append_table table; 61 loff_t start_pos; 62 u64 length; 63 u64 allocated_length; 64 u32 mkey; 65 enum dma_data_direction dma_dir; 66 u8 dmaed:1; 67 u8 stop_copy_chunk_num; 68 struct list_head buf_elm; 69 struct mlx5_vf_migration_file *migf; 70 /* Optimize mlx5vf_get_migration_page() for sequential access */ 71 struct scatterlist *last_offset_sg; 72 unsigned int sg_last_entry; 73 unsigned long last_offset; 74 }; 75 76 struct mlx5vf_async_data { 77 struct mlx5_async_work cb_work; 78 struct work_struct work; 79 struct mlx5_vhca_data_buffer *buf; 80 struct mlx5_vhca_data_buffer *header_buf; 81 int status; 82 u8 stop_copy_chunk:1; 83 void *out; 84 }; 85 86 struct mlx5vf_save_work_data { 87 struct mlx5_vf_migration_file *migf; 88 size_t next_required_umem_size; 89 struct work_struct work; 90 u8 chunk_num; 91 }; 92 93 #define MAX_NUM_CHUNKS 2 94 95 struct mlx5_vf_migration_file { 96 struct file *filp; 97 struct mutex lock; 98 enum mlx5_vf_migf_state state; 99 100 enum mlx5_vf_load_state load_state; 101 u32 pdn; 102 loff_t max_pos; 103 u64 record_size; 104 u32 record_tag; 105 u64 stop_copy_prep_size; 106 u64 pre_copy_initial_bytes; 107 size_t next_required_umem_size; 108 u8 num_ready_chunks; 109 /* Upon chunk mode preserve another set of buffers for stop_copy phase */ 110 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; 111 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS]; 112 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; 113 spinlock_t list_lock; 114 struct list_head buf_list; 115 struct list_head avail_list; 116 struct mlx5vf_pci_core_device *mvdev; 117 wait_queue_head_t poll_wait; 118 struct completion save_comp; 119 struct mlx5_async_ctx async_ctx; 120 struct mlx5vf_async_data async_data; 121 }; 122 123 struct mlx5_vhca_cq_buf { 124 struct mlx5_frag_buf_ctrl fbc; 125 struct mlx5_frag_buf frag_buf; 126 int cqe_size; 127 int nent; 128 }; 129 130 struct mlx5_vhca_cq { 131 struct mlx5_vhca_cq_buf buf; 132 struct mlx5_db db; 133 struct mlx5_core_cq mcq; 134 size_t ncqe; 135 }; 136 137 struct mlx5_vhca_recv_buf { 138 u32 npages; 139 struct page **page_list; 140 dma_addr_t *dma_addrs; 141 u32 next_rq_offset; 142 u32 mkey; 143 }; 144 145 struct mlx5_vhca_qp { 146 struct mlx5_frag_buf buf; 147 struct mlx5_db db; 148 struct mlx5_vhca_recv_buf recv_buf; 149 u32 tracked_page_size; 150 u32 max_msg_size; 151 u32 qpn; 152 struct { 153 unsigned int pc; 154 unsigned int cc; 155 unsigned int wqe_cnt; 156 __be32 *db; 157 struct mlx5_frag_buf_ctrl fbc; 158 } rq; 159 }; 160 161 struct mlx5_vhca_page_tracker { 162 u32 id; 163 u32 pdn; 164 u8 is_err:1; 165 struct mlx5_uars_page *uar; 166 struct mlx5_vhca_cq cq; 167 struct mlx5_vhca_qp *host_qp; 168 struct mlx5_vhca_qp *fw_qp; 169 struct mlx5_nb nb; 170 int status; 171 }; 172 173 struct mlx5vf_pci_core_device { 174 struct vfio_pci_core_device core_device; 175 int vf_id; 176 u16 vhca_id; 177 u8 migrate_cap:1; 178 u8 deferred_reset:1; 179 u8 mdev_detach:1; 180 u8 log_active:1; 181 u8 chunk_mode:1; 182 struct completion tracker_comp; 183 /* protect migration state */ 184 struct mutex state_mutex; 185 enum vfio_device_mig_state mig_state; 186 /* protect the reset_done flow */ 187 spinlock_t reset_lock; 188 struct mlx5_vf_migration_file *resuming_migf; 189 struct mlx5_vf_migration_file *saving_migf; 190 struct mlx5_vhca_page_tracker tracker; 191 struct workqueue_struct *cb_wq; 192 struct notifier_block nb; 193 struct mlx5_core_dev *mdev; 194 }; 195 196 enum { 197 MLX5VF_QUERY_INC = (1UL << 0), 198 MLX5VF_QUERY_FINAL = (1UL << 1), 199 }; 200 201 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 202 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 203 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, 204 size_t *state_size, u64 *total_size, 205 u8 query_flags); 206 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, 207 const struct vfio_migration_ops *mig_ops, 208 const struct vfio_log_ops *log_ops); 209 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); 210 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); 211 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, 212 struct mlx5_vf_migration_file *migf, 213 struct mlx5_vhca_data_buffer *buf, bool inc, 214 bool track); 215 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, 216 struct mlx5_vf_migration_file *migf, 217 struct mlx5_vhca_data_buffer *buf); 218 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); 219 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); 220 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); 221 struct mlx5_vhca_data_buffer * 222 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, 223 size_t length, enum dma_data_direction dma_dir); 224 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); 225 struct mlx5_vhca_data_buffer * 226 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, 227 size_t length, enum dma_data_direction dma_dir); 228 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); 229 int mlx5vf_add_migration_pages(struct mlx5_vhca_data_buffer *buf, 230 unsigned int npages); 231 struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, 232 unsigned long offset); 233 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); 234 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev); 235 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); 236 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, 237 u8 chunk_num, size_t next_required_umem_size); 238 int mlx5vf_start_page_tracker(struct vfio_device *vdev, 239 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); 240 int mlx5vf_stop_page_tracker(struct vfio_device *vdev); 241 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, 242 unsigned long length, struct iova_bitmap *dirty); 243 #endif /* MLX5_VFIO_CMD_H */ 244