1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 */ 5 6 #ifndef MLX5_VFIO_CMD_H 7 #define MLX5_VFIO_CMD_H 8 9 #include <linux/kernel.h> 10 #include <linux/vfio_pci_core.h> 11 #include <linux/mlx5/driver.h> 12 #include <linux/mlx5/vport.h> 13 #include <linux/mlx5/cq.h> 14 #include <linux/mlx5/qp.h> 15 16 enum mlx5_vf_migf_state { 17 MLX5_MIGF_STATE_ERROR = 1, 18 MLX5_MIGF_STATE_PRE_COPY_ERROR, 19 MLX5_MIGF_STATE_PRE_COPY, 20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, 21 MLX5_MIGF_STATE_COMPLETE, 22 }; 23 24 enum mlx5_vf_load_state { 25 MLX5_VF_LOAD_STATE_READ_HEADER, 26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA, 27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA, 28 MLX5_VF_LOAD_STATE_PREP_IMAGE, 29 MLX5_VF_LOAD_STATE_READ_IMAGE, 30 MLX5_VF_LOAD_STATE_LOAD_IMAGE, 31 }; 32 33 struct mlx5_vf_migration_tag_stop_copy_data { 34 __le64 stop_copy_size; 35 }; 36 37 enum mlx5_vf_migf_header_flags { 38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, 39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0, 40 }; 41 42 enum mlx5_vf_migf_header_tag { 43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0, 44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0, 45 }; 46 47 struct mlx5_vf_migration_header { 48 __le64 record_size; 49 /* For future use in case we may need to change the kernel protocol */ 50 __le32 flags; /* Use mlx5_vf_migf_header_flags */ 51 __le32 tag; /* Use mlx5_vf_migf_header_tag */ 52 __u8 data[]; /* Its size is given in the record_size */ 53 }; 54 55 struct mlx5_vhca_data_buffer { 56 struct page **page_list; 57 struct dma_iova_state state; 58 loff_t start_pos; 59 u64 length; 60 u32 npages; 61 u32 mkey; 62 u32 *mkey_in; 63 enum dma_data_direction dma_dir; 64 u8 stop_copy_chunk_num; 65 bool pre_copy_init_bytes_chunk; 66 struct list_head buf_elm; 67 struct mlx5_vf_migration_file *migf; 68 }; 69 70 struct mlx5vf_async_data { 71 struct mlx5_async_work cb_work; 72 struct work_struct work; 73 struct mlx5_vhca_data_buffer *buf; 74 struct mlx5_vhca_data_buffer *header_buf; 75 int status; 76 u8 stop_copy_chunk:1; 77 void *out; 78 }; 79 80 struct mlx5vf_save_work_data { 81 struct mlx5_vf_migration_file *migf; 82 size_t next_required_umem_size; 83 struct work_struct work; 84 u8 chunk_num; 85 }; 86 87 #define MAX_NUM_CHUNKS 2 88 89 struct mlx5_vf_migration_file { 90 struct file *filp; 91 struct mutex lock; 92 enum mlx5_vf_migf_state state; 93 94 enum mlx5_vf_load_state load_state; 95 u32 pdn; 96 loff_t max_pos; 97 u64 record_size; 98 u32 record_tag; 99 u64 stop_copy_prep_size; 100 u64 pre_copy_initial_bytes; 101 u64 pre_copy_initial_bytes_start; 102 size_t next_required_umem_size; 103 u8 num_ready_chunks; 104 /* Upon chunk mode preserve another set of buffers for stop_copy phase */ 105 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; 106 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS]; 107 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; 108 spinlock_t list_lock; 109 struct list_head buf_list; 110 struct list_head avail_list; 111 struct mlx5vf_pci_core_device *mvdev; 112 wait_queue_head_t poll_wait; 113 struct completion save_comp; 114 struct mlx5_async_ctx async_ctx; 115 struct mlx5vf_async_data async_data; 116 u8 inflight_save:1; 117 }; 118 119 struct mlx5_vhca_cq_buf { 120 struct mlx5_frag_buf_ctrl fbc; 121 struct mlx5_frag_buf frag_buf; 122 int cqe_size; 123 int nent; 124 }; 125 126 struct mlx5_vhca_cq { 127 struct mlx5_vhca_cq_buf buf; 128 struct mlx5_db db; 129 struct mlx5_core_cq mcq; 130 size_t ncqe; 131 }; 132 133 struct mlx5_vhca_recv_buf { 134 u32 npages; 135 struct page **page_list; 136 struct dma_iova_state state; 137 u32 next_rq_offset; 138 u32 *mkey_in; 139 u32 mkey; 140 }; 141 142 struct mlx5_vhca_qp { 143 struct mlx5_frag_buf buf; 144 struct mlx5_db db; 145 struct mlx5_vhca_recv_buf recv_buf; 146 u32 tracked_page_size; 147 u32 max_msg_size; 148 u32 qpn; 149 struct { 150 unsigned int pc; 151 unsigned int cc; 152 unsigned int wqe_cnt; 153 __be32 *db; 154 struct mlx5_frag_buf_ctrl fbc; 155 } rq; 156 }; 157 158 struct mlx5_vhca_page_tracker { 159 u32 id; 160 u32 pdn; 161 u8 is_err:1; 162 u8 object_changed:1; 163 struct mlx5_uars_page *uar; 164 struct mlx5_vhca_cq cq; 165 struct mlx5_vhca_qp *host_qp; 166 struct mlx5_vhca_qp *fw_qp; 167 struct mlx5_nb nb; 168 int status; 169 }; 170 171 struct mlx5vf_pci_core_device { 172 struct vfio_pci_core_device core_device; 173 int vf_id; 174 u16 vhca_id; 175 u8 migrate_cap:1; 176 u8 deferred_reset:1; 177 u8 mdev_detach:1; 178 u8 log_active:1; 179 u8 chunk_mode:1; 180 u8 mig_state_cap:1; 181 struct completion tracker_comp; 182 /* protect migration state */ 183 struct mutex state_mutex; 184 enum vfio_device_mig_state mig_state; 185 /* protect the reset_done flow */ 186 spinlock_t reset_lock; 187 struct mlx5_vf_migration_file *resuming_migf; 188 struct mlx5_vf_migration_file *saving_migf; 189 struct mlx5_vhca_page_tracker tracker; 190 struct workqueue_struct *cb_wq; 191 struct notifier_block nb; 192 struct mlx5_core_dev *mdev; 193 }; 194 195 enum { 196 MLX5VF_QUERY_INC = (1UL << 0), 197 MLX5VF_QUERY_FINAL = (1UL << 1), 198 MLX5VF_QUERY_CLEANUP = (1UL << 2), 199 }; 200 201 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 202 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 203 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, 204 size_t *state_size, u64 *total_size, 205 u8 *migration_state, u8 query_flags); 206 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, 207 const struct vfio_migration_ops *mig_ops, 208 const struct vfio_log_ops *log_ops); 209 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); 210 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); 211 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, 212 struct mlx5_vf_migration_file *migf, 213 struct mlx5_vhca_data_buffer *buf, bool inc, 214 bool track); 215 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, 216 struct mlx5_vf_migration_file *migf, 217 struct mlx5_vhca_data_buffer *buf); 218 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); 219 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); 220 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); 221 struct mlx5_vhca_data_buffer * 222 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, 223 enum dma_data_direction dma_dir); 224 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); 225 struct mlx5_vhca_data_buffer * 226 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, 227 enum dma_data_direction dma_dir); 228 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); 229 static inline struct page * 230 mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, 231 unsigned long offset) 232 { 233 int page_entry = offset / PAGE_SIZE; 234 235 if (page_entry >= buf->npages) 236 return NULL; 237 238 return buf->page_list[page_entry]; 239 } 240 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); 241 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev, 242 enum mlx5_vf_migf_state *last_save_state); 243 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); 244 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, 245 u8 chunk_num, size_t next_required_umem_size); 246 int mlx5vf_start_page_tracker(struct vfio_device *vdev, 247 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); 248 int mlx5vf_stop_page_tracker(struct vfio_device *vdev); 249 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, 250 unsigned long length, struct iova_bitmap *dirty); 251 #endif /* MLX5_VFIO_CMD_H */ 252