1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 */ 5 6 #ifndef MLX5_VFIO_CMD_H 7 #define MLX5_VFIO_CMD_H 8 9 #include <linux/kernel.h> 10 #include <linux/vfio_pci_core.h> 11 #include <linux/mlx5/driver.h> 12 #include <linux/mlx5/vport.h> 13 #include <linux/mlx5/cq.h> 14 #include <linux/mlx5/qp.h> 15 16 enum mlx5_vf_migf_state { 17 MLX5_MIGF_STATE_ERROR = 1, 18 MLX5_MIGF_STATE_PRE_COPY_ERROR, 19 MLX5_MIGF_STATE_PRE_COPY, 20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, 21 MLX5_MIGF_STATE_COMPLETE, 22 }; 23 24 enum mlx5_vf_load_state { 25 MLX5_VF_LOAD_STATE_READ_HEADER, 26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA, 27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA, 28 MLX5_VF_LOAD_STATE_PREP_IMAGE, 29 MLX5_VF_LOAD_STATE_READ_IMAGE, 30 MLX5_VF_LOAD_STATE_LOAD_IMAGE, 31 }; 32 33 struct mlx5_vf_migration_tag_stop_copy_data { 34 __le64 stop_copy_size; 35 }; 36 37 enum mlx5_vf_migf_header_flags { 38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, 39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0, 40 }; 41 42 enum mlx5_vf_migf_header_tag { 43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0, 44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0, 45 }; 46 47 struct mlx5_vf_migration_header { 48 __le64 record_size; 49 /* For future use in case we may need to change the kernel protocol */ 50 __le32 flags; /* Use mlx5_vf_migf_header_flags */ 51 __le32 tag; /* Use mlx5_vf_migf_header_tag */ 52 __u8 data[]; /* Its size is given in the record_size */ 53 }; 54 55 struct mlx5_vhca_data_buffer { 56 struct page **page_list; 57 struct dma_iova_state state; 58 loff_t start_pos; 59 u64 length; 60 u32 npages; 61 u32 mkey; 62 u32 *mkey_in; 63 enum dma_data_direction dma_dir; 64 u8 stop_copy_chunk_num; 65 struct list_head buf_elm; 66 struct mlx5_vf_migration_file *migf; 67 }; 68 69 struct mlx5vf_async_data { 70 struct mlx5_async_work cb_work; 71 struct work_struct work; 72 struct mlx5_vhca_data_buffer *buf; 73 struct mlx5_vhca_data_buffer *header_buf; 74 int status; 75 u8 stop_copy_chunk:1; 76 void *out; 77 }; 78 79 struct mlx5vf_save_work_data { 80 struct mlx5_vf_migration_file *migf; 81 size_t next_required_umem_size; 82 struct work_struct work; 83 u8 chunk_num; 84 }; 85 86 #define MAX_NUM_CHUNKS 2 87 88 struct mlx5_vf_migration_file { 89 struct file *filp; 90 struct mutex lock; 91 enum mlx5_vf_migf_state state; 92 93 enum mlx5_vf_load_state load_state; 94 u32 pdn; 95 loff_t max_pos; 96 u64 record_size; 97 u32 record_tag; 98 u64 stop_copy_prep_size; 99 u64 pre_copy_initial_bytes; 100 size_t next_required_umem_size; 101 u8 num_ready_chunks; 102 /* Upon chunk mode preserve another set of buffers for stop_copy phase */ 103 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; 104 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS]; 105 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; 106 spinlock_t list_lock; 107 struct list_head buf_list; 108 struct list_head avail_list; 109 struct mlx5vf_pci_core_device *mvdev; 110 wait_queue_head_t poll_wait; 111 struct completion save_comp; 112 struct mlx5_async_ctx async_ctx; 113 struct mlx5vf_async_data async_data; 114 }; 115 116 struct mlx5_vhca_cq_buf { 117 struct mlx5_frag_buf_ctrl fbc; 118 struct mlx5_frag_buf frag_buf; 119 int cqe_size; 120 int nent; 121 }; 122 123 struct mlx5_vhca_cq { 124 struct mlx5_vhca_cq_buf buf; 125 struct mlx5_db db; 126 struct mlx5_core_cq mcq; 127 size_t ncqe; 128 }; 129 130 struct mlx5_vhca_recv_buf { 131 u32 npages; 132 struct page **page_list; 133 struct dma_iova_state state; 134 u32 next_rq_offset; 135 u32 *mkey_in; 136 u32 mkey; 137 }; 138 139 struct mlx5_vhca_qp { 140 struct mlx5_frag_buf buf; 141 struct mlx5_db db; 142 struct mlx5_vhca_recv_buf recv_buf; 143 u32 tracked_page_size; 144 u32 max_msg_size; 145 u32 qpn; 146 struct { 147 unsigned int pc; 148 unsigned int cc; 149 unsigned int wqe_cnt; 150 __be32 *db; 151 struct mlx5_frag_buf_ctrl fbc; 152 } rq; 153 }; 154 155 struct mlx5_vhca_page_tracker { 156 u32 id; 157 u32 pdn; 158 u8 is_err:1; 159 u8 object_changed:1; 160 struct mlx5_uars_page *uar; 161 struct mlx5_vhca_cq cq; 162 struct mlx5_vhca_qp *host_qp; 163 struct mlx5_vhca_qp *fw_qp; 164 struct mlx5_nb nb; 165 int status; 166 }; 167 168 struct mlx5vf_pci_core_device { 169 struct vfio_pci_core_device core_device; 170 int vf_id; 171 u16 vhca_id; 172 u8 migrate_cap:1; 173 u8 deferred_reset:1; 174 u8 mdev_detach:1; 175 u8 log_active:1; 176 u8 chunk_mode:1; 177 struct completion tracker_comp; 178 /* protect migration state */ 179 struct mutex state_mutex; 180 enum vfio_device_mig_state mig_state; 181 /* protect the reset_done flow */ 182 spinlock_t reset_lock; 183 struct mlx5_vf_migration_file *resuming_migf; 184 struct mlx5_vf_migration_file *saving_migf; 185 struct mlx5_vhca_page_tracker tracker; 186 struct workqueue_struct *cb_wq; 187 struct notifier_block nb; 188 struct mlx5_core_dev *mdev; 189 }; 190 191 enum { 192 MLX5VF_QUERY_INC = (1UL << 0), 193 MLX5VF_QUERY_FINAL = (1UL << 1), 194 MLX5VF_QUERY_CLEANUP = (1UL << 2), 195 }; 196 197 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 198 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 199 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, 200 size_t *state_size, u64 *total_size, 201 u8 query_flags); 202 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, 203 const struct vfio_migration_ops *mig_ops, 204 const struct vfio_log_ops *log_ops); 205 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); 206 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); 207 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, 208 struct mlx5_vf_migration_file *migf, 209 struct mlx5_vhca_data_buffer *buf, bool inc, 210 bool track); 211 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, 212 struct mlx5_vf_migration_file *migf, 213 struct mlx5_vhca_data_buffer *buf); 214 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); 215 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); 216 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); 217 struct mlx5_vhca_data_buffer * 218 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, 219 enum dma_data_direction dma_dir); 220 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); 221 struct mlx5_vhca_data_buffer * 222 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, 223 enum dma_data_direction dma_dir); 224 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); 225 static inline struct page * 226 mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, 227 unsigned long offset) 228 { 229 int page_entry = offset / PAGE_SIZE; 230 231 if (page_entry >= buf->npages) 232 return NULL; 233 234 return buf->page_list[page_entry]; 235 } 236 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); 237 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev, 238 enum mlx5_vf_migf_state *last_save_state); 239 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); 240 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, 241 u8 chunk_num, size_t next_required_umem_size); 242 int mlx5vf_start_page_tracker(struct vfio_device *vdev, 243 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); 244 int mlx5vf_stop_page_tracker(struct vfio_device *vdev); 245 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, 246 unsigned long length, struct iova_bitmap *dirty); 247 #endif /* MLX5_VFIO_CMD_H */ 248