1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 4 */ 5 6 #ifndef MLX5_VFIO_CMD_H 7 #define MLX5_VFIO_CMD_H 8 9 #include <linux/kernel.h> 10 #include <linux/vfio_pci_core.h> 11 #include <linux/mlx5/driver.h> 12 #include <linux/mlx5/vport.h> 13 #include <linux/mlx5/cq.h> 14 #include <linux/mlx5/qp.h> 15 16 enum mlx5_vf_migf_state { 17 MLX5_MIGF_STATE_ERROR = 1, 18 MLX5_MIGF_STATE_PRE_COPY_ERROR, 19 MLX5_MIGF_STATE_PRE_COPY, 20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, 21 MLX5_MIGF_STATE_COMPLETE, 22 }; 23 24 enum mlx5_vf_load_state { 25 MLX5_VF_LOAD_STATE_READ_HEADER, 26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA, 27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA, 28 MLX5_VF_LOAD_STATE_PREP_IMAGE, 29 MLX5_VF_LOAD_STATE_READ_IMAGE, 30 MLX5_VF_LOAD_STATE_LOAD_IMAGE, 31 }; 32 33 struct mlx5_vf_migration_tag_stop_copy_data { 34 __le64 stop_copy_size; 35 }; 36 37 enum mlx5_vf_migf_header_flags { 38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, 39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0, 40 }; 41 42 enum mlx5_vf_migf_header_tag { 43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0, 44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0, 45 }; 46 47 struct mlx5_vf_migration_header { 48 __le64 record_size; 49 /* For future use in case we may need to change the kernel protocol */ 50 __le32 flags; /* Use mlx5_vf_migf_header_flags */ 51 __le32 tag; /* Use mlx5_vf_migf_header_tag */ 52 __u8 data[]; /* Its size is given in the record_size */ 53 }; 54 55 struct mlx5_vhca_data_buffer { 56 struct sg_append_table table; 57 loff_t start_pos; 58 u64 length; 59 u64 allocated_length; 60 u32 mkey; 61 enum dma_data_direction dma_dir; 62 u8 dmaed:1; 63 u8 stop_copy_chunk_num; 64 struct list_head buf_elm; 65 struct mlx5_vf_migration_file *migf; 66 /* Optimize mlx5vf_get_migration_page() for sequential access */ 67 struct scatterlist *last_offset_sg; 68 unsigned int sg_last_entry; 69 unsigned long last_offset; 70 }; 71 72 struct mlx5vf_async_data { 73 struct mlx5_async_work cb_work; 74 struct work_struct work; 75 struct mlx5_vhca_data_buffer *buf; 76 struct mlx5_vhca_data_buffer *header_buf; 77 int status; 78 u8 stop_copy_chunk:1; 79 void *out; 80 }; 81 82 struct mlx5vf_save_work_data { 83 struct mlx5_vf_migration_file *migf; 84 size_t next_required_umem_size; 85 struct work_struct work; 86 u8 chunk_num; 87 }; 88 89 #define MAX_NUM_CHUNKS 2 90 91 struct mlx5_vf_migration_file { 92 struct file *filp; 93 struct mutex lock; 94 enum mlx5_vf_migf_state state; 95 96 enum mlx5_vf_load_state load_state; 97 u32 pdn; 98 loff_t max_pos; 99 u64 record_size; 100 u32 record_tag; 101 u64 stop_copy_prep_size; 102 u64 pre_copy_initial_bytes; 103 size_t next_required_umem_size; 104 u8 num_ready_chunks; 105 /* Upon chunk mode preserve another set of buffers for stop_copy phase */ 106 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; 107 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS]; 108 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; 109 spinlock_t list_lock; 110 struct list_head buf_list; 111 struct list_head avail_list; 112 struct mlx5vf_pci_core_device *mvdev; 113 wait_queue_head_t poll_wait; 114 struct completion save_comp; 115 struct mlx5_async_ctx async_ctx; 116 struct mlx5vf_async_data async_data; 117 }; 118 119 struct mlx5_vhca_cq_buf { 120 struct mlx5_frag_buf_ctrl fbc; 121 struct mlx5_frag_buf frag_buf; 122 int cqe_size; 123 int nent; 124 }; 125 126 struct mlx5_vhca_cq { 127 struct mlx5_vhca_cq_buf buf; 128 struct mlx5_db db; 129 struct mlx5_core_cq mcq; 130 size_t ncqe; 131 }; 132 133 struct mlx5_vhca_recv_buf { 134 u32 npages; 135 struct page **page_list; 136 dma_addr_t *dma_addrs; 137 u32 next_rq_offset; 138 u32 mkey; 139 }; 140 141 struct mlx5_vhca_qp { 142 struct mlx5_frag_buf buf; 143 struct mlx5_db db; 144 struct mlx5_vhca_recv_buf recv_buf; 145 u32 tracked_page_size; 146 u32 max_msg_size; 147 u32 qpn; 148 struct { 149 unsigned int pc; 150 unsigned int cc; 151 unsigned int wqe_cnt; 152 __be32 *db; 153 struct mlx5_frag_buf_ctrl fbc; 154 } rq; 155 }; 156 157 struct mlx5_vhca_page_tracker { 158 u32 id; 159 u32 pdn; 160 u8 is_err:1; 161 u8 object_changed:1; 162 struct mlx5_uars_page *uar; 163 struct mlx5_vhca_cq cq; 164 struct mlx5_vhca_qp *host_qp; 165 struct mlx5_vhca_qp *fw_qp; 166 struct mlx5_nb nb; 167 int status; 168 }; 169 170 struct mlx5vf_pci_core_device { 171 struct vfio_pci_core_device core_device; 172 int vf_id; 173 u16 vhca_id; 174 u8 migrate_cap:1; 175 u8 deferred_reset:1; 176 u8 mdev_detach:1; 177 u8 log_active:1; 178 u8 chunk_mode:1; 179 struct completion tracker_comp; 180 /* protect migration state */ 181 struct mutex state_mutex; 182 enum vfio_device_mig_state mig_state; 183 /* protect the reset_done flow */ 184 spinlock_t reset_lock; 185 struct mlx5_vf_migration_file *resuming_migf; 186 struct mlx5_vf_migration_file *saving_migf; 187 struct mlx5_vhca_page_tracker tracker; 188 struct workqueue_struct *cb_wq; 189 struct notifier_block nb; 190 struct mlx5_core_dev *mdev; 191 }; 192 193 enum { 194 MLX5VF_QUERY_INC = (1UL << 0), 195 MLX5VF_QUERY_FINAL = (1UL << 1), 196 MLX5VF_QUERY_CLEANUP = (1UL << 2), 197 }; 198 199 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 200 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); 201 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, 202 size_t *state_size, u64 *total_size, 203 u8 query_flags); 204 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, 205 const struct vfio_migration_ops *mig_ops, 206 const struct vfio_log_ops *log_ops); 207 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); 208 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); 209 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, 210 struct mlx5_vf_migration_file *migf, 211 struct mlx5_vhca_data_buffer *buf, bool inc, 212 bool track); 213 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, 214 struct mlx5_vf_migration_file *migf, 215 struct mlx5_vhca_data_buffer *buf); 216 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); 217 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); 218 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); 219 struct mlx5_vhca_data_buffer * 220 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, 221 size_t length, enum dma_data_direction dma_dir); 222 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); 223 struct mlx5_vhca_data_buffer * 224 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, 225 size_t length, enum dma_data_direction dma_dir); 226 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); 227 struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, 228 unsigned long offset); 229 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); 230 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev, 231 enum mlx5_vf_migf_state *last_save_state); 232 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); 233 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, 234 u8 chunk_num, size_t next_required_umem_size); 235 int mlx5vf_start_page_tracker(struct vfio_device *vdev, 236 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); 237 int mlx5vf_stop_page_tracker(struct vfio_device *vdev); 238 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, 239 unsigned long length, struct iova_bitmap *dirty); 240 #endif /* MLX5_VFIO_CMD_H */ 241