1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4 */
5
6 #ifndef MLX5_VFIO_CMD_H
7 #define MLX5_VFIO_CMD_H
8
9 #include <linux/kernel.h>
10 #include <linux/vfio_pci_core.h>
11 #include <linux/mlx5/driver.h>
12 #include <linux/mlx5/vport.h>
13 #include <linux/mlx5/cq.h>
14 #include <linux/mlx5/qp.h>
15
16 enum mlx5_vf_migf_state {
17 MLX5_MIGF_STATE_ERROR = 1,
18 MLX5_MIGF_STATE_PRE_COPY_ERROR,
19 MLX5_MIGF_STATE_PRE_COPY,
20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,
21 MLX5_MIGF_STATE_COMPLETE,
22 };
23
24 enum mlx5_vf_load_state {
25 MLX5_VF_LOAD_STATE_READ_HEADER,
26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,
27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA,
28 MLX5_VF_LOAD_STATE_PREP_IMAGE,
29 MLX5_VF_LOAD_STATE_READ_IMAGE,
30 MLX5_VF_LOAD_STATE_LOAD_IMAGE,
31 };
32
33 struct mlx5_vf_migration_tag_stop_copy_data {
34 __le64 stop_copy_size;
35 };
36
37 enum mlx5_vf_migf_header_flags {
38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,
39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,
40 };
41
42 enum mlx5_vf_migf_header_tag {
43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0,
44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,
45 };
46
47 struct mlx5_vf_migration_header {
48 __le64 record_size;
49 /* For future use in case we may need to change the kernel protocol */
50 __le32 flags; /* Use mlx5_vf_migf_header_flags */
51 __le32 tag; /* Use mlx5_vf_migf_header_tag */
52 __u8 data[]; /* Its size is given in the record_size */
53 };
54
55 struct mlx5_vhca_data_buffer {
56 struct page **page_list;
57 struct dma_iova_state state;
58 loff_t start_pos;
59 u64 length;
60 u32 npages;
61 u32 mkey;
62 u32 *mkey_in;
63 enum dma_data_direction dma_dir;
64 u8 stop_copy_chunk_num;
65 struct list_head buf_elm;
66 struct mlx5_vf_migration_file *migf;
67 };
68
69 struct mlx5vf_async_data {
70 struct mlx5_async_work cb_work;
71 struct work_struct work;
72 struct mlx5_vhca_data_buffer *buf;
73 struct mlx5_vhca_data_buffer *header_buf;
74 int status;
75 u8 stop_copy_chunk:1;
76 void *out;
77 };
78
79 struct mlx5vf_save_work_data {
80 struct mlx5_vf_migration_file *migf;
81 size_t next_required_umem_size;
82 struct work_struct work;
83 u8 chunk_num;
84 };
85
86 #define MAX_NUM_CHUNKS 2
87
88 struct mlx5_vf_migration_file {
89 struct file *filp;
90 struct mutex lock;
91 enum mlx5_vf_migf_state state;
92
93 enum mlx5_vf_load_state load_state;
94 u32 pdn;
95 loff_t max_pos;
96 u64 record_size;
97 u32 record_tag;
98 u64 stop_copy_prep_size;
99 u64 pre_copy_initial_bytes;
100 size_t next_required_umem_size;
101 u8 num_ready_chunks;
102 /* Upon chunk mode preserve another set of buffers for stop_copy phase */
103 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];
104 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];
105 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];
106 spinlock_t list_lock;
107 struct list_head buf_list;
108 struct list_head avail_list;
109 struct mlx5vf_pci_core_device *mvdev;
110 wait_queue_head_t poll_wait;
111 struct completion save_comp;
112 struct mlx5_async_ctx async_ctx;
113 struct mlx5vf_async_data async_data;
114 };
115
116 struct mlx5_vhca_cq_buf {
117 struct mlx5_frag_buf_ctrl fbc;
118 struct mlx5_frag_buf frag_buf;
119 int cqe_size;
120 int nent;
121 };
122
123 struct mlx5_vhca_cq {
124 struct mlx5_vhca_cq_buf buf;
125 struct mlx5_db db;
126 struct mlx5_core_cq mcq;
127 size_t ncqe;
128 };
129
130 struct mlx5_vhca_recv_buf {
131 u32 npages;
132 struct page **page_list;
133 struct dma_iova_state state;
134 u32 next_rq_offset;
135 u32 *mkey_in;
136 u32 mkey;
137 };
138
139 struct mlx5_vhca_qp {
140 struct mlx5_frag_buf buf;
141 struct mlx5_db db;
142 struct mlx5_vhca_recv_buf recv_buf;
143 u32 tracked_page_size;
144 u32 max_msg_size;
145 u32 qpn;
146 struct {
147 unsigned int pc;
148 unsigned int cc;
149 unsigned int wqe_cnt;
150 __be32 *db;
151 struct mlx5_frag_buf_ctrl fbc;
152 } rq;
153 };
154
155 struct mlx5_vhca_page_tracker {
156 u32 id;
157 u32 pdn;
158 u8 is_err:1;
159 u8 object_changed:1;
160 struct mlx5_uars_page *uar;
161 struct mlx5_vhca_cq cq;
162 struct mlx5_vhca_qp *host_qp;
163 struct mlx5_vhca_qp *fw_qp;
164 struct mlx5_nb nb;
165 int status;
166 };
167
168 struct mlx5vf_pci_core_device {
169 struct vfio_pci_core_device core_device;
170 int vf_id;
171 u16 vhca_id;
172 u8 migrate_cap:1;
173 u8 deferred_reset:1;
174 u8 mdev_detach:1;
175 u8 log_active:1;
176 u8 chunk_mode:1;
177 struct completion tracker_comp;
178 /* protect migration state */
179 struct mutex state_mutex;
180 enum vfio_device_mig_state mig_state;
181 /* protect the reset_done flow */
182 spinlock_t reset_lock;
183 struct mlx5_vf_migration_file *resuming_migf;
184 struct mlx5_vf_migration_file *saving_migf;
185 struct mlx5_vhca_page_tracker tracker;
186 struct workqueue_struct *cb_wq;
187 struct notifier_block nb;
188 struct mlx5_core_dev *mdev;
189 };
190
191 enum {
192 MLX5VF_QUERY_INC = (1UL << 0),
193 MLX5VF_QUERY_FINAL = (1UL << 1),
194 MLX5VF_QUERY_CLEANUP = (1UL << 2),
195 };
196
197 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
198 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
199 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
200 size_t *state_size, u64 *total_size,
201 u8 query_flags);
202 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
203 const struct vfio_migration_ops *mig_ops,
204 const struct vfio_log_ops *log_ops);
205 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
206 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
207 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
208 struct mlx5_vf_migration_file *migf,
209 struct mlx5_vhca_data_buffer *buf, bool inc,
210 bool track);
211 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
212 struct mlx5_vf_migration_file *migf,
213 struct mlx5_vhca_data_buffer *buf);
214 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);
215 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);
216 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);
217 struct mlx5_vhca_data_buffer *
218 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
219 enum dma_data_direction dma_dir);
220 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);
221 struct mlx5_vhca_data_buffer *
222 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
223 enum dma_data_direction dma_dir);
224 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);
225 static inline struct page *
mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer * buf,unsigned long offset)226 mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
227 unsigned long offset)
228 {
229 int page_entry = offset / PAGE_SIZE;
230
231 if (page_entry >= buf->npages)
232 return NULL;
233
234 return buf->page_list[page_entry];
235 }
236 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
237 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,
238 enum mlx5_vf_migf_state *last_save_state);
239 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
240 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
241 u8 chunk_num, size_t next_required_umem_size);
242 int mlx5vf_start_page_tracker(struct vfio_device *vdev,
243 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
244 int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
245 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
246 unsigned long length, struct iova_bitmap *dirty);
247 #endif /* MLX5_VFIO_CMD_H */
248