1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6 #ifndef MLX5_VFIO_CMD_H
7 #define MLX5_VFIO_CMD_H
9 #include <linux/kernel.h>
10 #include <linux/vfio_pci_core.h>
11 #include <linux/mlx5/driver.h>
12 #include <linux/mlx5/vport.h>
13 #include <linux/mlx5/cq.h>
14 #include <linux/mlx5/qp.h>
16 enum mlx5_vf_migf_state {
17 MLX5_MIGF_STATE_ERROR = 1,
18 MLX5_MIGF_STATE_PRE_COPY_ERROR,
19 MLX5_MIGF_STATE_PRE_COPY,
20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,
21 MLX5_MIGF_STATE_COMPLETE,
24 enum mlx5_vf_load_state {
25 MLX5_VF_LOAD_STATE_READ_HEADER,
26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,
27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA,
28 MLX5_VF_LOAD_STATE_PREP_IMAGE,
29 MLX5_VF_LOAD_STATE_READ_IMAGE,
30 MLX5_VF_LOAD_STATE_LOAD_IMAGE,
33 struct mlx5_vf_migration_tag_stop_copy_data {
34 __le64 stop_copy_size;
37 enum mlx5_vf_migf_header_flags {
38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,
39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,
42 enum mlx5_vf_migf_header_tag {
43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0,
44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,
47 struct mlx5_vf_migration_header {
49 /* For future use in case we may need to change the kernel protocol */
50 __le32 flags; /* Use mlx5_vf_migf_header_flags */
51 __le32 tag; /* Use mlx5_vf_migf_header_tag */
52 __u8 data[]; /* Its size is given in the record_size */
55 struct mlx5_vhca_data_buffer {
56 struct sg_append_table table;
61 enum dma_data_direction dma_dir;
63 u8 stop_copy_chunk_num;
64 struct list_head buf_elm;
65 struct mlx5_vf_migration_file *migf;
66 /* Optimize mlx5vf_get_migration_page() for sequential access */
67 struct scatterlist *last_offset_sg;
68 unsigned int sg_last_entry;
69 unsigned long last_offset;
72 struct mlx5vf_async_data {
73 struct mlx5_async_work cb_work;
74 struct work_struct work;
75 struct mlx5_vhca_data_buffer *buf;
76 struct mlx5_vhca_data_buffer *header_buf;
82 struct mlx5vf_save_work_data {
83 struct mlx5_vf_migration_file *migf;
84 size_t next_required_umem_size;
85 struct work_struct work;
89 #define MAX_NUM_CHUNKS 2
91 struct mlx5_vf_migration_file {
94 enum mlx5_vf_migf_state state;
96 enum mlx5_vf_load_state load_state;
101 u64 stop_copy_prep_size;
102 u64 pre_copy_initial_bytes;
103 size_t next_required_umem_size;
105 /* Upon chunk mode preserve another set of buffers for stop_copy phase */
106 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];
107 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];
108 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];
109 spinlock_t list_lock;
110 struct list_head buf_list;
111 struct list_head avail_list;
112 struct mlx5vf_pci_core_device *mvdev;
113 wait_queue_head_t poll_wait;
114 struct completion save_comp;
115 struct mlx5_async_ctx async_ctx;
116 struct mlx5vf_async_data async_data;
119 struct mlx5_vhca_cq_buf {
120 struct mlx5_frag_buf_ctrl fbc;
121 struct mlx5_frag_buf frag_buf;
126 struct mlx5_vhca_cq {
127 struct mlx5_vhca_cq_buf buf;
129 struct mlx5_core_cq mcq;
133 struct mlx5_vhca_recv_buf {
135 struct page **page_list;
136 dma_addr_t *dma_addrs;
141 struct mlx5_vhca_qp {
142 struct mlx5_frag_buf buf;
144 struct mlx5_vhca_recv_buf recv_buf;
145 u32 tracked_page_size;
151 unsigned int wqe_cnt;
153 struct mlx5_frag_buf_ctrl fbc;
157 struct mlx5_vhca_page_tracker {
162 struct mlx5_uars_page *uar;
163 struct mlx5_vhca_cq cq;
164 struct mlx5_vhca_qp *host_qp;
165 struct mlx5_vhca_qp *fw_qp;
170 struct mlx5vf_pci_core_device {
171 struct vfio_pci_core_device core_device;
179 struct completion tracker_comp;
180 /* protect migration state */
181 struct mutex state_mutex;
182 enum vfio_device_mig_state mig_state;
183 /* protect the reset_done flow */
184 spinlock_t reset_lock;
185 struct mlx5_vf_migration_file *resuming_migf;
186 struct mlx5_vf_migration_file *saving_migf;
187 struct mlx5_vhca_page_tracker tracker;
188 struct workqueue_struct *cb_wq;
189 struct notifier_block nb;
190 struct mlx5_core_dev *mdev;
194 MLX5VF_QUERY_INC = (1UL << 0),
195 MLX5VF_QUERY_FINAL = (1UL << 1),
196 MLX5VF_QUERY_CLEANUP = (1UL << 2),
199 int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
200 int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
201 int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
202 size_t *state_size, u64 *total_size,
204 void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
205 const struct vfio_migration_ops *mig_ops,
206 const struct vfio_log_ops *log_ops);
207 void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
208 void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
209 int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
210 struct mlx5_vf_migration_file *migf,
211 struct mlx5_vhca_data_buffer *buf, bool inc,
213 int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
214 struct mlx5_vf_migration_file *migf,
215 struct mlx5_vhca_data_buffer *buf);
216 int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);
217 void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);
218 void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);
219 struct mlx5_vhca_data_buffer *
220 mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf,
221 size_t length, enum dma_data_direction dma_dir);
222 void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);
223 struct mlx5_vhca_data_buffer *
224 mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf,
225 size_t length, enum dma_data_direction dma_dir);
226 void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);
227 struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
228 unsigned long offset);
229 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
230 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,
231 enum mlx5_vf_migf_state *last_save_state);
232 void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
233 void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
234 u8 chunk_num, size_t next_required_umem_size);
235 int mlx5vf_start_page_tracker(struct vfio_device *vdev,
236 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
237 int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
238 int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
239 unsigned long length, struct iova_bitmap *dirty);
240 #endif /* MLX5_VFIO_CMD_H */