1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies */
3
4 #include <linux/pci.h>
5 #include "mlx5_core.h"
6 #include "pci_vsc.h"
7
8 #define MLX5_EXTRACT_C(source, offset, size) \
9 ((((u32)(source)) >> (offset)) & MLX5_ONES32(size))
10 #define MLX5_EXTRACT(src, start, len) \
11 (((len) == 32) ? (src) : MLX5_EXTRACT_C(src, start, len))
12 #define MLX5_ONES32(size) \
13 ((size) ? (0xffffffff >> (32 - (size))) : 0)
14 #define MLX5_MASK32(offset, size) \
15 (MLX5_ONES32(size) << (offset))
16 #define MLX5_MERGE_C(rsrc1, rsrc2, start, len) \
17 ((((rsrc2) << (start)) & (MLX5_MASK32((start), (len)))) | \
18 ((rsrc1) & (~MLX5_MASK32((start), (len)))))
19 #define MLX5_MERGE(rsrc1, rsrc2, start, len) \
20 (((len) == 32) ? (rsrc2) : MLX5_MERGE_C(rsrc1, rsrc2, start, len))
21 #define vsc_read(dev, offset, val) \
22 pci_read_config_dword((dev)->pdev, (dev)->vsc_addr + (offset), (val))
23 #define vsc_write(dev, offset, val) \
24 pci_write_config_dword((dev)->pdev, (dev)->vsc_addr + (offset), (val))
25 #define VSC_MAX_RETRIES 2048
26
27 /* Reading VSC registers can take relatively long time.
28 * Yield the cpu every 128 registers read.
29 */
30 #define VSC_GW_READ_BLOCK_COUNT 128
31
32 enum {
33 VSC_CTRL_OFFSET = 0x4,
34 VSC_COUNTER_OFFSET = 0x8,
35 VSC_SEMAPHORE_OFFSET = 0xc,
36 VSC_ADDR_OFFSET = 0x10,
37 VSC_DATA_OFFSET = 0x14,
38
39 VSC_FLAG_BIT_OFFS = 31,
40 VSC_FLAG_BIT_LEN = 1,
41
42 VSC_SYND_BIT_OFFS = 30,
43 VSC_SYND_BIT_LEN = 1,
44
45 VSC_ADDR_BIT_OFFS = 0,
46 VSC_ADDR_BIT_LEN = 30,
47
48 VSC_SPACE_BIT_OFFS = 0,
49 VSC_SPACE_BIT_LEN = 16,
50
51 VSC_SIZE_VLD_BIT_OFFS = 28,
52 VSC_SIZE_VLD_BIT_LEN = 1,
53
54 VSC_STATUS_BIT_OFFS = 29,
55 VSC_STATUS_BIT_LEN = 3,
56 };
57
mlx5_pci_vsc_init(struct mlx5_core_dev * dev)58 void mlx5_pci_vsc_init(struct mlx5_core_dev *dev)
59 {
60 if (!mlx5_core_is_pf(dev))
61 return;
62
63 dev->vsc_addr = pci_find_capability(dev->pdev,
64 PCI_CAP_ID_VNDR);
65 if (!dev->vsc_addr)
66 mlx5_core_warn(dev, "Failed to get valid vendor specific ID\n");
67 }
68
mlx5_vsc_gw_lock(struct mlx5_core_dev * dev)69 int mlx5_vsc_gw_lock(struct mlx5_core_dev *dev)
70 {
71 u32 counter = 0;
72 int retries = 0;
73 u32 lock_val;
74 int ret;
75
76 pci_cfg_access_lock(dev->pdev);
77 do {
78 if (retries > VSC_MAX_RETRIES) {
79 ret = -EBUSY;
80 goto pci_unlock;
81 }
82 if (pci_channel_offline(dev->pdev)) {
83 ret = -EACCES;
84 goto pci_unlock;
85 }
86
87 /* Check if semaphore is already locked */
88 ret = vsc_read(dev, VSC_SEMAPHORE_OFFSET, &lock_val);
89 if (ret)
90 goto pci_unlock;
91
92 if (lock_val) {
93 retries++;
94 usleep_range(1000, 2000);
95 continue;
96 }
97
98 /* Read and write counter value, if written value is
99 * the same, semaphore was acquired successfully.
100 */
101 ret = vsc_read(dev, VSC_COUNTER_OFFSET, &counter);
102 if (ret)
103 goto pci_unlock;
104
105 ret = vsc_write(dev, VSC_SEMAPHORE_OFFSET, counter);
106 if (ret)
107 goto pci_unlock;
108
109 ret = vsc_read(dev, VSC_SEMAPHORE_OFFSET, &lock_val);
110 if (ret)
111 goto pci_unlock;
112
113 retries++;
114 } while (counter != lock_val);
115
116 return 0;
117
118 pci_unlock:
119 pci_cfg_access_unlock(dev->pdev);
120 return ret;
121 }
122
mlx5_vsc_gw_unlock(struct mlx5_core_dev * dev)123 int mlx5_vsc_gw_unlock(struct mlx5_core_dev *dev)
124 {
125 int ret;
126
127 ret = vsc_write(dev, VSC_SEMAPHORE_OFFSET, MLX5_VSC_UNLOCK);
128 pci_cfg_access_unlock(dev->pdev);
129 return ret;
130 }
131
mlx5_vsc_gw_set_space(struct mlx5_core_dev * dev,u16 space,u32 * ret_space_size)132 int mlx5_vsc_gw_set_space(struct mlx5_core_dev *dev, u16 space,
133 u32 *ret_space_size)
134 {
135 int ret;
136 u32 val = 0;
137
138 if (!mlx5_vsc_accessible(dev))
139 return -EINVAL;
140
141 if (ret_space_size)
142 *ret_space_size = 0;
143
144 /* Get a unique val */
145 ret = vsc_read(dev, VSC_CTRL_OFFSET, &val);
146 if (ret)
147 goto out;
148
149 /* Try to modify the lock */
150 val = MLX5_MERGE(val, space, VSC_SPACE_BIT_OFFS, VSC_SPACE_BIT_LEN);
151 ret = vsc_write(dev, VSC_CTRL_OFFSET, val);
152 if (ret)
153 goto out;
154
155 /* Verify lock was modified */
156 ret = vsc_read(dev, VSC_CTRL_OFFSET, &val);
157 if (ret)
158 goto out;
159
160 if (MLX5_EXTRACT(val, VSC_STATUS_BIT_OFFS, VSC_STATUS_BIT_LEN) == 0)
161 return -EINVAL;
162
163 /* Get space max address if indicated by size valid bit */
164 if (ret_space_size &&
165 MLX5_EXTRACT(val, VSC_SIZE_VLD_BIT_OFFS, VSC_SIZE_VLD_BIT_LEN)) {
166 ret = vsc_read(dev, VSC_ADDR_OFFSET, &val);
167 if (ret) {
168 mlx5_core_warn(dev, "Failed to get max space size\n");
169 goto out;
170 }
171 *ret_space_size = MLX5_EXTRACT(val, VSC_ADDR_BIT_OFFS,
172 VSC_ADDR_BIT_LEN);
173 }
174 return 0;
175
176 out:
177 return ret;
178 }
179
mlx5_vsc_wait_on_flag(struct mlx5_core_dev * dev,u8 expected_val)180 static int mlx5_vsc_wait_on_flag(struct mlx5_core_dev *dev, u8 expected_val)
181 {
182 int retries = 0;
183 u32 flag;
184 int ret;
185
186 do {
187 if (retries > VSC_MAX_RETRIES)
188 return -EBUSY;
189
190 ret = vsc_read(dev, VSC_ADDR_OFFSET, &flag);
191 if (ret)
192 return ret;
193 flag = MLX5_EXTRACT(flag, VSC_FLAG_BIT_OFFS, VSC_FLAG_BIT_LEN);
194 retries++;
195
196 if ((retries & 0xf) == 0)
197 usleep_range(1000, 2000);
198
199 } while (flag != expected_val);
200
201 return 0;
202 }
203
mlx5_vsc_gw_write(struct mlx5_core_dev * dev,unsigned int address,u32 data)204 static int mlx5_vsc_gw_write(struct mlx5_core_dev *dev, unsigned int address,
205 u32 data)
206 {
207 int ret;
208
209 if (MLX5_EXTRACT(address, VSC_SYND_BIT_OFFS,
210 VSC_FLAG_BIT_LEN + VSC_SYND_BIT_LEN))
211 return -EINVAL;
212
213 /* Set flag to 0x1 */
214 address = MLX5_MERGE(address, 1, VSC_FLAG_BIT_OFFS, 1);
215 ret = vsc_write(dev, VSC_DATA_OFFSET, data);
216 if (ret)
217 goto out;
218
219 ret = vsc_write(dev, VSC_ADDR_OFFSET, address);
220 if (ret)
221 goto out;
222
223 /* Wait for the flag to be cleared */
224 ret = mlx5_vsc_wait_on_flag(dev, 0);
225
226 out:
227 return ret;
228 }
229
mlx5_vsc_gw_read(struct mlx5_core_dev * dev,unsigned int address,u32 * data)230 static int mlx5_vsc_gw_read(struct mlx5_core_dev *dev, unsigned int address,
231 u32 *data)
232 {
233 int ret;
234
235 if (MLX5_EXTRACT(address, VSC_SYND_BIT_OFFS,
236 VSC_FLAG_BIT_LEN + VSC_SYND_BIT_LEN))
237 return -EINVAL;
238
239 ret = vsc_write(dev, VSC_ADDR_OFFSET, address);
240 if (ret)
241 goto out;
242
243 ret = mlx5_vsc_wait_on_flag(dev, 1);
244 if (ret)
245 goto out;
246
247 ret = vsc_read(dev, VSC_DATA_OFFSET, data);
248 out:
249 return ret;
250 }
251
mlx5_vsc_gw_read_fast(struct mlx5_core_dev * dev,unsigned int read_addr,unsigned int * next_read_addr,u32 * data)252 static int mlx5_vsc_gw_read_fast(struct mlx5_core_dev *dev,
253 unsigned int read_addr,
254 unsigned int *next_read_addr,
255 u32 *data)
256 {
257 int ret;
258
259 ret = mlx5_vsc_gw_read(dev, read_addr, data);
260 if (ret)
261 goto out;
262
263 ret = vsc_read(dev, VSC_ADDR_OFFSET, next_read_addr);
264 if (ret)
265 goto out;
266
267 *next_read_addr = MLX5_EXTRACT(*next_read_addr, VSC_ADDR_BIT_OFFS,
268 VSC_ADDR_BIT_LEN);
269
270 if (*next_read_addr <= read_addr)
271 ret = -EINVAL;
272 out:
273 return ret;
274 }
275
mlx5_vsc_gw_read_block_fast(struct mlx5_core_dev * dev,u32 * data,int length)276 int mlx5_vsc_gw_read_block_fast(struct mlx5_core_dev *dev, u32 *data,
277 int length)
278 {
279 unsigned int next_read_addr = 0;
280 unsigned int read_addr = 0;
281 unsigned int count = 0;
282
283 while (read_addr < length) {
284 if (mlx5_vsc_gw_read_fast(dev, read_addr, &next_read_addr,
285 &data[(read_addr >> 2)]))
286 return read_addr;
287
288 read_addr = next_read_addr;
289 if (++count == VSC_GW_READ_BLOCK_COUNT) {
290 cond_resched();
291 count = 0;
292 }
293 }
294 return length;
295 }
296
mlx5_vsc_sem_set_space(struct mlx5_core_dev * dev,u16 space,enum mlx5_vsc_state state)297 int mlx5_vsc_sem_set_space(struct mlx5_core_dev *dev, u16 space,
298 enum mlx5_vsc_state state)
299 {
300 u32 data, id = 0;
301 int ret;
302
303 ret = mlx5_vsc_gw_set_space(dev, MLX5_SEMAPHORE_SPACE_DOMAIN, NULL);
304 if (ret) {
305 mlx5_core_warn(dev, "Failed to set gw space %d\n", ret);
306 return ret;
307 }
308
309 if (state == MLX5_VSC_LOCK) {
310 /* Get a unique ID based on the counter */
311 ret = vsc_read(dev, VSC_COUNTER_OFFSET, &id);
312 if (ret)
313 return ret;
314 }
315
316 /* Try to modify lock */
317 ret = mlx5_vsc_gw_write(dev, space, id);
318 if (ret)
319 return ret;
320
321 /* Verify lock was modified */
322 ret = mlx5_vsc_gw_read(dev, space, &data);
323 if (ret)
324 return -EINVAL;
325
326 if (data != id)
327 return -EBUSY;
328
329 return 0;
330 }
331