xref: /linux/drivers/pci/ats.c (revision 870b7fdc660b38c4e1bd8bf48e62aa352ddf8f42)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Express I/O Virtualization (IOV) support
4  *   Address Translation Service 1.0
5  *   Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
6  *   PASID support added by Joerg Roedel <joerg.roedel@amd.com>
7  *
8  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
9  * Copyright (C) 2011 Advanced Micro Devices,
10  */
11 
12 #include <linux/bitfield.h>
13 #include <linux/export.h>
14 #include <linux/pci-ats.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 
18 #include "pci.h"
19 
20 void pci_ats_init(struct pci_dev *dev)
21 {
22 	int pos;
23 
24 	if (pci_ats_disabled())
25 		return;
26 
27 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
28 	if (!pos)
29 		return;
30 
31 	dev->ats_cap = pos;
32 }
33 
34 /**
35  * pci_ats_supported - check if the device can use ATS
36  * @dev: the PCI device
37  *
38  * Returns true if the device supports ATS and is allowed to use it, false
39  * otherwise.
40  */
41 bool pci_ats_supported(struct pci_dev *dev)
42 {
43 	if (!dev->ats_cap)
44 		return false;
45 
46 	return (dev->untrusted == 0);
47 }
48 EXPORT_SYMBOL_GPL(pci_ats_supported);
49 
50 /**
51  * pci_prepare_ats - Setup the PS for ATS
52  * @dev: the PCI device
53  * @ps: the IOMMU page shift
54  *
55  * This must be done by the IOMMU driver on the PF before any VFs are created to
56  * ensure that the VF can have ATS enabled.
57  *
58  * Returns 0 on success, or negative on failure.
59  */
60 int pci_prepare_ats(struct pci_dev *dev, int ps)
61 {
62 	u16 ctrl;
63 
64 	if (!pci_ats_supported(dev))
65 		return -EINVAL;
66 
67 	if (WARN_ON(dev->ats_enabled))
68 		return -EBUSY;
69 
70 	if (ps < PCI_ATS_MIN_STU)
71 		return -EINVAL;
72 
73 	if (dev->is_virtfn)
74 		return 0;
75 
76 	dev->ats_stu = ps;
77 	ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
78 	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
79 	return 0;
80 }
81 EXPORT_SYMBOL_GPL(pci_prepare_ats);
82 
83 /**
84  * pci_enable_ats - enable the ATS capability
85  * @dev: the PCI device
86  * @ps: the IOMMU page shift
87  *
88  * Returns 0 on success, or negative on failure.
89  */
90 int pci_enable_ats(struct pci_dev *dev, int ps)
91 {
92 	u16 ctrl;
93 	struct pci_dev *pdev;
94 
95 	if (!pci_ats_supported(dev))
96 		return -EINVAL;
97 
98 	if (WARN_ON(dev->ats_enabled))
99 		return -EBUSY;
100 
101 	if (ps < PCI_ATS_MIN_STU)
102 		return -EINVAL;
103 
104 	/*
105 	 * Note that enabling ATS on a VF fails unless it's already enabled
106 	 * with the same STU on the PF.
107 	 */
108 	ctrl = PCI_ATS_CTRL_ENABLE;
109 	if (dev->is_virtfn) {
110 		pdev = pci_physfn(dev);
111 		if (pdev->ats_stu != ps)
112 			return -EINVAL;
113 	} else {
114 		dev->ats_stu = ps;
115 		ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
116 	}
117 	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
118 
119 	dev->ats_enabled = 1;
120 	return 0;
121 }
122 EXPORT_SYMBOL_GPL(pci_enable_ats);
123 
124 /**
125  * pci_disable_ats - disable the ATS capability
126  * @dev: the PCI device
127  */
128 void pci_disable_ats(struct pci_dev *dev)
129 {
130 	u16 ctrl;
131 
132 	if (WARN_ON(!dev->ats_enabled))
133 		return;
134 
135 	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
136 	ctrl &= ~PCI_ATS_CTRL_ENABLE;
137 	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
138 
139 	dev->ats_enabled = 0;
140 }
141 EXPORT_SYMBOL_GPL(pci_disable_ats);
142 
143 void pci_restore_ats_state(struct pci_dev *dev)
144 {
145 	u16 ctrl;
146 
147 	if (!dev->ats_enabled)
148 		return;
149 
150 	ctrl = PCI_ATS_CTRL_ENABLE;
151 	if (!dev->is_virtfn)
152 		ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
153 	pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
154 }
155 
156 /**
157  * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
158  * @dev: the PCI device
159  *
160  * Returns the queue depth on success, or negative on failure.
161  *
162  * The ATS spec uses 0 in the Invalidate Queue Depth field to
163  * indicate that the function can accept 32 Invalidate Request.
164  * But here we use the `real' values (i.e. 1~32) for the Queue
165  * Depth; and 0 indicates the function shares the Queue with
166  * other functions (doesn't exclusively own a Queue).
167  */
168 int pci_ats_queue_depth(struct pci_dev *dev)
169 {
170 	u16 cap;
171 
172 	if (!dev->ats_cap)
173 		return -EINVAL;
174 
175 	if (dev->is_virtfn)
176 		return 0;
177 
178 	pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
179 	return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
180 }
181 
182 /**
183  * pci_ats_page_aligned - Return Page Aligned Request bit status.
184  * @pdev: the PCI device
185  *
186  * Returns 1, if the Untranslated Addresses generated by the device
187  * are always aligned or 0 otherwise.
188  *
189  * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
190  * is set, it indicates the Untranslated Addresses generated by the
191  * device are always aligned to a 4096 byte boundary.
192  */
193 int pci_ats_page_aligned(struct pci_dev *pdev)
194 {
195 	u16 cap;
196 
197 	if (!pdev->ats_cap)
198 		return 0;
199 
200 	pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
201 
202 	if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
203 		return 1;
204 
205 	return 0;
206 }
207 
208 #ifdef CONFIG_PCI_PRI
209 void pci_pri_init(struct pci_dev *pdev)
210 {
211 	u16 status;
212 
213 	pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
214 
215 	if (!pdev->pri_cap)
216 		return;
217 
218 	pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
219 	if (status & PCI_PRI_STATUS_PASID)
220 		pdev->pasid_required = 1;
221 }
222 
223 /**
224  * pci_enable_pri - Enable PRI capability
225  * @pdev: PCI device structure
226  * @reqs: outstanding requests
227  *
228  * Returns 0 on success, negative value on error
229  */
230 int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
231 {
232 	u16 control, status;
233 	u32 max_requests;
234 	int pri = pdev->pri_cap;
235 
236 	/*
237 	 * VFs must not implement the PRI Capability.  If their PF
238 	 * implements PRI, it is shared by the VFs, so if the PF PRI is
239 	 * enabled, it is also enabled for the VF.
240 	 */
241 	if (pdev->is_virtfn) {
242 		if (pci_physfn(pdev)->pri_enabled)
243 			return 0;
244 		return -EINVAL;
245 	}
246 
247 	if (WARN_ON(pdev->pri_enabled))
248 		return -EBUSY;
249 
250 	if (!pri)
251 		return -EINVAL;
252 
253 	pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
254 	if (!(status & PCI_PRI_STATUS_STOPPED))
255 		return -EBUSY;
256 
257 	pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
258 	reqs = min(max_requests, reqs);
259 	pdev->pri_reqs_alloc = reqs;
260 	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
261 
262 	control = PCI_PRI_CTRL_ENABLE;
263 	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
264 
265 	pdev->pri_enabled = 1;
266 
267 	return 0;
268 }
269 
270 /**
271  * pci_disable_pri - Disable PRI capability
272  * @pdev: PCI device structure
273  *
274  * Only clears the enabled-bit, regardless of its former value
275  */
276 void pci_disable_pri(struct pci_dev *pdev)
277 {
278 	u16 control;
279 	int pri = pdev->pri_cap;
280 
281 	/* VFs share the PF PRI */
282 	if (pdev->is_virtfn)
283 		return;
284 
285 	if (WARN_ON(!pdev->pri_enabled))
286 		return;
287 
288 	if (!pri)
289 		return;
290 
291 	pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
292 	control &= ~PCI_PRI_CTRL_ENABLE;
293 	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
294 
295 	pdev->pri_enabled = 0;
296 }
297 EXPORT_SYMBOL_GPL(pci_disable_pri);
298 
299 /**
300  * pci_restore_pri_state - Restore PRI
301  * @pdev: PCI device structure
302  */
303 void pci_restore_pri_state(struct pci_dev *pdev)
304 {
305 	u16 control = PCI_PRI_CTRL_ENABLE;
306 	u32 reqs = pdev->pri_reqs_alloc;
307 	int pri = pdev->pri_cap;
308 
309 	if (pdev->is_virtfn)
310 		return;
311 
312 	if (!pdev->pri_enabled)
313 		return;
314 
315 	if (!pri)
316 		return;
317 
318 	pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
319 	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
320 }
321 
322 /**
323  * pci_reset_pri - Resets device's PRI state
324  * @pdev: PCI device structure
325  *
326  * The PRI capability must be disabled before this function is called.
327  * Returns 0 on success, negative value on error.
328  */
329 int pci_reset_pri(struct pci_dev *pdev)
330 {
331 	u16 control;
332 	int pri = pdev->pri_cap;
333 
334 	if (pdev->is_virtfn)
335 		return 0;
336 
337 	if (WARN_ON(pdev->pri_enabled))
338 		return -EBUSY;
339 
340 	if (!pri)
341 		return -EINVAL;
342 
343 	control = PCI_PRI_CTRL_RESET;
344 	pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
345 
346 	return 0;
347 }
348 
349 /**
350  * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
351  *				 status.
352  * @pdev: PCI device structure
353  *
354  * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
355  */
356 int pci_prg_resp_pasid_required(struct pci_dev *pdev)
357 {
358 	if (pdev->is_virtfn)
359 		pdev = pci_physfn(pdev);
360 
361 	return pdev->pasid_required;
362 }
363 
364 /**
365  * pci_pri_supported - Check if PRI is supported.
366  * @pdev: PCI device structure
367  *
368  * Returns true if PRI capability is present, false otherwise.
369  */
370 bool pci_pri_supported(struct pci_dev *pdev)
371 {
372 	/* VFs share the PF PRI */
373 	if (pci_physfn(pdev)->pri_cap)
374 		return true;
375 	return false;
376 }
377 EXPORT_SYMBOL_GPL(pci_pri_supported);
378 #endif /* CONFIG_PCI_PRI */
379 
380 #ifdef CONFIG_PCI_PASID
381 void pci_pasid_init(struct pci_dev *pdev)
382 {
383 	pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
384 }
385 
386 /**
387  * pci_enable_pasid - Enable the PASID capability
388  * @pdev: PCI device structure
389  * @features: Features to enable
390  *
391  * Returns 0 on success, negative value on error. This function checks
392  * whether the features are actually supported by the device and returns
393  * an error if not.
394  */
395 int pci_enable_pasid(struct pci_dev *pdev, int features)
396 {
397 	u16 control, supported;
398 	int pasid = pdev->pasid_cap;
399 
400 	/*
401 	 * VFs must not implement the PASID Capability, but if a PF
402 	 * supports PASID, its VFs share the PF PASID configuration.
403 	 */
404 	if (pdev->is_virtfn) {
405 		if (pci_physfn(pdev)->pasid_enabled)
406 			return 0;
407 		return -EINVAL;
408 	}
409 
410 	if (WARN_ON(pdev->pasid_enabled))
411 		return -EBUSY;
412 
413 	if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp)
414 		return -EINVAL;
415 
416 	if (!pasid)
417 		return -EINVAL;
418 
419 	if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))
420 		return -EINVAL;
421 
422 	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
423 	supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
424 
425 	/* User wants to enable anything unsupported? */
426 	if ((supported & features) != features)
427 		return -EINVAL;
428 
429 	control = PCI_PASID_CTRL_ENABLE | features;
430 	pdev->pasid_features = features;
431 
432 	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
433 
434 	pdev->pasid_enabled = 1;
435 
436 	return 0;
437 }
438 EXPORT_SYMBOL_GPL(pci_enable_pasid);
439 
440 /**
441  * pci_disable_pasid - Disable the PASID capability
442  * @pdev: PCI device structure
443  */
444 void pci_disable_pasid(struct pci_dev *pdev)
445 {
446 	u16 control = 0;
447 	int pasid = pdev->pasid_cap;
448 
449 	/* VFs share the PF PASID configuration */
450 	if (pdev->is_virtfn)
451 		return;
452 
453 	if (WARN_ON(!pdev->pasid_enabled))
454 		return;
455 
456 	if (!pasid)
457 		return;
458 
459 	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
460 
461 	pdev->pasid_enabled = 0;
462 }
463 EXPORT_SYMBOL_GPL(pci_disable_pasid);
464 
465 /**
466  * pci_restore_pasid_state - Restore PASID capabilities
467  * @pdev: PCI device structure
468  */
469 void pci_restore_pasid_state(struct pci_dev *pdev)
470 {
471 	u16 control;
472 	int pasid = pdev->pasid_cap;
473 
474 	if (pdev->is_virtfn)
475 		return;
476 
477 	if (!pdev->pasid_enabled)
478 		return;
479 
480 	if (!pasid)
481 		return;
482 
483 	control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
484 	pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
485 }
486 
487 /**
488  * pci_pasid_features - Check which PASID features are supported
489  * @pdev: PCI device structure
490  *
491  * Return a negative value when no PASID capability is present.
492  * Otherwise return a bitmask with supported features. Current
493  * features reported are:
494  * PCI_PASID_CAP_EXEC - Execute permission supported
495  * PCI_PASID_CAP_PRIV - Privileged mode supported
496  */
497 int pci_pasid_features(struct pci_dev *pdev)
498 {
499 	u16 supported;
500 	int pasid;
501 
502 	if (pdev->is_virtfn)
503 		pdev = pci_physfn(pdev);
504 
505 	pasid = pdev->pasid_cap;
506 	if (!pasid)
507 		return -EINVAL;
508 
509 	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
510 
511 	supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
512 
513 	return supported;
514 }
515 EXPORT_SYMBOL_GPL(pci_pasid_features);
516 
517 /**
518  * pci_max_pasids - Get maximum number of PASIDs supported by device
519  * @pdev: PCI device structure
520  *
521  * Returns negative value when PASID capability is not present.
522  * Otherwise it returns the number of supported PASIDs.
523  */
524 int pci_max_pasids(struct pci_dev *pdev)
525 {
526 	u16 supported;
527 	int pasid;
528 
529 	if (pdev->is_virtfn)
530 		pdev = pci_physfn(pdev);
531 
532 	pasid = pdev->pasid_cap;
533 	if (!pasid)
534 		return -EINVAL;
535 
536 	pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
537 
538 	return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));
539 }
540 EXPORT_SYMBOL_GPL(pci_max_pasids);
541 #endif /* CONFIG_PCI_PASID */
542