xref: /linux/drivers/pci/iov.c (revision 06d07429858317ded2db7986113a9e0129cd599b)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * PCI Express I/O Virtualization (IOV) support
4  *   Single Root IOV 1.0
5  *   Address Translation Service 1.0
6  *
7  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
8  */
9 
10 #include <linux/pci.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/string.h>
14 #include <linux/delay.h>
15 #include "pci.h"
16 
17 #define VIRTFN_ID_LEN	17	/* "virtfn%u\0" for 2^32 - 1 */
18 
pci_iov_virtfn_bus(struct pci_dev * dev,int vf_id)19 int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id)
20 {
21 	if (!dev->is_physfn)
22 		return -EINVAL;
23 	return dev->bus->number + ((dev->devfn + dev->sriov->offset +
24 				    dev->sriov->stride * vf_id) >> 8);
25 }
26 
pci_iov_virtfn_devfn(struct pci_dev * dev,int vf_id)27 int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id)
28 {
29 	if (!dev->is_physfn)
30 		return -EINVAL;
31 	return (dev->devfn + dev->sriov->offset +
32 		dev->sriov->stride * vf_id) & 0xff;
33 }
34 EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn);
35 
pci_iov_vf_id(struct pci_dev * dev)36 int pci_iov_vf_id(struct pci_dev *dev)
37 {
38 	struct pci_dev *pf;
39 
40 	if (!dev->is_virtfn)
41 		return -EINVAL;
42 
43 	pf = pci_physfn(dev);
44 	return (pci_dev_id(dev) - (pci_dev_id(pf) + pf->sriov->offset)) /
45 	       pf->sriov->stride;
46 }
47 EXPORT_SYMBOL_GPL(pci_iov_vf_id);
48 
49 /**
50  * pci_iov_get_pf_drvdata - Return the drvdata of a PF
51  * @dev: VF pci_dev
52  * @pf_driver: Device driver required to own the PF
53  *
54  * This must be called from a context that ensures that a VF driver is attached.
55  * The value returned is invalid once the VF driver completes its remove()
56  * callback.
57  *
58  * Locking is achieved by the driver core. A VF driver cannot be probed until
59  * pci_enable_sriov() is called and pci_disable_sriov() does not return until
60  * all VF drivers have completed their remove().
61  *
62  * The PF driver must call pci_disable_sriov() before it begins to destroy the
63  * drvdata.
64  */
pci_iov_get_pf_drvdata(struct pci_dev * dev,struct pci_driver * pf_driver)65 void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver)
66 {
67 	struct pci_dev *pf_dev;
68 
69 	if (!dev->is_virtfn)
70 		return ERR_PTR(-EINVAL);
71 	pf_dev = dev->physfn;
72 	if (pf_dev->driver != pf_driver)
73 		return ERR_PTR(-EINVAL);
74 	return pci_get_drvdata(pf_dev);
75 }
76 EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata);
77 
78 /*
79  * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may
80  * change when NumVFs changes.
81  *
82  * Update iov->offset and iov->stride when NumVFs is written.
83  */
pci_iov_set_numvfs(struct pci_dev * dev,int nr_virtfn)84 static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn)
85 {
86 	struct pci_sriov *iov = dev->sriov;
87 
88 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn);
89 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset);
90 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride);
91 }
92 
93 /*
94  * The PF consumes one bus number.  NumVFs, First VF Offset, and VF Stride
95  * determine how many additional bus numbers will be consumed by VFs.
96  *
97  * Iterate over all valid NumVFs, validate offset and stride, and calculate
98  * the maximum number of bus numbers that could ever be required.
99  */
compute_max_vf_buses(struct pci_dev * dev)100 static int compute_max_vf_buses(struct pci_dev *dev)
101 {
102 	struct pci_sriov *iov = dev->sriov;
103 	int nr_virtfn, busnr, rc = 0;
104 
105 	for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) {
106 		pci_iov_set_numvfs(dev, nr_virtfn);
107 		if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) {
108 			rc = -EIO;
109 			goto out;
110 		}
111 
112 		busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
113 		if (busnr > iov->max_VF_buses)
114 			iov->max_VF_buses = busnr;
115 	}
116 
117 out:
118 	pci_iov_set_numvfs(dev, 0);
119 	return rc;
120 }
121 
virtfn_add_bus(struct pci_bus * bus,int busnr)122 static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
123 {
124 	struct pci_bus *child;
125 
126 	if (bus->number == busnr)
127 		return bus;
128 
129 	child = pci_find_bus(pci_domain_nr(bus), busnr);
130 	if (child)
131 		return child;
132 
133 	child = pci_add_new_bus(bus, NULL, busnr);
134 	if (!child)
135 		return NULL;
136 
137 	pci_bus_insert_busn_res(child, busnr, busnr);
138 
139 	return child;
140 }
141 
virtfn_remove_bus(struct pci_bus * physbus,struct pci_bus * virtbus)142 static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
143 {
144 	if (physbus != virtbus && list_empty(&virtbus->devices))
145 		pci_remove_bus(virtbus);
146 }
147 
pci_iov_resource_size(struct pci_dev * dev,int resno)148 resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
149 {
150 	if (!dev->is_physfn)
151 		return 0;
152 
153 	return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
154 }
155 
pci_read_vf_config_common(struct pci_dev * virtfn)156 static void pci_read_vf_config_common(struct pci_dev *virtfn)
157 {
158 	struct pci_dev *physfn = virtfn->physfn;
159 
160 	/*
161 	 * Some config registers are the same across all associated VFs.
162 	 * Read them once from VF0 so we can skip reading them from the
163 	 * other VFs.
164 	 *
165 	 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to
166 	 * have the same Revision ID and Subsystem ID, but we assume they
167 	 * do.
168 	 */
169 	pci_read_config_dword(virtfn, PCI_CLASS_REVISION,
170 			      &physfn->sriov->class);
171 	pci_read_config_byte(virtfn, PCI_HEADER_TYPE,
172 			     &physfn->sriov->hdr_type);
173 	pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID,
174 			     &physfn->sriov->subsystem_vendor);
175 	pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID,
176 			     &physfn->sriov->subsystem_device);
177 }
178 
pci_iov_sysfs_link(struct pci_dev * dev,struct pci_dev * virtfn,int id)179 int pci_iov_sysfs_link(struct pci_dev *dev,
180 		struct pci_dev *virtfn, int id)
181 {
182 	char buf[VIRTFN_ID_LEN];
183 	int rc;
184 
185 	sprintf(buf, "virtfn%u", id);
186 	rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
187 	if (rc)
188 		goto failed;
189 	rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn");
190 	if (rc)
191 		goto failed1;
192 
193 	kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
194 
195 	return 0;
196 
197 failed1:
198 	sysfs_remove_link(&dev->dev.kobj, buf);
199 failed:
200 	return rc;
201 }
202 
203 #ifdef CONFIG_PCI_MSI
sriov_vf_total_msix_show(struct device * dev,struct device_attribute * attr,char * buf)204 static ssize_t sriov_vf_total_msix_show(struct device *dev,
205 					struct device_attribute *attr,
206 					char *buf)
207 {
208 	struct pci_dev *pdev = to_pci_dev(dev);
209 	u32 vf_total_msix = 0;
210 
211 	device_lock(dev);
212 	if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix)
213 		goto unlock;
214 
215 	vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev);
216 unlock:
217 	device_unlock(dev);
218 	return sysfs_emit(buf, "%u\n", vf_total_msix);
219 }
220 static DEVICE_ATTR_RO(sriov_vf_total_msix);
221 
sriov_vf_msix_count_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)222 static ssize_t sriov_vf_msix_count_store(struct device *dev,
223 					 struct device_attribute *attr,
224 					 const char *buf, size_t count)
225 {
226 	struct pci_dev *vf_dev = to_pci_dev(dev);
227 	struct pci_dev *pdev = pci_physfn(vf_dev);
228 	int val, ret = 0;
229 
230 	if (kstrtoint(buf, 0, &val) < 0)
231 		return -EINVAL;
232 
233 	if (val < 0)
234 		return -EINVAL;
235 
236 	device_lock(&pdev->dev);
237 	if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) {
238 		ret = -EOPNOTSUPP;
239 		goto err_pdev;
240 	}
241 
242 	device_lock(&vf_dev->dev);
243 	if (vf_dev->driver) {
244 		/*
245 		 * A driver is already attached to this VF and has configured
246 		 * itself based on the current MSI-X vector count. Changing
247 		 * the vector size could mess up the driver, so block it.
248 		 */
249 		ret = -EBUSY;
250 		goto err_dev;
251 	}
252 
253 	ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val);
254 
255 err_dev:
256 	device_unlock(&vf_dev->dev);
257 err_pdev:
258 	device_unlock(&pdev->dev);
259 	return ret ? : count;
260 }
261 static DEVICE_ATTR_WO(sriov_vf_msix_count);
262 #endif
263 
264 static struct attribute *sriov_vf_dev_attrs[] = {
265 #ifdef CONFIG_PCI_MSI
266 	&dev_attr_sriov_vf_msix_count.attr,
267 #endif
268 	NULL,
269 };
270 
sriov_vf_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)271 static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj,
272 					  struct attribute *a, int n)
273 {
274 	struct device *dev = kobj_to_dev(kobj);
275 	struct pci_dev *pdev = to_pci_dev(dev);
276 
277 	if (!pdev->is_virtfn)
278 		return 0;
279 
280 	return a->mode;
281 }
282 
283 const struct attribute_group sriov_vf_dev_attr_group = {
284 	.attrs = sriov_vf_dev_attrs,
285 	.is_visible = sriov_vf_attrs_are_visible,
286 };
287 
pci_iov_add_virtfn(struct pci_dev * dev,int id)288 int pci_iov_add_virtfn(struct pci_dev *dev, int id)
289 {
290 	int i;
291 	int rc = -ENOMEM;
292 	u64 size;
293 	struct pci_dev *virtfn;
294 	struct resource *res;
295 	struct pci_sriov *iov = dev->sriov;
296 	struct pci_bus *bus;
297 
298 	bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
299 	if (!bus)
300 		goto failed;
301 
302 	virtfn = pci_alloc_dev(bus);
303 	if (!virtfn)
304 		goto failed0;
305 
306 	virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
307 	virtfn->vendor = dev->vendor;
308 	virtfn->device = iov->vf_device;
309 	virtfn->is_virtfn = 1;
310 	virtfn->physfn = pci_dev_get(dev);
311 	virtfn->no_command_memory = 1;
312 
313 	if (id == 0)
314 		pci_read_vf_config_common(virtfn);
315 
316 	rc = pci_setup_device(virtfn);
317 	if (rc)
318 		goto failed1;
319 
320 	virtfn->dev.parent = dev->dev.parent;
321 	virtfn->multifunction = 0;
322 
323 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
324 		res = &dev->resource[i + PCI_IOV_RESOURCES];
325 		if (!res->parent)
326 			continue;
327 		virtfn->resource[i].name = pci_name(virtfn);
328 		virtfn->resource[i].flags = res->flags;
329 		size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
330 		virtfn->resource[i].start = res->start + size * id;
331 		virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
332 		rc = request_resource(res, &virtfn->resource[i]);
333 		BUG_ON(rc);
334 	}
335 
336 	pci_device_add(virtfn, virtfn->bus);
337 	rc = pci_iov_sysfs_link(dev, virtfn, id);
338 	if (rc)
339 		goto failed1;
340 
341 	pci_bus_add_device(virtfn);
342 
343 	return 0;
344 
345 failed1:
346 	pci_stop_and_remove_bus_device(virtfn);
347 	pci_dev_put(dev);
348 failed0:
349 	virtfn_remove_bus(dev->bus, bus);
350 failed:
351 
352 	return rc;
353 }
354 
pci_iov_remove_virtfn(struct pci_dev * dev,int id)355 void pci_iov_remove_virtfn(struct pci_dev *dev, int id)
356 {
357 	char buf[VIRTFN_ID_LEN];
358 	struct pci_dev *virtfn;
359 
360 	virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
361 					     pci_iov_virtfn_bus(dev, id),
362 					     pci_iov_virtfn_devfn(dev, id));
363 	if (!virtfn)
364 		return;
365 
366 	sprintf(buf, "virtfn%u", id);
367 	sysfs_remove_link(&dev->dev.kobj, buf);
368 	/*
369 	 * pci_stop_dev() could have been called for this virtfn already,
370 	 * so the directory for the virtfn may have been removed before.
371 	 * Double check to avoid spurious sysfs warnings.
372 	 */
373 	if (virtfn->dev.kobj.sd)
374 		sysfs_remove_link(&virtfn->dev.kobj, "physfn");
375 
376 	pci_stop_and_remove_bus_device(virtfn);
377 	virtfn_remove_bus(dev->bus, virtfn->bus);
378 
379 	/* balance pci_get_domain_bus_and_slot() */
380 	pci_dev_put(virtfn);
381 	pci_dev_put(dev);
382 }
383 
sriov_totalvfs_show(struct device * dev,struct device_attribute * attr,char * buf)384 static ssize_t sriov_totalvfs_show(struct device *dev,
385 				   struct device_attribute *attr,
386 				   char *buf)
387 {
388 	struct pci_dev *pdev = to_pci_dev(dev);
389 
390 	return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev));
391 }
392 
sriov_numvfs_show(struct device * dev,struct device_attribute * attr,char * buf)393 static ssize_t sriov_numvfs_show(struct device *dev,
394 				 struct device_attribute *attr,
395 				 char *buf)
396 {
397 	struct pci_dev *pdev = to_pci_dev(dev);
398 	u16 num_vfs;
399 
400 	/* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */
401 	device_lock(&pdev->dev);
402 	num_vfs = pdev->sriov->num_VFs;
403 	device_unlock(&pdev->dev);
404 
405 	return sysfs_emit(buf, "%u\n", num_vfs);
406 }
407 
408 /*
409  * num_vfs > 0; number of VFs to enable
410  * num_vfs = 0; disable all VFs
411  *
412  * Note: SRIOV spec does not allow partial VF
413  *	 disable, so it's all or none.
414  */
sriov_numvfs_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)415 static ssize_t sriov_numvfs_store(struct device *dev,
416 				  struct device_attribute *attr,
417 				  const char *buf, size_t count)
418 {
419 	struct pci_dev *pdev = to_pci_dev(dev);
420 	int ret = 0;
421 	u16 num_vfs;
422 
423 	if (kstrtou16(buf, 0, &num_vfs) < 0)
424 		return -EINVAL;
425 
426 	if (num_vfs > pci_sriov_get_totalvfs(pdev))
427 		return -ERANGE;
428 
429 	device_lock(&pdev->dev);
430 
431 	if (num_vfs == pdev->sriov->num_VFs)
432 		goto exit;
433 
434 	/* is PF driver loaded */
435 	if (!pdev->driver) {
436 		pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n");
437 		ret = -ENOENT;
438 		goto exit;
439 	}
440 
441 	/* is PF driver loaded w/callback */
442 	if (!pdev->driver->sriov_configure) {
443 		pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n");
444 		ret = -ENOENT;
445 		goto exit;
446 	}
447 
448 	if (num_vfs == 0) {
449 		/* disable VFs */
450 		ret = pdev->driver->sriov_configure(pdev, 0);
451 		goto exit;
452 	}
453 
454 	/* enable VFs */
455 	if (pdev->sriov->num_VFs) {
456 		pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n",
457 			 pdev->sriov->num_VFs, num_vfs);
458 		ret = -EBUSY;
459 		goto exit;
460 	}
461 
462 	ret = pdev->driver->sriov_configure(pdev, num_vfs);
463 	if (ret < 0)
464 		goto exit;
465 
466 	if (ret != num_vfs)
467 		pci_warn(pdev, "%d VFs requested; only %d enabled\n",
468 			 num_vfs, ret);
469 
470 exit:
471 	device_unlock(&pdev->dev);
472 
473 	if (ret < 0)
474 		return ret;
475 
476 	return count;
477 }
478 
sriov_offset_show(struct device * dev,struct device_attribute * attr,char * buf)479 static ssize_t sriov_offset_show(struct device *dev,
480 				 struct device_attribute *attr,
481 				 char *buf)
482 {
483 	struct pci_dev *pdev = to_pci_dev(dev);
484 
485 	return sysfs_emit(buf, "%u\n", pdev->sriov->offset);
486 }
487 
sriov_stride_show(struct device * dev,struct device_attribute * attr,char * buf)488 static ssize_t sriov_stride_show(struct device *dev,
489 				 struct device_attribute *attr,
490 				 char *buf)
491 {
492 	struct pci_dev *pdev = to_pci_dev(dev);
493 
494 	return sysfs_emit(buf, "%u\n", pdev->sriov->stride);
495 }
496 
sriov_vf_device_show(struct device * dev,struct device_attribute * attr,char * buf)497 static ssize_t sriov_vf_device_show(struct device *dev,
498 				    struct device_attribute *attr,
499 				    char *buf)
500 {
501 	struct pci_dev *pdev = to_pci_dev(dev);
502 
503 	return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device);
504 }
505 
sriov_drivers_autoprobe_show(struct device * dev,struct device_attribute * attr,char * buf)506 static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
507 					    struct device_attribute *attr,
508 					    char *buf)
509 {
510 	struct pci_dev *pdev = to_pci_dev(dev);
511 
512 	return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe);
513 }
514 
sriov_drivers_autoprobe_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)515 static ssize_t sriov_drivers_autoprobe_store(struct device *dev,
516 					     struct device_attribute *attr,
517 					     const char *buf, size_t count)
518 {
519 	struct pci_dev *pdev = to_pci_dev(dev);
520 	bool drivers_autoprobe;
521 
522 	if (kstrtobool(buf, &drivers_autoprobe) < 0)
523 		return -EINVAL;
524 
525 	pdev->sriov->drivers_autoprobe = drivers_autoprobe;
526 
527 	return count;
528 }
529 
530 static DEVICE_ATTR_RO(sriov_totalvfs);
531 static DEVICE_ATTR_RW(sriov_numvfs);
532 static DEVICE_ATTR_RO(sriov_offset);
533 static DEVICE_ATTR_RO(sriov_stride);
534 static DEVICE_ATTR_RO(sriov_vf_device);
535 static DEVICE_ATTR_RW(sriov_drivers_autoprobe);
536 
537 static struct attribute *sriov_pf_dev_attrs[] = {
538 	&dev_attr_sriov_totalvfs.attr,
539 	&dev_attr_sriov_numvfs.attr,
540 	&dev_attr_sriov_offset.attr,
541 	&dev_attr_sriov_stride.attr,
542 	&dev_attr_sriov_vf_device.attr,
543 	&dev_attr_sriov_drivers_autoprobe.attr,
544 #ifdef CONFIG_PCI_MSI
545 	&dev_attr_sriov_vf_total_msix.attr,
546 #endif
547 	NULL,
548 };
549 
sriov_pf_attrs_are_visible(struct kobject * kobj,struct attribute * a,int n)550 static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj,
551 					  struct attribute *a, int n)
552 {
553 	struct device *dev = kobj_to_dev(kobj);
554 
555 	if (!dev_is_pf(dev))
556 		return 0;
557 
558 	return a->mode;
559 }
560 
561 const struct attribute_group sriov_pf_dev_attr_group = {
562 	.attrs = sriov_pf_dev_attrs,
563 	.is_visible = sriov_pf_attrs_are_visible,
564 };
565 
pcibios_sriov_enable(struct pci_dev * pdev,u16 num_vfs)566 int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
567 {
568 	return 0;
569 }
570 
pcibios_sriov_disable(struct pci_dev * pdev)571 int __weak pcibios_sriov_disable(struct pci_dev *pdev)
572 {
573 	return 0;
574 }
575 
sriov_add_vfs(struct pci_dev * dev,u16 num_vfs)576 static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs)
577 {
578 	unsigned int i;
579 	int rc;
580 
581 	if (dev->no_vf_scan)
582 		return 0;
583 
584 	for (i = 0; i < num_vfs; i++) {
585 		rc = pci_iov_add_virtfn(dev, i);
586 		if (rc)
587 			goto failed;
588 	}
589 	return 0;
590 failed:
591 	while (i--)
592 		pci_iov_remove_virtfn(dev, i);
593 
594 	return rc;
595 }
596 
sriov_enable(struct pci_dev * dev,int nr_virtfn)597 static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
598 {
599 	int rc;
600 	int i;
601 	int nres;
602 	u16 initial;
603 	struct resource *res;
604 	struct pci_dev *pdev;
605 	struct pci_sriov *iov = dev->sriov;
606 	int bars = 0;
607 	int bus;
608 
609 	if (!nr_virtfn)
610 		return 0;
611 
612 	if (iov->num_VFs)
613 		return -EINVAL;
614 
615 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial);
616 	if (initial > iov->total_VFs ||
617 	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs)))
618 		return -EIO;
619 
620 	if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs ||
621 	    (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial)))
622 		return -EINVAL;
623 
624 	nres = 0;
625 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
626 		bars |= (1 << (i + PCI_IOV_RESOURCES));
627 		res = &dev->resource[i + PCI_IOV_RESOURCES];
628 		if (res->parent)
629 			nres++;
630 	}
631 	if (nres != iov->nres) {
632 		pci_err(dev, "not enough MMIO resources for SR-IOV\n");
633 		return -ENOMEM;
634 	}
635 
636 	bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1);
637 	if (bus > dev->bus->busn_res.end) {
638 		pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n",
639 			nr_virtfn, bus, &dev->bus->busn_res);
640 		return -ENOMEM;
641 	}
642 
643 	if (pci_enable_resources(dev, bars)) {
644 		pci_err(dev, "SR-IOV: IOV BARS not allocated\n");
645 		return -ENOMEM;
646 	}
647 
648 	if (iov->link != dev->devfn) {
649 		pdev = pci_get_slot(dev->bus, iov->link);
650 		if (!pdev)
651 			return -ENODEV;
652 
653 		if (!pdev->is_physfn) {
654 			pci_dev_put(pdev);
655 			return -ENOSYS;
656 		}
657 
658 		rc = sysfs_create_link(&dev->dev.kobj,
659 					&pdev->dev.kobj, "dep_link");
660 		pci_dev_put(pdev);
661 		if (rc)
662 			return rc;
663 	}
664 
665 	iov->initial_VFs = initial;
666 	if (nr_virtfn < initial)
667 		initial = nr_virtfn;
668 
669 	rc = pcibios_sriov_enable(dev, initial);
670 	if (rc) {
671 		pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc);
672 		goto err_pcibios;
673 	}
674 
675 	pci_iov_set_numvfs(dev, nr_virtfn);
676 	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
677 	pci_cfg_access_lock(dev);
678 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
679 	msleep(100);
680 	pci_cfg_access_unlock(dev);
681 
682 	rc = sriov_add_vfs(dev, initial);
683 	if (rc)
684 		goto err_pcibios;
685 
686 	kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE);
687 	iov->num_VFs = nr_virtfn;
688 
689 	return 0;
690 
691 err_pcibios:
692 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
693 	pci_cfg_access_lock(dev);
694 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
695 	ssleep(1);
696 	pci_cfg_access_unlock(dev);
697 
698 	pcibios_sriov_disable(dev);
699 
700 	if (iov->link != dev->devfn)
701 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
702 
703 	pci_iov_set_numvfs(dev, 0);
704 	return rc;
705 }
706 
sriov_del_vfs(struct pci_dev * dev)707 static void sriov_del_vfs(struct pci_dev *dev)
708 {
709 	struct pci_sriov *iov = dev->sriov;
710 	int i;
711 
712 	for (i = 0; i < iov->num_VFs; i++)
713 		pci_iov_remove_virtfn(dev, i);
714 }
715 
sriov_disable(struct pci_dev * dev)716 static void sriov_disable(struct pci_dev *dev)
717 {
718 	struct pci_sriov *iov = dev->sriov;
719 
720 	if (!iov->num_VFs)
721 		return;
722 
723 	sriov_del_vfs(dev);
724 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
725 	pci_cfg_access_lock(dev);
726 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
727 	ssleep(1);
728 	pci_cfg_access_unlock(dev);
729 
730 	pcibios_sriov_disable(dev);
731 
732 	if (iov->link != dev->devfn)
733 		sysfs_remove_link(&dev->dev.kobj, "dep_link");
734 
735 	iov->num_VFs = 0;
736 	pci_iov_set_numvfs(dev, 0);
737 }
738 
sriov_init(struct pci_dev * dev,int pos)739 static int sriov_init(struct pci_dev *dev, int pos)
740 {
741 	int i, bar64;
742 	int rc;
743 	int nres;
744 	u32 pgsz;
745 	u16 ctrl, total;
746 	struct pci_sriov *iov;
747 	struct resource *res;
748 	const char *res_name;
749 	struct pci_dev *pdev;
750 
751 	pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl);
752 	if (ctrl & PCI_SRIOV_CTRL_VFE) {
753 		pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0);
754 		ssleep(1);
755 	}
756 
757 	ctrl = 0;
758 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
759 		if (pdev->is_physfn)
760 			goto found;
761 
762 	pdev = NULL;
763 	if (pci_ari_enabled(dev->bus))
764 		ctrl |= PCI_SRIOV_CTRL_ARI;
765 
766 found:
767 	pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl);
768 
769 	pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total);
770 	if (!total)
771 		return 0;
772 
773 	pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz);
774 	i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0;
775 	pgsz &= ~((1 << i) - 1);
776 	if (!pgsz)
777 		return -EIO;
778 
779 	pgsz &= ~(pgsz - 1);
780 	pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz);
781 
782 	iov = kzalloc(sizeof(*iov), GFP_KERNEL);
783 	if (!iov)
784 		return -ENOMEM;
785 
786 	nres = 0;
787 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
788 		res = &dev->resource[i + PCI_IOV_RESOURCES];
789 		res_name = pci_resource_name(dev, i + PCI_IOV_RESOURCES);
790 
791 		/*
792 		 * If it is already FIXED, don't change it, something
793 		 * (perhaps EA or header fixups) wants it this way.
794 		 */
795 		if (res->flags & IORESOURCE_PCI_FIXED)
796 			bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
797 		else
798 			bar64 = __pci_read_base(dev, pci_bar_unknown, res,
799 						pos + PCI_SRIOV_BAR + i * 4);
800 		if (!res->flags)
801 			continue;
802 		if (resource_size(res) & (PAGE_SIZE - 1)) {
803 			rc = -EIO;
804 			goto failed;
805 		}
806 		iov->barsz[i] = resource_size(res);
807 		res->end = res->start + resource_size(res) * total - 1;
808 		pci_info(dev, "%s %pR: contains BAR %d for %d VFs\n",
809 			 res_name, res, i, total);
810 		i += bar64;
811 		nres++;
812 	}
813 
814 	iov->pos = pos;
815 	iov->nres = nres;
816 	iov->ctrl = ctrl;
817 	iov->total_VFs = total;
818 	iov->driver_max_VFs = total;
819 	pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device);
820 	iov->pgsz = pgsz;
821 	iov->self = dev;
822 	iov->drivers_autoprobe = true;
823 	pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap);
824 	pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link);
825 	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)
826 		iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link);
827 
828 	if (pdev)
829 		iov->dev = pci_dev_get(pdev);
830 	else
831 		iov->dev = dev;
832 
833 	dev->sriov = iov;
834 	dev->is_physfn = 1;
835 	rc = compute_max_vf_buses(dev);
836 	if (rc)
837 		goto fail_max_buses;
838 
839 	return 0;
840 
841 fail_max_buses:
842 	dev->sriov = NULL;
843 	dev->is_physfn = 0;
844 failed:
845 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
846 		res = &dev->resource[i + PCI_IOV_RESOURCES];
847 		res->flags = 0;
848 	}
849 
850 	kfree(iov);
851 	return rc;
852 }
853 
sriov_release(struct pci_dev * dev)854 static void sriov_release(struct pci_dev *dev)
855 {
856 	BUG_ON(dev->sriov->num_VFs);
857 
858 	if (dev != dev->sriov->dev)
859 		pci_dev_put(dev->sriov->dev);
860 
861 	kfree(dev->sriov);
862 	dev->sriov = NULL;
863 }
864 
sriov_restore_state(struct pci_dev * dev)865 static void sriov_restore_state(struct pci_dev *dev)
866 {
867 	int i;
868 	u16 ctrl;
869 	struct pci_sriov *iov = dev->sriov;
870 
871 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl);
872 	if (ctrl & PCI_SRIOV_CTRL_VFE)
873 		return;
874 
875 	/*
876 	 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because
877 	 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI.
878 	 */
879 	ctrl &= ~PCI_SRIOV_CTRL_ARI;
880 	ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI;
881 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl);
882 
883 	for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
884 		pci_update_resource(dev, i + PCI_IOV_RESOURCES);
885 
886 	pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz);
887 	pci_iov_set_numvfs(dev, iov->num_VFs);
888 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
889 	if (iov->ctrl & PCI_SRIOV_CTRL_VFE)
890 		msleep(100);
891 }
892 
893 /**
894  * pci_iov_init - initialize the IOV capability
895  * @dev: the PCI device
896  *
897  * Returns 0 on success, or negative on failure.
898  */
pci_iov_init(struct pci_dev * dev)899 int pci_iov_init(struct pci_dev *dev)
900 {
901 	int pos;
902 
903 	if (!pci_is_pcie(dev))
904 		return -ENODEV;
905 
906 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV);
907 	if (pos)
908 		return sriov_init(dev, pos);
909 
910 	return -ENODEV;
911 }
912 
913 /**
914  * pci_iov_release - release resources used by the IOV capability
915  * @dev: the PCI device
916  */
pci_iov_release(struct pci_dev * dev)917 void pci_iov_release(struct pci_dev *dev)
918 {
919 	if (dev->is_physfn)
920 		sriov_release(dev);
921 }
922 
923 /**
924  * pci_iov_remove - clean up SR-IOV state after PF driver is detached
925  * @dev: the PCI device
926  */
pci_iov_remove(struct pci_dev * dev)927 void pci_iov_remove(struct pci_dev *dev)
928 {
929 	struct pci_sriov *iov = dev->sriov;
930 
931 	if (!dev->is_physfn)
932 		return;
933 
934 	iov->driver_max_VFs = iov->total_VFs;
935 	if (iov->num_VFs)
936 		pci_warn(dev, "driver left SR-IOV enabled after remove\n");
937 }
938 
939 /**
940  * pci_iov_update_resource - update a VF BAR
941  * @dev: the PCI device
942  * @resno: the resource number
943  *
944  * Update a VF BAR in the SR-IOV capability of a PF.
945  */
pci_iov_update_resource(struct pci_dev * dev,int resno)946 void pci_iov_update_resource(struct pci_dev *dev, int resno)
947 {
948 	struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL;
949 	struct resource *res = dev->resource + resno;
950 	int vf_bar = resno - PCI_IOV_RESOURCES;
951 	struct pci_bus_region region;
952 	u16 cmd;
953 	u32 new;
954 	int reg;
955 
956 	/*
957 	 * The generic pci_restore_bars() path calls this for all devices,
958 	 * including VFs and non-SR-IOV devices.  If this is not a PF, we
959 	 * have nothing to do.
960 	 */
961 	if (!iov)
962 		return;
963 
964 	pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd);
965 	if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) {
966 		dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n",
967 			 vf_bar, res);
968 		return;
969 	}
970 
971 	/*
972 	 * Ignore unimplemented BARs, unused resource slots for 64-bit
973 	 * BARs, and non-movable resources, e.g., those described via
974 	 * Enhanced Allocation.
975 	 */
976 	if (!res->flags)
977 		return;
978 
979 	if (res->flags & IORESOURCE_UNSET)
980 		return;
981 
982 	if (res->flags & IORESOURCE_PCI_FIXED)
983 		return;
984 
985 	pcibios_resource_to_bus(dev->bus, &region, res);
986 	new = region.start;
987 	new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK;
988 
989 	reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar;
990 	pci_write_config_dword(dev, reg, new);
991 	if (res->flags & IORESOURCE_MEM_64) {
992 		new = region.start >> 16 >> 16;
993 		pci_write_config_dword(dev, reg + 4, new);
994 	}
995 }
996 
pcibios_iov_resource_alignment(struct pci_dev * dev,int resno)997 resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
998 						      int resno)
999 {
1000 	return pci_iov_resource_size(dev, resno);
1001 }
1002 
1003 /**
1004  * pci_sriov_resource_alignment - get resource alignment for VF BAR
1005  * @dev: the PCI device
1006  * @resno: the resource number
1007  *
1008  * Returns the alignment of the VF BAR found in the SR-IOV capability.
1009  * This is not the same as the resource size which is defined as
1010  * the VF BAR size multiplied by the number of VFs.  The alignment
1011  * is just the VF BAR size.
1012  */
pci_sriov_resource_alignment(struct pci_dev * dev,int resno)1013 resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
1014 {
1015 	return pcibios_iov_resource_alignment(dev, resno);
1016 }
1017 
1018 /**
1019  * pci_restore_iov_state - restore the state of the IOV capability
1020  * @dev: the PCI device
1021  */
pci_restore_iov_state(struct pci_dev * dev)1022 void pci_restore_iov_state(struct pci_dev *dev)
1023 {
1024 	if (dev->is_physfn)
1025 		sriov_restore_state(dev);
1026 }
1027 
1028 /**
1029  * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs
1030  * @dev: the PCI device
1031  * @auto_probe: set VF drivers auto probe flag
1032  */
pci_vf_drivers_autoprobe(struct pci_dev * dev,bool auto_probe)1033 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe)
1034 {
1035 	if (dev->is_physfn)
1036 		dev->sriov->drivers_autoprobe = auto_probe;
1037 }
1038 
1039 /**
1040  * pci_iov_bus_range - find bus range used by Virtual Function
1041  * @bus: the PCI bus
1042  *
1043  * Returns max number of buses (exclude current one) used by Virtual
1044  * Functions.
1045  */
pci_iov_bus_range(struct pci_bus * bus)1046 int pci_iov_bus_range(struct pci_bus *bus)
1047 {
1048 	int max = 0;
1049 	struct pci_dev *dev;
1050 
1051 	list_for_each_entry(dev, &bus->devices, bus_list) {
1052 		if (!dev->is_physfn)
1053 			continue;
1054 		if (dev->sriov->max_VF_buses > max)
1055 			max = dev->sriov->max_VF_buses;
1056 	}
1057 
1058 	return max ? max - bus->number : 0;
1059 }
1060 
1061 /**
1062  * pci_enable_sriov - enable the SR-IOV capability
1063  * @dev: the PCI device
1064  * @nr_virtfn: number of virtual functions to enable
1065  *
1066  * Returns 0 on success, or negative on failure.
1067  */
pci_enable_sriov(struct pci_dev * dev,int nr_virtfn)1068 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
1069 {
1070 	might_sleep();
1071 
1072 	if (!dev->is_physfn)
1073 		return -ENOSYS;
1074 
1075 	return sriov_enable(dev, nr_virtfn);
1076 }
1077 EXPORT_SYMBOL_GPL(pci_enable_sriov);
1078 
1079 /**
1080  * pci_disable_sriov - disable the SR-IOV capability
1081  * @dev: the PCI device
1082  */
pci_disable_sriov(struct pci_dev * dev)1083 void pci_disable_sriov(struct pci_dev *dev)
1084 {
1085 	might_sleep();
1086 
1087 	if (!dev->is_physfn)
1088 		return;
1089 
1090 	sriov_disable(dev);
1091 }
1092 EXPORT_SYMBOL_GPL(pci_disable_sriov);
1093 
1094 /**
1095  * pci_num_vf - return number of VFs associated with a PF device_release_driver
1096  * @dev: the PCI device
1097  *
1098  * Returns number of VFs, or 0 if SR-IOV is not enabled.
1099  */
pci_num_vf(struct pci_dev * dev)1100 int pci_num_vf(struct pci_dev *dev)
1101 {
1102 	if (!dev->is_physfn)
1103 		return 0;
1104 
1105 	return dev->sriov->num_VFs;
1106 }
1107 EXPORT_SYMBOL_GPL(pci_num_vf);
1108 
1109 /**
1110  * pci_vfs_assigned - returns number of VFs are assigned to a guest
1111  * @dev: the PCI device
1112  *
1113  * Returns number of VFs belonging to this device that are assigned to a guest.
1114  * If device is not a physical function returns 0.
1115  */
pci_vfs_assigned(struct pci_dev * dev)1116 int pci_vfs_assigned(struct pci_dev *dev)
1117 {
1118 	struct pci_dev *vfdev;
1119 	unsigned int vfs_assigned = 0;
1120 	unsigned short dev_id;
1121 
1122 	/* only search if we are a PF */
1123 	if (!dev->is_physfn)
1124 		return 0;
1125 
1126 	/*
1127 	 * determine the device ID for the VFs, the vendor ID will be the
1128 	 * same as the PF so there is no need to check for that one
1129 	 */
1130 	dev_id = dev->sriov->vf_device;
1131 
1132 	/* loop through all the VFs to see if we own any that are assigned */
1133 	vfdev = pci_get_device(dev->vendor, dev_id, NULL);
1134 	while (vfdev) {
1135 		/*
1136 		 * It is considered assigned if it is a virtual function with
1137 		 * our dev as the physical function and the assigned bit is set
1138 		 */
1139 		if (vfdev->is_virtfn && (vfdev->physfn == dev) &&
1140 			pci_is_dev_assigned(vfdev))
1141 			vfs_assigned++;
1142 
1143 		vfdev = pci_get_device(dev->vendor, dev_id, vfdev);
1144 	}
1145 
1146 	return vfs_assigned;
1147 }
1148 EXPORT_SYMBOL_GPL(pci_vfs_assigned);
1149 
1150 /**
1151  * pci_sriov_set_totalvfs -- reduce the TotalVFs available
1152  * @dev: the PCI PF device
1153  * @numvfs: number that should be used for TotalVFs supported
1154  *
1155  * Should be called from PF driver's probe routine with
1156  * device's mutex held.
1157  *
1158  * Returns 0 if PF is an SRIOV-capable device and
1159  * value of numvfs valid. If not a PF return -ENOSYS;
1160  * if numvfs is invalid return -EINVAL;
1161  * if VFs already enabled, return -EBUSY.
1162  */
pci_sriov_set_totalvfs(struct pci_dev * dev,u16 numvfs)1163 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
1164 {
1165 	if (!dev->is_physfn)
1166 		return -ENOSYS;
1167 
1168 	if (numvfs > dev->sriov->total_VFs)
1169 		return -EINVAL;
1170 
1171 	/* Shouldn't change if VFs already enabled */
1172 	if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE)
1173 		return -EBUSY;
1174 
1175 	dev->sriov->driver_max_VFs = numvfs;
1176 	return 0;
1177 }
1178 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs);
1179 
1180 /**
1181  * pci_sriov_get_totalvfs -- get total VFs supported on this device
1182  * @dev: the PCI PF device
1183  *
1184  * For a PCIe device with SRIOV support, return the PCIe
1185  * SRIOV capability value of TotalVFs or the value of driver_max_VFs
1186  * if the driver reduced it.  Otherwise 0.
1187  */
pci_sriov_get_totalvfs(struct pci_dev * dev)1188 int pci_sriov_get_totalvfs(struct pci_dev *dev)
1189 {
1190 	if (!dev->is_physfn)
1191 		return 0;
1192 
1193 	return dev->sriov->driver_max_VFs;
1194 }
1195 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs);
1196 
1197 /**
1198  * pci_sriov_configure_simple - helper to configure SR-IOV
1199  * @dev: the PCI device
1200  * @nr_virtfn: number of virtual functions to enable, 0 to disable
1201  *
1202  * Enable or disable SR-IOV for devices that don't require any PF setup
1203  * before enabling SR-IOV.  Return value is negative on error, or number of
1204  * VFs allocated on success.
1205  */
pci_sriov_configure_simple(struct pci_dev * dev,int nr_virtfn)1206 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn)
1207 {
1208 	int rc;
1209 
1210 	might_sleep();
1211 
1212 	if (!dev->is_physfn)
1213 		return -ENODEV;
1214 
1215 	if (pci_vfs_assigned(dev)) {
1216 		pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n");
1217 		return -EPERM;
1218 	}
1219 
1220 	if (nr_virtfn == 0) {
1221 		sriov_disable(dev);
1222 		return 0;
1223 	}
1224 
1225 	rc = sriov_enable(dev, nr_virtfn);
1226 	if (rc < 0)
1227 		return rc;
1228 
1229 	return nr_virtfn;
1230 }
1231 EXPORT_SYMBOL_GPL(pci_sriov_configure_simple);
1232