xref: /linux/drivers/pci/pci-driver.c (revision b889fcf63cb62e7fdb7816565e28f44dbe4a76a5)
1 /*
2  * drivers/pci/pci-driver.c
3  *
4  * (C) Copyright 2002-2004, 2007 Greg Kroah-Hartman <greg@kroah.com>
5  * (C) Copyright 2007 Novell Inc.
6  *
7  * Released under the GPL v2 only.
8  *
9  */
10 
11 #include <linux/pci.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/device.h>
15 #include <linux/mempolicy.h>
16 #include <linux/string.h>
17 #include <linux/slab.h>
18 #include <linux/sched.h>
19 #include <linux/cpu.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/suspend.h>
22 #include "pci.h"
23 
24 struct pci_dynid {
25 	struct list_head node;
26 	struct pci_device_id id;
27 };
28 
29 /**
30  * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
31  * @drv: target pci driver
32  * @vendor: PCI vendor ID
33  * @device: PCI device ID
34  * @subvendor: PCI subvendor ID
35  * @subdevice: PCI subdevice ID
36  * @class: PCI class
37  * @class_mask: PCI class mask
38  * @driver_data: private driver data
39  *
40  * Adds a new dynamic pci device ID to this driver and causes the
41  * driver to probe for all devices again.  @drv must have been
42  * registered prior to calling this function.
43  *
44  * CONTEXT:
45  * Does GFP_KERNEL allocation.
46  *
47  * RETURNS:
48  * 0 on success, -errno on failure.
49  */
50 int pci_add_dynid(struct pci_driver *drv,
51 		  unsigned int vendor, unsigned int device,
52 		  unsigned int subvendor, unsigned int subdevice,
53 		  unsigned int class, unsigned int class_mask,
54 		  unsigned long driver_data)
55 {
56 	struct pci_dynid *dynid;
57 	int retval;
58 
59 	dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
60 	if (!dynid)
61 		return -ENOMEM;
62 
63 	dynid->id.vendor = vendor;
64 	dynid->id.device = device;
65 	dynid->id.subvendor = subvendor;
66 	dynid->id.subdevice = subdevice;
67 	dynid->id.class = class;
68 	dynid->id.class_mask = class_mask;
69 	dynid->id.driver_data = driver_data;
70 
71 	spin_lock(&drv->dynids.lock);
72 	list_add_tail(&dynid->node, &drv->dynids.list);
73 	spin_unlock(&drv->dynids.lock);
74 
75 	retval = driver_attach(&drv->driver);
76 
77 	return retval;
78 }
79 
80 static void pci_free_dynids(struct pci_driver *drv)
81 {
82 	struct pci_dynid *dynid, *n;
83 
84 	spin_lock(&drv->dynids.lock);
85 	list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
86 		list_del(&dynid->node);
87 		kfree(dynid);
88 	}
89 	spin_unlock(&drv->dynids.lock);
90 }
91 
92 /**
93  * store_new_id - sysfs frontend to pci_add_dynid()
94  * @driver: target device driver
95  * @buf: buffer for scanning device ID data
96  * @count: input size
97  *
98  * Allow PCI IDs to be added to an existing driver via sysfs.
99  */
100 static ssize_t
101 store_new_id(struct device_driver *driver, const char *buf, size_t count)
102 {
103 	struct pci_driver *pdrv = to_pci_driver(driver);
104 	const struct pci_device_id *ids = pdrv->id_table;
105 	__u32 vendor, device, subvendor=PCI_ANY_ID,
106 		subdevice=PCI_ANY_ID, class=0, class_mask=0;
107 	unsigned long driver_data=0;
108 	int fields=0;
109 	int retval;
110 
111 	fields = sscanf(buf, "%x %x %x %x %x %x %lx",
112 			&vendor, &device, &subvendor, &subdevice,
113 			&class, &class_mask, &driver_data);
114 	if (fields < 2)
115 		return -EINVAL;
116 
117 	/* Only accept driver_data values that match an existing id_table
118 	   entry */
119 	if (ids) {
120 		retval = -EINVAL;
121 		while (ids->vendor || ids->subvendor || ids->class_mask) {
122 			if (driver_data == ids->driver_data) {
123 				retval = 0;
124 				break;
125 			}
126 			ids++;
127 		}
128 		if (retval)	/* No match */
129 			return retval;
130 	}
131 
132 	retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
133 			       class, class_mask, driver_data);
134 	if (retval)
135 		return retval;
136 	return count;
137 }
138 
139 /**
140  * store_remove_id - remove a PCI device ID from this driver
141  * @driver: target device driver
142  * @buf: buffer for scanning device ID data
143  * @count: input size
144  *
145  * Removes a dynamic pci device ID to this driver.
146  */
147 static ssize_t
148 store_remove_id(struct device_driver *driver, const char *buf, size_t count)
149 {
150 	struct pci_dynid *dynid, *n;
151 	struct pci_driver *pdrv = to_pci_driver(driver);
152 	__u32 vendor, device, subvendor = PCI_ANY_ID,
153 		subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
154 	int fields = 0;
155 	int retval = -ENODEV;
156 
157 	fields = sscanf(buf, "%x %x %x %x %x %x",
158 			&vendor, &device, &subvendor, &subdevice,
159 			&class, &class_mask);
160 	if (fields < 2)
161 		return -EINVAL;
162 
163 	spin_lock(&pdrv->dynids.lock);
164 	list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
165 		struct pci_device_id *id = &dynid->id;
166 		if ((id->vendor == vendor) &&
167 		    (id->device == device) &&
168 		    (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
169 		    (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
170 		    !((id->class ^ class) & class_mask)) {
171 			list_del(&dynid->node);
172 			kfree(dynid);
173 			retval = 0;
174 			break;
175 		}
176 	}
177 	spin_unlock(&pdrv->dynids.lock);
178 
179 	if (retval)
180 		return retval;
181 	return count;
182 }
183 
184 static struct driver_attribute pci_drv_attrs[] = {
185 	__ATTR(new_id, S_IWUSR, NULL, store_new_id),
186 	__ATTR(remove_id, S_IWUSR, NULL, store_remove_id),
187 	__ATTR_NULL,
188 };
189 
190 /**
191  * pci_match_id - See if a pci device matches a given pci_id table
192  * @ids: array of PCI device id structures to search in
193  * @dev: the PCI device structure to match against.
194  *
195  * Used by a driver to check whether a PCI device present in the
196  * system is in its list of supported devices.  Returns the matching
197  * pci_device_id structure or %NULL if there is no match.
198  *
199  * Deprecated, don't use this as it will not catch any dynamic ids
200  * that a driver might want to check for.
201  */
202 const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
203 					 struct pci_dev *dev)
204 {
205 	if (ids) {
206 		while (ids->vendor || ids->subvendor || ids->class_mask) {
207 			if (pci_match_one_device(ids, dev))
208 				return ids;
209 			ids++;
210 		}
211 	}
212 	return NULL;
213 }
214 
215 /**
216  * pci_match_device - Tell if a PCI device structure has a matching PCI device id structure
217  * @drv: the PCI driver to match against
218  * @dev: the PCI device structure to match against
219  *
220  * Used by a driver to check whether a PCI device present in the
221  * system is in its list of supported devices.  Returns the matching
222  * pci_device_id structure or %NULL if there is no match.
223  */
224 static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
225 						    struct pci_dev *dev)
226 {
227 	struct pci_dynid *dynid;
228 
229 	/* Look at the dynamic ids first, before the static ones */
230 	spin_lock(&drv->dynids.lock);
231 	list_for_each_entry(dynid, &drv->dynids.list, node) {
232 		if (pci_match_one_device(&dynid->id, dev)) {
233 			spin_unlock(&drv->dynids.lock);
234 			return &dynid->id;
235 		}
236 	}
237 	spin_unlock(&drv->dynids.lock);
238 
239 	return pci_match_id(drv->id_table, dev);
240 }
241 
242 struct drv_dev_and_id {
243 	struct pci_driver *drv;
244 	struct pci_dev *dev;
245 	const struct pci_device_id *id;
246 };
247 
248 static long local_pci_probe(void *_ddi)
249 {
250 	struct drv_dev_and_id *ddi = _ddi;
251 	struct pci_dev *pci_dev = ddi->dev;
252 	struct pci_driver *pci_drv = ddi->drv;
253 	struct device *dev = &pci_dev->dev;
254 	int rc;
255 
256 	/*
257 	 * Unbound PCI devices are always put in D0, regardless of
258 	 * runtime PM status.  During probe, the device is set to
259 	 * active and the usage count is incremented.  If the driver
260 	 * supports runtime PM, it should call pm_runtime_put_noidle()
261 	 * in its probe routine and pm_runtime_get_noresume() in its
262 	 * remove routine.
263 	 */
264 	pm_runtime_get_sync(dev);
265 	pci_dev->driver = pci_drv;
266 	rc = pci_drv->probe(pci_dev, ddi->id);
267 	if (rc) {
268 		pci_dev->driver = NULL;
269 		pm_runtime_put_sync(dev);
270 	}
271 	return rc;
272 }
273 
274 static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
275 			  const struct pci_device_id *id)
276 {
277 	int error, node;
278 	struct drv_dev_and_id ddi = { drv, dev, id };
279 
280 	/* Execute driver initialization on node where the device's
281 	   bus is attached to.  This way the driver likely allocates
282 	   its local memory on the right node without any need to
283 	   change it. */
284 	node = dev_to_node(&dev->dev);
285 	if (node >= 0) {
286 		int cpu;
287 
288 		get_online_cpus();
289 		cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
290 		if (cpu < nr_cpu_ids)
291 			error = work_on_cpu(cpu, local_pci_probe, &ddi);
292 		else
293 			error = local_pci_probe(&ddi);
294 		put_online_cpus();
295 	} else
296 		error = local_pci_probe(&ddi);
297 	return error;
298 }
299 
300 /**
301  * __pci_device_probe - check if a driver wants to claim a specific PCI device
302  * @drv: driver to call to check if it wants the PCI device
303  * @pci_dev: PCI device being probed
304  *
305  * returns 0 on success, else error.
306  * side-effect: pci_dev->driver is set to drv when drv claims pci_dev.
307  */
308 static int
309 __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
310 {
311 	const struct pci_device_id *id;
312 	int error = 0;
313 
314 	if (!pci_dev->driver && drv->probe) {
315 		error = -ENODEV;
316 
317 		id = pci_match_device(drv, pci_dev);
318 		if (id)
319 			error = pci_call_probe(drv, pci_dev, id);
320 		if (error >= 0)
321 			error = 0;
322 	}
323 	return error;
324 }
325 
326 static int pci_device_probe(struct device * dev)
327 {
328 	int error = 0;
329 	struct pci_driver *drv;
330 	struct pci_dev *pci_dev;
331 
332 	drv = to_pci_driver(dev->driver);
333 	pci_dev = to_pci_dev(dev);
334 	pci_dev_get(pci_dev);
335 	error = __pci_device_probe(drv, pci_dev);
336 	if (error)
337 		pci_dev_put(pci_dev);
338 
339 	return error;
340 }
341 
342 static int pci_device_remove(struct device * dev)
343 {
344 	struct pci_dev * pci_dev = to_pci_dev(dev);
345 	struct pci_driver * drv = pci_dev->driver;
346 
347 	if (drv) {
348 		if (drv->remove) {
349 			pm_runtime_get_sync(dev);
350 			drv->remove(pci_dev);
351 			pm_runtime_put_noidle(dev);
352 		}
353 		pci_dev->driver = NULL;
354 	}
355 
356 	/* Undo the runtime PM settings in local_pci_probe() */
357 	pm_runtime_put_sync(dev);
358 
359 	/*
360 	 * If the device is still on, set the power state as "unknown",
361 	 * since it might change by the next time we load the driver.
362 	 */
363 	if (pci_dev->current_state == PCI_D0)
364 		pci_dev->current_state = PCI_UNKNOWN;
365 
366 	/*
367 	 * We would love to complain here if pci_dev->is_enabled is set, that
368 	 * the driver should have called pci_disable_device(), but the
369 	 * unfortunate fact is there are too many odd BIOS and bridge setups
370 	 * that don't like drivers doing that all of the time.
371 	 * Oh well, we can dream of sane hardware when we sleep, no matter how
372 	 * horrible the crap we have to deal with is when we are awake...
373 	 */
374 
375 	pci_dev_put(pci_dev);
376 	return 0;
377 }
378 
379 static void pci_device_shutdown(struct device *dev)
380 {
381 	struct pci_dev *pci_dev = to_pci_dev(dev);
382 	struct pci_driver *drv = pci_dev->driver;
383 
384 	pm_runtime_resume(dev);
385 
386 	if (drv && drv->shutdown)
387 		drv->shutdown(pci_dev);
388 	pci_msi_shutdown(pci_dev);
389 	pci_msix_shutdown(pci_dev);
390 
391 	/*
392 	 * Turn off Bus Master bit on the device to tell it to not
393 	 * continue to do DMA
394 	 */
395 	pci_disable_device(pci_dev);
396 }
397 
398 #ifdef CONFIG_PM
399 
400 /* Auxiliary functions used for system resume and run-time resume. */
401 
402 /**
403  * pci_restore_standard_config - restore standard config registers of PCI device
404  * @pci_dev: PCI device to handle
405  */
406 static int pci_restore_standard_config(struct pci_dev *pci_dev)
407 {
408 	pci_update_current_state(pci_dev, PCI_UNKNOWN);
409 
410 	if (pci_dev->current_state != PCI_D0) {
411 		int error = pci_set_power_state(pci_dev, PCI_D0);
412 		if (error)
413 			return error;
414 	}
415 
416 	pci_restore_state(pci_dev);
417 	return 0;
418 }
419 
420 #endif
421 
422 #ifdef CONFIG_PM_SLEEP
423 
424 static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
425 {
426 	pci_power_up(pci_dev);
427 	pci_restore_state(pci_dev);
428 	pci_fixup_device(pci_fixup_resume_early, pci_dev);
429 }
430 
431 /*
432  * Default "suspend" method for devices that have no driver provided suspend,
433  * or not even a driver at all (second part).
434  */
435 static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
436 {
437 	/*
438 	 * mark its power state as "unknown", since we don't know if
439 	 * e.g. the BIOS will change its device state when we suspend.
440 	 */
441 	if (pci_dev->current_state == PCI_D0)
442 		pci_dev->current_state = PCI_UNKNOWN;
443 }
444 
445 /*
446  * Default "resume" method for devices that have no driver provided resume,
447  * or not even a driver at all (second part).
448  */
449 static int pci_pm_reenable_device(struct pci_dev *pci_dev)
450 {
451 	int retval;
452 
453 	/* if the device was enabled before suspend, reenable */
454 	retval = pci_reenable_device(pci_dev);
455 	/*
456 	 * if the device was busmaster before the suspend, make it busmaster
457 	 * again
458 	 */
459 	if (pci_dev->is_busmaster)
460 		pci_set_master(pci_dev);
461 
462 	return retval;
463 }
464 
465 static int pci_legacy_suspend(struct device *dev, pm_message_t state)
466 {
467 	struct pci_dev * pci_dev = to_pci_dev(dev);
468 	struct pci_driver * drv = pci_dev->driver;
469 
470 	if (drv && drv->suspend) {
471 		pci_power_t prev = pci_dev->current_state;
472 		int error;
473 
474 		error = drv->suspend(pci_dev, state);
475 		suspend_report_result(drv->suspend, error);
476 		if (error)
477 			return error;
478 
479 		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
480 		    && pci_dev->current_state != PCI_UNKNOWN) {
481 			WARN_ONCE(pci_dev->current_state != prev,
482 				"PCI PM: Device state not saved by %pF\n",
483 				drv->suspend);
484 		}
485 	}
486 
487 	pci_fixup_device(pci_fixup_suspend, pci_dev);
488 
489 	return 0;
490 }
491 
492 static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
493 {
494 	struct pci_dev * pci_dev = to_pci_dev(dev);
495 	struct pci_driver * drv = pci_dev->driver;
496 
497 	if (drv && drv->suspend_late) {
498 		pci_power_t prev = pci_dev->current_state;
499 		int error;
500 
501 		error = drv->suspend_late(pci_dev, state);
502 		suspend_report_result(drv->suspend_late, error);
503 		if (error)
504 			return error;
505 
506 		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
507 		    && pci_dev->current_state != PCI_UNKNOWN) {
508 			WARN_ONCE(pci_dev->current_state != prev,
509 				"PCI PM: Device state not saved by %pF\n",
510 				drv->suspend_late);
511 			return 0;
512 		}
513 	}
514 
515 	if (!pci_dev->state_saved)
516 		pci_save_state(pci_dev);
517 
518 	pci_pm_set_unknown_state(pci_dev);
519 
520 	return 0;
521 }
522 
523 static int pci_legacy_resume_early(struct device *dev)
524 {
525 	struct pci_dev * pci_dev = to_pci_dev(dev);
526 	struct pci_driver * drv = pci_dev->driver;
527 
528 	return drv && drv->resume_early ?
529 			drv->resume_early(pci_dev) : 0;
530 }
531 
532 static int pci_legacy_resume(struct device *dev)
533 {
534 	struct pci_dev * pci_dev = to_pci_dev(dev);
535 	struct pci_driver * drv = pci_dev->driver;
536 
537 	pci_fixup_device(pci_fixup_resume, pci_dev);
538 
539 	return drv && drv->resume ?
540 			drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
541 }
542 
543 /* Auxiliary functions used by the new power management framework */
544 
545 static void pci_pm_default_resume(struct pci_dev *pci_dev)
546 {
547 	pci_fixup_device(pci_fixup_resume, pci_dev);
548 
549 	if (!pci_is_bridge(pci_dev))
550 		pci_enable_wake(pci_dev, PCI_D0, false);
551 }
552 
553 static void pci_pm_default_suspend(struct pci_dev *pci_dev)
554 {
555 	/* Disable non-bridge devices without PM support */
556 	if (!pci_is_bridge(pci_dev))
557 		pci_disable_enabled_device(pci_dev);
558 }
559 
560 static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
561 {
562 	struct pci_driver *drv = pci_dev->driver;
563 	bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
564 		|| drv->resume_early);
565 
566 	/*
567 	 * Legacy PM support is used by default, so warn if the new framework is
568 	 * supported as well.  Drivers are supposed to support either the
569 	 * former, or the latter, but not both at the same time.
570 	 */
571 	WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
572 		drv->name, pci_dev->vendor, pci_dev->device);
573 
574 	return ret;
575 }
576 
577 /* New power management framework */
578 
579 static int pci_pm_prepare(struct device *dev)
580 {
581 	struct device_driver *drv = dev->driver;
582 	int error = 0;
583 
584 	/*
585 	 * PCI devices suspended at run time need to be resumed at this
586 	 * point, because in general it is necessary to reconfigure them for
587 	 * system suspend.  Namely, if the device is supposed to wake up the
588 	 * system from the sleep state, we may need to reconfigure it for this
589 	 * purpose.  In turn, if the device is not supposed to wake up the
590 	 * system from the sleep state, we'll have to prevent it from signaling
591 	 * wake-up.
592 	 */
593 	pm_runtime_resume(dev);
594 
595 	if (drv && drv->pm && drv->pm->prepare)
596 		error = drv->pm->prepare(dev);
597 
598 	return error;
599 }
600 
601 static void pci_pm_complete(struct device *dev)
602 {
603 	struct device_driver *drv = dev->driver;
604 
605 	if (drv && drv->pm && drv->pm->complete)
606 		drv->pm->complete(dev);
607 }
608 
609 #else /* !CONFIG_PM_SLEEP */
610 
611 #define pci_pm_prepare	NULL
612 #define pci_pm_complete	NULL
613 
614 #endif /* !CONFIG_PM_SLEEP */
615 
616 #ifdef CONFIG_SUSPEND
617 
618 static int pci_pm_suspend(struct device *dev)
619 {
620 	struct pci_dev *pci_dev = to_pci_dev(dev);
621 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
622 
623 	if (pci_has_legacy_pm_support(pci_dev))
624 		return pci_legacy_suspend(dev, PMSG_SUSPEND);
625 
626 	if (!pm) {
627 		pci_pm_default_suspend(pci_dev);
628 		goto Fixup;
629 	}
630 
631 	if (pm->suspend) {
632 		pci_power_t prev = pci_dev->current_state;
633 		int error;
634 
635 		error = pm->suspend(dev);
636 		suspend_report_result(pm->suspend, error);
637 		if (error)
638 			return error;
639 
640 		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
641 		    && pci_dev->current_state != PCI_UNKNOWN) {
642 			WARN_ONCE(pci_dev->current_state != prev,
643 				"PCI PM: State of device not saved by %pF\n",
644 				pm->suspend);
645 		}
646 	}
647 
648  Fixup:
649 	pci_fixup_device(pci_fixup_suspend, pci_dev);
650 
651 	return 0;
652 }
653 
654 static int pci_pm_suspend_noirq(struct device *dev)
655 {
656 	struct pci_dev *pci_dev = to_pci_dev(dev);
657 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
658 
659 	if (pci_has_legacy_pm_support(pci_dev))
660 		return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
661 
662 	if (!pm) {
663 		pci_save_state(pci_dev);
664 		return 0;
665 	}
666 
667 	if (pm->suspend_noirq) {
668 		pci_power_t prev = pci_dev->current_state;
669 		int error;
670 
671 		error = pm->suspend_noirq(dev);
672 		suspend_report_result(pm->suspend_noirq, error);
673 		if (error)
674 			return error;
675 
676 		if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
677 		    && pci_dev->current_state != PCI_UNKNOWN) {
678 			WARN_ONCE(pci_dev->current_state != prev,
679 				"PCI PM: State of device not saved by %pF\n",
680 				pm->suspend_noirq);
681 			return 0;
682 		}
683 	}
684 
685 	if (!pci_dev->state_saved) {
686 		pci_save_state(pci_dev);
687 		if (!pci_is_bridge(pci_dev))
688 			pci_prepare_to_sleep(pci_dev);
689 	}
690 
691 	pci_pm_set_unknown_state(pci_dev);
692 
693 	/*
694 	 * Some BIOSes from ASUS have a bug: If a USB EHCI host controller's
695 	 * PCI COMMAND register isn't 0, the BIOS assumes that the controller
696 	 * hasn't been quiesced and tries to turn it off.  If the controller
697 	 * is already in D3, this can hang or cause memory corruption.
698 	 *
699 	 * Since the value of the COMMAND register doesn't matter once the
700 	 * device has been suspended, we can safely set it to 0 here.
701 	 */
702 	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
703 		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
704 
705 	return 0;
706 }
707 
708 static int pci_pm_resume_noirq(struct device *dev)
709 {
710 	struct pci_dev *pci_dev = to_pci_dev(dev);
711 	struct device_driver *drv = dev->driver;
712 	int error = 0;
713 
714 	pci_pm_default_resume_early(pci_dev);
715 
716 	if (pci_has_legacy_pm_support(pci_dev))
717 		return pci_legacy_resume_early(dev);
718 
719 	if (drv && drv->pm && drv->pm->resume_noirq)
720 		error = drv->pm->resume_noirq(dev);
721 
722 	return error;
723 }
724 
725 static int pci_pm_resume(struct device *dev)
726 {
727 	struct pci_dev *pci_dev = to_pci_dev(dev);
728 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
729 	int error = 0;
730 
731 	/*
732 	 * This is necessary for the suspend error path in which resume is
733 	 * called without restoring the standard config registers of the device.
734 	 */
735 	if (pci_dev->state_saved)
736 		pci_restore_standard_config(pci_dev);
737 
738 	if (pci_has_legacy_pm_support(pci_dev))
739 		return pci_legacy_resume(dev);
740 
741 	pci_pm_default_resume(pci_dev);
742 
743 	if (pm) {
744 		if (pm->resume)
745 			error = pm->resume(dev);
746 	} else {
747 		pci_pm_reenable_device(pci_dev);
748 	}
749 
750 	return error;
751 }
752 
753 #else /* !CONFIG_SUSPEND */
754 
755 #define pci_pm_suspend		NULL
756 #define pci_pm_suspend_noirq	NULL
757 #define pci_pm_resume		NULL
758 #define pci_pm_resume_noirq	NULL
759 
760 #endif /* !CONFIG_SUSPEND */
761 
762 #ifdef CONFIG_HIBERNATE_CALLBACKS
763 
764 static int pci_pm_freeze(struct device *dev)
765 {
766 	struct pci_dev *pci_dev = to_pci_dev(dev);
767 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
768 
769 	if (pci_has_legacy_pm_support(pci_dev))
770 		return pci_legacy_suspend(dev, PMSG_FREEZE);
771 
772 	if (!pm) {
773 		pci_pm_default_suspend(pci_dev);
774 		return 0;
775 	}
776 
777 	if (pm->freeze) {
778 		int error;
779 
780 		error = pm->freeze(dev);
781 		suspend_report_result(pm->freeze, error);
782 		if (error)
783 			return error;
784 	}
785 
786 	return 0;
787 }
788 
789 static int pci_pm_freeze_noirq(struct device *dev)
790 {
791 	struct pci_dev *pci_dev = to_pci_dev(dev);
792 	struct device_driver *drv = dev->driver;
793 
794 	if (pci_has_legacy_pm_support(pci_dev))
795 		return pci_legacy_suspend_late(dev, PMSG_FREEZE);
796 
797 	if (drv && drv->pm && drv->pm->freeze_noirq) {
798 		int error;
799 
800 		error = drv->pm->freeze_noirq(dev);
801 		suspend_report_result(drv->pm->freeze_noirq, error);
802 		if (error)
803 			return error;
804 	}
805 
806 	if (!pci_dev->state_saved)
807 		pci_save_state(pci_dev);
808 
809 	pci_pm_set_unknown_state(pci_dev);
810 
811 	return 0;
812 }
813 
814 static int pci_pm_thaw_noirq(struct device *dev)
815 {
816 	struct pci_dev *pci_dev = to_pci_dev(dev);
817 	struct device_driver *drv = dev->driver;
818 	int error = 0;
819 
820 	if (pci_has_legacy_pm_support(pci_dev))
821 		return pci_legacy_resume_early(dev);
822 
823 	pci_update_current_state(pci_dev, PCI_D0);
824 
825 	if (drv && drv->pm && drv->pm->thaw_noirq)
826 		error = drv->pm->thaw_noirq(dev);
827 
828 	return error;
829 }
830 
831 static int pci_pm_thaw(struct device *dev)
832 {
833 	struct pci_dev *pci_dev = to_pci_dev(dev);
834 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
835 	int error = 0;
836 
837 	if (pci_has_legacy_pm_support(pci_dev))
838 		return pci_legacy_resume(dev);
839 
840 	if (pm) {
841 		if (pm->thaw)
842 			error = pm->thaw(dev);
843 	} else {
844 		pci_pm_reenable_device(pci_dev);
845 	}
846 
847 	pci_dev->state_saved = false;
848 
849 	return error;
850 }
851 
852 static int pci_pm_poweroff(struct device *dev)
853 {
854 	struct pci_dev *pci_dev = to_pci_dev(dev);
855 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
856 
857 	if (pci_has_legacy_pm_support(pci_dev))
858 		return pci_legacy_suspend(dev, PMSG_HIBERNATE);
859 
860 	if (!pm) {
861 		pci_pm_default_suspend(pci_dev);
862 		goto Fixup;
863 	}
864 
865 	if (pm->poweroff) {
866 		int error;
867 
868 		error = pm->poweroff(dev);
869 		suspend_report_result(pm->poweroff, error);
870 		if (error)
871 			return error;
872 	}
873 
874  Fixup:
875 	pci_fixup_device(pci_fixup_suspend, pci_dev);
876 
877 	return 0;
878 }
879 
880 static int pci_pm_poweroff_noirq(struct device *dev)
881 {
882 	struct pci_dev *pci_dev = to_pci_dev(dev);
883 	struct device_driver *drv = dev->driver;
884 
885 	if (pci_has_legacy_pm_support(to_pci_dev(dev)))
886 		return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
887 
888 	if (!drv || !drv->pm)
889 		return 0;
890 
891 	if (drv->pm->poweroff_noirq) {
892 		int error;
893 
894 		error = drv->pm->poweroff_noirq(dev);
895 		suspend_report_result(drv->pm->poweroff_noirq, error);
896 		if (error)
897 			return error;
898 	}
899 
900 	if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
901 		pci_prepare_to_sleep(pci_dev);
902 
903 	/*
904 	 * The reason for doing this here is the same as for the analogous code
905 	 * in pci_pm_suspend_noirq().
906 	 */
907 	if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
908 		pci_write_config_word(pci_dev, PCI_COMMAND, 0);
909 
910 	return 0;
911 }
912 
913 static int pci_pm_restore_noirq(struct device *dev)
914 {
915 	struct pci_dev *pci_dev = to_pci_dev(dev);
916 	struct device_driver *drv = dev->driver;
917 	int error = 0;
918 
919 	pci_pm_default_resume_early(pci_dev);
920 
921 	if (pci_has_legacy_pm_support(pci_dev))
922 		return pci_legacy_resume_early(dev);
923 
924 	if (drv && drv->pm && drv->pm->restore_noirq)
925 		error = drv->pm->restore_noirq(dev);
926 
927 	return error;
928 }
929 
930 static int pci_pm_restore(struct device *dev)
931 {
932 	struct pci_dev *pci_dev = to_pci_dev(dev);
933 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
934 	int error = 0;
935 
936 	/*
937 	 * This is necessary for the hibernation error path in which restore is
938 	 * called without restoring the standard config registers of the device.
939 	 */
940 	if (pci_dev->state_saved)
941 		pci_restore_standard_config(pci_dev);
942 
943 	if (pci_has_legacy_pm_support(pci_dev))
944 		return pci_legacy_resume(dev);
945 
946 	pci_pm_default_resume(pci_dev);
947 
948 	if (pm) {
949 		if (pm->restore)
950 			error = pm->restore(dev);
951 	} else {
952 		pci_pm_reenable_device(pci_dev);
953 	}
954 
955 	return error;
956 }
957 
958 #else /* !CONFIG_HIBERNATE_CALLBACKS */
959 
960 #define pci_pm_freeze		NULL
961 #define pci_pm_freeze_noirq	NULL
962 #define pci_pm_thaw		NULL
963 #define pci_pm_thaw_noirq	NULL
964 #define pci_pm_poweroff		NULL
965 #define pci_pm_poweroff_noirq	NULL
966 #define pci_pm_restore		NULL
967 #define pci_pm_restore_noirq	NULL
968 
969 #endif /* !CONFIG_HIBERNATE_CALLBACKS */
970 
971 #ifdef CONFIG_PM_RUNTIME
972 
973 static int pci_pm_runtime_suspend(struct device *dev)
974 {
975 	struct pci_dev *pci_dev = to_pci_dev(dev);
976 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
977 	pci_power_t prev = pci_dev->current_state;
978 	int error;
979 
980 	/*
981 	 * If pci_dev->driver is not set (unbound), the device should
982 	 * always remain in D0 regardless of the runtime PM status
983 	 */
984 	if (!pci_dev->driver)
985 		return 0;
986 
987 	if (!pm || !pm->runtime_suspend)
988 		return -ENOSYS;
989 
990 	pci_dev->no_d3cold = false;
991 	error = pm->runtime_suspend(dev);
992 	suspend_report_result(pm->runtime_suspend, error);
993 	if (error)
994 		return error;
995 	if (!pci_dev->d3cold_allowed)
996 		pci_dev->no_d3cold = true;
997 
998 	pci_fixup_device(pci_fixup_suspend, pci_dev);
999 
1000 	if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1001 	    && pci_dev->current_state != PCI_UNKNOWN) {
1002 		WARN_ONCE(pci_dev->current_state != prev,
1003 			"PCI PM: State of device not saved by %pF\n",
1004 			pm->runtime_suspend);
1005 		return 0;
1006 	}
1007 
1008 	if (!pci_dev->state_saved) {
1009 		pci_save_state(pci_dev);
1010 		pci_finish_runtime_suspend(pci_dev);
1011 	}
1012 
1013 	return 0;
1014 }
1015 
1016 static int pci_pm_runtime_resume(struct device *dev)
1017 {
1018 	int rc;
1019 	struct pci_dev *pci_dev = to_pci_dev(dev);
1020 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1021 
1022 	/*
1023 	 * If pci_dev->driver is not set (unbound), the device should
1024 	 * always remain in D0 regardless of the runtime PM status
1025 	 */
1026 	if (!pci_dev->driver)
1027 		return 0;
1028 
1029 	if (!pm || !pm->runtime_resume)
1030 		return -ENOSYS;
1031 
1032 	pci_restore_standard_config(pci_dev);
1033 	pci_fixup_device(pci_fixup_resume_early, pci_dev);
1034 	__pci_enable_wake(pci_dev, PCI_D0, true, false);
1035 	pci_fixup_device(pci_fixup_resume, pci_dev);
1036 
1037 	rc = pm->runtime_resume(dev);
1038 
1039 	pci_dev->runtime_d3cold = false;
1040 
1041 	return rc;
1042 }
1043 
1044 static int pci_pm_runtime_idle(struct device *dev)
1045 {
1046 	struct pci_dev *pci_dev = to_pci_dev(dev);
1047 	const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1048 
1049 	/*
1050 	 * If pci_dev->driver is not set (unbound), the device should
1051 	 * always remain in D0 regardless of the runtime PM status
1052 	 */
1053 	if (!pci_dev->driver)
1054 		goto out;
1055 
1056 	if (!pm)
1057 		return -ENOSYS;
1058 
1059 	if (pm->runtime_idle) {
1060 		int ret = pm->runtime_idle(dev);
1061 		if (ret)
1062 			return ret;
1063 	}
1064 
1065 out:
1066 	pm_runtime_suspend(dev);
1067 	return 0;
1068 }
1069 
1070 #else /* !CONFIG_PM_RUNTIME */
1071 
1072 #define pci_pm_runtime_suspend	NULL
1073 #define pci_pm_runtime_resume	NULL
1074 #define pci_pm_runtime_idle	NULL
1075 
1076 #endif /* !CONFIG_PM_RUNTIME */
1077 
1078 #ifdef CONFIG_PM
1079 
1080 const struct dev_pm_ops pci_dev_pm_ops = {
1081 	.prepare = pci_pm_prepare,
1082 	.complete = pci_pm_complete,
1083 	.suspend = pci_pm_suspend,
1084 	.resume = pci_pm_resume,
1085 	.freeze = pci_pm_freeze,
1086 	.thaw = pci_pm_thaw,
1087 	.poweroff = pci_pm_poweroff,
1088 	.restore = pci_pm_restore,
1089 	.suspend_noirq = pci_pm_suspend_noirq,
1090 	.resume_noirq = pci_pm_resume_noirq,
1091 	.freeze_noirq = pci_pm_freeze_noirq,
1092 	.thaw_noirq = pci_pm_thaw_noirq,
1093 	.poweroff_noirq = pci_pm_poweroff_noirq,
1094 	.restore_noirq = pci_pm_restore_noirq,
1095 	.runtime_suspend = pci_pm_runtime_suspend,
1096 	.runtime_resume = pci_pm_runtime_resume,
1097 	.runtime_idle = pci_pm_runtime_idle,
1098 };
1099 
1100 #define PCI_PM_OPS_PTR	(&pci_dev_pm_ops)
1101 
1102 #else /* !COMFIG_PM_OPS */
1103 
1104 #define PCI_PM_OPS_PTR	NULL
1105 
1106 #endif /* !COMFIG_PM_OPS */
1107 
1108 /**
1109  * __pci_register_driver - register a new pci driver
1110  * @drv: the driver structure to register
1111  * @owner: owner module of drv
1112  * @mod_name: module name string
1113  *
1114  * Adds the driver structure to the list of registered drivers.
1115  * Returns a negative value on error, otherwise 0.
1116  * If no error occurred, the driver remains registered even if
1117  * no device was claimed during registration.
1118  */
1119 int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1120 			  const char *mod_name)
1121 {
1122 	/* initialize common driver fields */
1123 	drv->driver.name = drv->name;
1124 	drv->driver.bus = &pci_bus_type;
1125 	drv->driver.owner = owner;
1126 	drv->driver.mod_name = mod_name;
1127 
1128 	spin_lock_init(&drv->dynids.lock);
1129 	INIT_LIST_HEAD(&drv->dynids.list);
1130 
1131 	/* register with core */
1132 	return driver_register(&drv->driver);
1133 }
1134 
1135 /**
1136  * pci_unregister_driver - unregister a pci driver
1137  * @drv: the driver structure to unregister
1138  *
1139  * Deletes the driver structure from the list of registered PCI drivers,
1140  * gives it a chance to clean up by calling its remove() function for
1141  * each device it was responsible for, and marks those devices as
1142  * driverless.
1143  */
1144 
1145 void
1146 pci_unregister_driver(struct pci_driver *drv)
1147 {
1148 	driver_unregister(&drv->driver);
1149 	pci_free_dynids(drv);
1150 }
1151 
1152 static struct pci_driver pci_compat_driver = {
1153 	.name = "compat"
1154 };
1155 
1156 /**
1157  * pci_dev_driver - get the pci_driver of a device
1158  * @dev: the device to query
1159  *
1160  * Returns the appropriate pci_driver structure or %NULL if there is no
1161  * registered driver for the device.
1162  */
1163 struct pci_driver *
1164 pci_dev_driver(const struct pci_dev *dev)
1165 {
1166 	if (dev->driver)
1167 		return dev->driver;
1168 	else {
1169 		int i;
1170 		for(i=0; i<=PCI_ROM_RESOURCE; i++)
1171 			if (dev->resource[i].flags & IORESOURCE_BUSY)
1172 				return &pci_compat_driver;
1173 	}
1174 	return NULL;
1175 }
1176 
1177 /**
1178  * pci_bus_match - Tell if a PCI device structure has a matching PCI device id structure
1179  * @dev: the PCI device structure to match against
1180  * @drv: the device driver to search for matching PCI device id structures
1181  *
1182  * Used by a driver to check whether a PCI device present in the
1183  * system is in its list of supported devices. Returns the matching
1184  * pci_device_id structure or %NULL if there is no match.
1185  */
1186 static int pci_bus_match(struct device *dev, struct device_driver *drv)
1187 {
1188 	struct pci_dev *pci_dev = to_pci_dev(dev);
1189 	struct pci_driver *pci_drv = to_pci_driver(drv);
1190 	const struct pci_device_id *found_id;
1191 
1192 	found_id = pci_match_device(pci_drv, pci_dev);
1193 	if (found_id)
1194 		return 1;
1195 
1196 	return 0;
1197 }
1198 
1199 /**
1200  * pci_dev_get - increments the reference count of the pci device structure
1201  * @dev: the device being referenced
1202  *
1203  * Each live reference to a device should be refcounted.
1204  *
1205  * Drivers for PCI devices should normally record such references in
1206  * their probe() methods, when they bind to a device, and release
1207  * them by calling pci_dev_put(), in their disconnect() methods.
1208  *
1209  * A pointer to the device with the incremented reference counter is returned.
1210  */
1211 struct pci_dev *pci_dev_get(struct pci_dev *dev)
1212 {
1213 	if (dev)
1214 		get_device(&dev->dev);
1215 	return dev;
1216 }
1217 
1218 /**
1219  * pci_dev_put - release a use of the pci device structure
1220  * @dev: device that's been disconnected
1221  *
1222  * Must be called when a user of a device is finished with it.  When the last
1223  * user of the device calls this function, the memory of the device is freed.
1224  */
1225 void pci_dev_put(struct pci_dev *dev)
1226 {
1227 	if (dev)
1228 		put_device(&dev->dev);
1229 }
1230 
1231 static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1232 {
1233 	struct pci_dev *pdev;
1234 
1235 	if (!dev)
1236 		return -ENODEV;
1237 
1238 	pdev = to_pci_dev(dev);
1239 	if (!pdev)
1240 		return -ENODEV;
1241 
1242 	if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1243 		return -ENOMEM;
1244 
1245 	if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1246 		return -ENOMEM;
1247 
1248 	if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1249 			   pdev->subsystem_device))
1250 		return -ENOMEM;
1251 
1252 	if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1253 		return -ENOMEM;
1254 
1255 	if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02x",
1256 			   pdev->vendor, pdev->device,
1257 			   pdev->subsystem_vendor, pdev->subsystem_device,
1258 			   (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1259 			   (u8)(pdev->class)))
1260 		return -ENOMEM;
1261 	return 0;
1262 }
1263 
1264 struct bus_type pci_bus_type = {
1265 	.name		= "pci",
1266 	.match		= pci_bus_match,
1267 	.uevent		= pci_uevent,
1268 	.probe		= pci_device_probe,
1269 	.remove		= pci_device_remove,
1270 	.shutdown	= pci_device_shutdown,
1271 	.dev_attrs	= pci_dev_attrs,
1272 	.bus_attrs	= pci_bus_attrs,
1273 	.drv_attrs	= pci_drv_attrs,
1274 	.pm		= PCI_PM_OPS_PTR,
1275 };
1276 
1277 static int __init pci_driver_init(void)
1278 {
1279 	return bus_register(&pci_bus_type);
1280 }
1281 
1282 postcore_initcall(pci_driver_init);
1283 
1284 EXPORT_SYMBOL_GPL(pci_add_dynid);
1285 EXPORT_SYMBOL(pci_match_id);
1286 EXPORT_SYMBOL(__pci_register_driver);
1287 EXPORT_SYMBOL(pci_unregister_driver);
1288 EXPORT_SYMBOL(pci_dev_driver);
1289 EXPORT_SYMBOL(pci_bus_type);
1290 EXPORT_SYMBOL(pci_dev_get);
1291 EXPORT_SYMBOL(pci_dev_put);
1292