1 // SPDX-License-Identifier: GPL-2.0
13 #include <linux/pci-epc.h>
14 #include <linux/pci-epf.h>
15 #include <linux/pci-ep-cfs.h>
29  * pci_epc_put() - release the PCI endpoint controller
39 	module_put(epc->ops->owner);  in pci_epc_put()
40 	put_device(&epc->dev);  in pci_epc_put()
45  * pci_epc_get() - get the PCI endpoint controller
53 	int ret = -EINVAL;  in pci_epc_get()
62 	if (try_module_get(epc->ops->owner))  in pci_epc_get()
72  * pci_epc_get_first_free_bar() - helper to get first unreserved BAR
86  * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar
101 	/* If 'bar - 1' is a 64-bit BAR, move to the next BAR */  in pci_epc_get_next_free_bar()
102 	if (bar > 0 && epc_features->bar[bar - 1].only_64bit)  in pci_epc_get_next_free_bar()
107 		if (epc_features->bar[i].type != BAR_RESERVED)  in pci_epc_get_next_free_bar()
118 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_function_is_valid()
121 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_function_is_valid()
128  * pci_epc_get_features() - get the features supported by EPC
147 	if (!epc->ops->get_features)  in pci_epc_get_features()
150 	mutex_lock(&epc->lock);  in pci_epc_get_features()
151 	epc_features = epc->ops->get_features(epc, func_no, vfunc_no);  in pci_epc_get_features()
152 	mutex_unlock(&epc->lock);  in pci_epc_get_features()
159  * pci_epc_stop() - stop the PCI link
166 	if (IS_ERR(epc) || !epc->ops->stop)  in pci_epc_stop()
169 	mutex_lock(&epc->lock);  in pci_epc_stop()
170 	epc->ops->stop(epc);  in pci_epc_stop()
171 	mutex_unlock(&epc->lock);  in pci_epc_stop()
176  * pci_epc_start() - start the PCI link
186 		return -EINVAL;  in pci_epc_start()
188 	if (!epc->ops->start)  in pci_epc_start()
191 	mutex_lock(&epc->lock);  in pci_epc_start()
192 	ret = epc->ops->start(epc);  in pci_epc_start()
193 	mutex_unlock(&epc->lock);  in pci_epc_start()
200  * pci_epc_raise_irq() - interrupt the host system
204  * @type: specify the type of interrupt; INTX, MSI or MSI-X
205  * @interrupt_num: the MSI or MSI-X interrupt number with range (1-N)
207  * Invoke to raise an INTX, MSI or MSI-X interrupt
215 		return -EINVAL;  in pci_epc_raise_irq()
217 	if (!epc->ops->raise_irq)  in pci_epc_raise_irq()
220 	mutex_lock(&epc->lock);  in pci_epc_raise_irq()
221 	ret = epc->ops->raise_irq(epc, func_no, vfunc_no, type, interrupt_num);  in pci_epc_raise_irq()
222 	mutex_unlock(&epc->lock);  in pci_epc_raise_irq()
229  * pci_epc_map_msi_irq() - Map physical address to MSI address and return
230  *                         MSI data
231  * @epc: the EPC device which has the MSI capability
235  * @interrupt_num: the MSI interrupt number with range (1-N)
237  * @msi_data: the data that should be written in order to raise MSI interrupt
239  * @msi_addr_offset: Offset of MSI address from the aligned outbound address
240  *                   to which the MSI address is mapped
242  * Invoke to map physical address to MSI address and return MSI data. The
256 		return -EINVAL;  in pci_epc_map_msi_irq()
258 	if (!epc->ops->map_msi_irq)  in pci_epc_map_msi_irq()
259 		return -EINVAL;  in pci_epc_map_msi_irq()
261 	mutex_lock(&epc->lock);  in pci_epc_map_msi_irq()
262 	ret = epc->ops->map_msi_irq(epc, func_no, vfunc_no, phys_addr,  in pci_epc_map_msi_irq()
265 	mutex_unlock(&epc->lock);  in pci_epc_map_msi_irq()
272  * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated
273  * @epc: the EPC device to which MSI interrupts was requested
277  * Invoke to get the number of MSI interrupts allocated by the RC
286 	if (!epc->ops->get_msi)  in pci_epc_get_msi()
289 	mutex_lock(&epc->lock);  in pci_epc_get_msi()
290 	interrupt = epc->ops->get_msi(epc, func_no, vfunc_no);  in pci_epc_get_msi()
291 	mutex_unlock(&epc->lock);  in pci_epc_get_msi()
301  * pci_epc_set_msi() - set the number of MSI interrupt numbers required
302  * @epc: the EPC device on which MSI has to be configured
305  * @nr_irqs: number of MSI interrupts required by the EPF
307  * Invoke to set the required number of MSI interrupts.
314 		return -EINVAL;  in pci_epc_set_msi()
317 		return -EINVAL;  in pci_epc_set_msi()
319 	if (!epc->ops->set_msi)  in pci_epc_set_msi()
322 	mutex_lock(&epc->lock);  in pci_epc_set_msi()
323 	ret = epc->ops->set_msi(epc, func_no, vfunc_no, nr_irqs);  in pci_epc_set_msi()
324 	mutex_unlock(&epc->lock);  in pci_epc_set_msi()
331  * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
332  * @epc: the EPC device to which MSI-X interrupts was requested
336  * Invoke to get the number of MSI-X interrupts allocated by the RC
345 	if (!epc->ops->get_msix)  in pci_epc_get_msix()
348 	mutex_lock(&epc->lock);  in pci_epc_get_msix()
349 	interrupt = epc->ops->get_msix(epc, func_no, vfunc_no);  in pci_epc_get_msix()
350 	mutex_unlock(&epc->lock);  in pci_epc_get_msix()
360  * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
361  * @epc: the EPC device on which MSI-X has to be configured
364  * @nr_irqs: number of MSI-X interrupts required by the EPF
365  * @bir: BAR where the MSI-X table resides
366  * @offset: Offset pointing to the start of MSI-X table
368  * Invoke to set the required number of MSI-X interrupts.
376 		return -EINVAL;  in pci_epc_set_msix()
379 		return -EINVAL;  in pci_epc_set_msix()
381 	if (!epc->ops->set_msix)  in pci_epc_set_msix()
384 	mutex_lock(&epc->lock);  in pci_epc_set_msix()
385 	ret = epc->ops->set_msix(epc, func_no, vfunc_no, nr_irqs, bir, offset);  in pci_epc_set_msix()
386 	mutex_unlock(&epc->lock);  in pci_epc_set_msix()
393  * pci_epc_unmap_addr() - unmap CPU address from PCI address
407 	if (!epc->ops->unmap_addr)  in pci_epc_unmap_addr()
410 	mutex_lock(&epc->lock);  in pci_epc_unmap_addr()
411 	epc->ops->unmap_addr(epc, func_no, vfunc_no, phys_addr);  in pci_epc_unmap_addr()
412 	mutex_unlock(&epc->lock);  in pci_epc_unmap_addr()
417  * pci_epc_map_addr() - map CPU address to PCI address
425  * Invoke to map CPU address with PCI address.
433 		return -EINVAL;  in pci_epc_map_addr()
435 	if (!epc->ops->map_addr)  in pci_epc_map_addr()
438 	mutex_lock(&epc->lock);  in pci_epc_map_addr()
439 	ret = epc->ops->map_addr(epc, func_no, vfunc_no, phys_addr, pci_addr,  in pci_epc_map_addr()
441 	mutex_unlock(&epc->lock);  in pci_epc_map_addr()
448  * pci_epc_mem_map() - allocate and map a PCI address to a CPU address
453  * @pci_size: the number of bytes to map starting from @pci_addr
454  * @map: where to return the mapping information
456  * Allocate a controller memory address region and map it to a RC PCI address
463  * indicated by @map->pci_size. This size may be less than the requested
465  * @map->virt_addr (@map->phys_addr indicates the physical address).
467  * respectively indicated by @map->map_size and @map->virt_base (and
468  * @map->phys_base for the physical address of @map->virt_base).
473 		    u64 pci_addr, size_t pci_size, struct pci_epc_map *map)  in pci_epc_mem_map()  argument
480 		return -EINVAL;  in pci_epc_mem_map()
482 	if (!pci_size || !map)  in pci_epc_mem_map()
483 		return -EINVAL;  in pci_epc_mem_map()
486 	 * Align the PCI address to map. If the controller defines the  in pci_epc_mem_map()
487 	 * .align_addr() operation, use it to determine the PCI address to map  in pci_epc_mem_map()
491 	memset(map, 0, sizeof(*map));  in pci_epc_mem_map()
492 	map->pci_addr = pci_addr;  in pci_epc_mem_map()
493 	if (epc->ops->align_addr)  in pci_epc_mem_map()
494 		map->map_pci_addr =  in pci_epc_mem_map()
495 			epc->ops->align_addr(epc, pci_addr,  in pci_epc_mem_map()
498 		map->map_pci_addr = pci_addr;  in pci_epc_mem_map()
499 	map->map_size = map_size;  in pci_epc_mem_map()
500 	if (map->map_pci_addr + map->map_size < pci_addr + pci_size)  in pci_epc_mem_map()
501 		map->pci_size = map->map_pci_addr + map->map_size - pci_addr;  in pci_epc_mem_map()
503 		map->pci_size = pci_size;  in pci_epc_mem_map()
505 	map->virt_base = pci_epc_mem_alloc_addr(epc, &map->phys_base,  in pci_epc_mem_map()
506 						map->map_size);  in pci_epc_mem_map()
507 	if (!map->virt_base)  in pci_epc_mem_map()
508 		return -ENOMEM;  in pci_epc_mem_map()
510 	map->phys_addr = map->phys_base + map_offset;  in pci_epc_mem_map()
511 	map->virt_addr = map->virt_base + map_offset;  in pci_epc_mem_map()
513 	ret = pci_epc_map_addr(epc, func_no, vfunc_no, map->phys_base,  in pci_epc_mem_map()
514 			       map->map_pci_addr, map->map_size);  in pci_epc_mem_map()
516 		pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base,  in pci_epc_mem_map()
517 				      map->map_size);  in pci_epc_mem_map()
526  * pci_epc_mem_unmap() - unmap and free a CPU address region
530  * @map: the mapping information
536 		       struct pci_epc_map *map)  in pci_epc_mem_unmap()  argument
541 	if (!map || !map->virt_base)  in pci_epc_mem_unmap()
544 	pci_epc_unmap_addr(epc, func_no, vfunc_no, map->phys_base);  in pci_epc_mem_unmap()
545 	pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base,  in pci_epc_mem_unmap()
546 			      map->map_size);  in pci_epc_mem_unmap()
551  * pci_epc_clear_bar() - reset the BAR
565 	if (epf_bar->barno == BAR_5 &&  in pci_epc_clear_bar()
566 	    epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)  in pci_epc_clear_bar()
569 	if (!epc->ops->clear_bar)  in pci_epc_clear_bar()
572 	mutex_lock(&epc->lock);  in pci_epc_clear_bar()
573 	epc->ops->clear_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_clear_bar()
574 	mutex_unlock(&epc->lock);  in pci_epc_clear_bar()
579  * pci_epc_set_bar() - configure BAR in order for host to assign PCI addr space
591 	enum pci_barno bar = epf_bar->barno;  in pci_epc_set_bar()
592 	int flags = epf_bar->flags;  in pci_epc_set_bar()
597 		return -EINVAL;  in pci_epc_set_bar()
599 	if (epc_features->bar[bar].type == BAR_RESIZABLE &&  in pci_epc_set_bar()
600 	    (epf_bar->size < SZ_1M || (u64)epf_bar->size > (SZ_128G * 1024)))  in pci_epc_set_bar()
601 		return -EINVAL;  in pci_epc_set_bar()
603 	if (epc_features->bar[bar].type == BAR_FIXED &&  in pci_epc_set_bar()
604 	    (epc_features->bar[bar].fixed_size != epf_bar->size))  in pci_epc_set_bar()
605 		return -EINVAL;  in pci_epc_set_bar()
607 	if (!is_power_of_2(epf_bar->size))  in pci_epc_set_bar()
608 		return -EINVAL;  in pci_epc_set_bar()
610 	if ((epf_bar->barno == BAR_5 && flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ||  in pci_epc_set_bar()
613 	    (upper_32_bits(epf_bar->size) &&  in pci_epc_set_bar()
615 		return -EINVAL;  in pci_epc_set_bar()
617 	if (!epc->ops->set_bar)  in pci_epc_set_bar()
620 	mutex_lock(&epc->lock);  in pci_epc_set_bar()
621 	ret = epc->ops->set_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_set_bar()
622 	mutex_unlock(&epc->lock);  in pci_epc_set_bar()
629  * pci_epc_bar_size_to_rebar_cap() - convert a size to the representation used
644 		return -EINVAL;  in pci_epc_bar_size_to_rebar_cap()
646 	*cap = ilog2(size) - ilog2(SZ_1M);  in pci_epc_bar_size_to_rebar_cap()
656  * pci_epc_write_header() - write standard configuration header
673 		return -EINVAL;  in pci_epc_write_header()
677 		return -EINVAL;  in pci_epc_write_header()
679 	if (!epc->ops->write_header)  in pci_epc_write_header()
682 	mutex_lock(&epc->lock);  in pci_epc_write_header()
683 	ret = epc->ops->write_header(epc, func_no, vfunc_no, header);  in pci_epc_write_header()
684 	mutex_unlock(&epc->lock);  in pci_epc_write_header()
691  * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller
708 	if (IS_ERR_OR_NULL(epc) || epf->is_vf)  in pci_epc_add_epf()
709 		return -EINVAL;  in pci_epc_add_epf()
711 	if (type == PRIMARY_INTERFACE && epf->epc)  in pci_epc_add_epf()
712 		return -EBUSY;  in pci_epc_add_epf()
714 	if (type == SECONDARY_INTERFACE && epf->sec_epc)  in pci_epc_add_epf()
715 		return -EBUSY;  in pci_epc_add_epf()
717 	mutex_lock(&epc->list_lock);  in pci_epc_add_epf()
718 	func_no = find_first_zero_bit(&epc->function_num_map,  in pci_epc_add_epf()
721 		ret = -EINVAL;  in pci_epc_add_epf()
725 	if (func_no > epc->max_functions - 1) {  in pci_epc_add_epf()
726 		dev_err(&epc->dev, "Exceeding max supported Function Number\n");  in pci_epc_add_epf()
727 		ret = -EINVAL;  in pci_epc_add_epf()
731 	set_bit(func_no, &epc->function_num_map);  in pci_epc_add_epf()
733 		epf->func_no = func_no;  in pci_epc_add_epf()
734 		epf->epc = epc;  in pci_epc_add_epf()
735 		list = &epf->list;  in pci_epc_add_epf()
737 		epf->sec_epc_func_no = func_no;  in pci_epc_add_epf()
738 		epf->sec_epc = epc;  in pci_epc_add_epf()
739 		list = &epf->sec_epc_list;  in pci_epc_add_epf()
742 	list_add_tail(list, &epc->pci_epf);  in pci_epc_add_epf()
744 	mutex_unlock(&epc->list_lock);  in pci_epc_add_epf()
751  * pci_epc_remove_epf() - remove PCI endpoint function from endpoint controller
768 	mutex_lock(&epc->list_lock);  in pci_epc_remove_epf()
770 		func_no = epf->func_no;  in pci_epc_remove_epf()
771 		list = &epf->list;  in pci_epc_remove_epf()
772 		epf->epc = NULL;  in pci_epc_remove_epf()
774 		func_no = epf->sec_epc_func_no;  in pci_epc_remove_epf()
775 		list = &epf->sec_epc_list;  in pci_epc_remove_epf()
776 		epf->sec_epc = NULL;  in pci_epc_remove_epf()
778 	clear_bit(func_no, &epc->function_num_map);  in pci_epc_remove_epf()
780 	mutex_unlock(&epc->list_lock);  in pci_epc_remove_epf()
785  * pci_epc_linkup() - Notify the EPF device that EPC device has established a
799 	mutex_lock(&epc->list_lock);  in pci_epc_linkup()
800 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_linkup()
801 		mutex_lock(&epf->lock);  in pci_epc_linkup()
802 		if (epf->event_ops && epf->event_ops->link_up)  in pci_epc_linkup()
803 			epf->event_ops->link_up(epf);  in pci_epc_linkup()
804 		mutex_unlock(&epf->lock);  in pci_epc_linkup()
806 	mutex_unlock(&epc->list_lock);  in pci_epc_linkup()
811  * pci_epc_linkdown() - Notify the EPF device that EPC device has dropped the
825 	mutex_lock(&epc->list_lock);  in pci_epc_linkdown()
826 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_linkdown()
827 		mutex_lock(&epf->lock);  in pci_epc_linkdown()
828 		if (epf->event_ops && epf->event_ops->link_down)  in pci_epc_linkdown()
829 			epf->event_ops->link_down(epf);  in pci_epc_linkdown()
830 		mutex_unlock(&epf->lock);  in pci_epc_linkdown()
832 	mutex_unlock(&epc->list_lock);  in pci_epc_linkdown()
837  * pci_epc_init_notify() - Notify the EPF device that EPC device initialization
851 	mutex_lock(&epc->list_lock);  in pci_epc_init_notify()
852 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_init_notify()
853 		mutex_lock(&epf->lock);  in pci_epc_init_notify()
854 		if (epf->event_ops && epf->event_ops->epc_init)  in pci_epc_init_notify()
855 			epf->event_ops->epc_init(epf);  in pci_epc_init_notify()
856 		mutex_unlock(&epf->lock);  in pci_epc_init_notify()
858 	epc->init_complete = true;  in pci_epc_init_notify()
859 	mutex_unlock(&epc->list_lock);  in pci_epc_init_notify()
864  * pci_epc_notify_pending_init() - Notify the pending EPC device initialization
875 	if (epc->init_complete) {  in pci_epc_notify_pending_init()
876 		mutex_lock(&epf->lock);  in pci_epc_notify_pending_init()
877 		if (epf->event_ops && epf->event_ops->epc_init)  in pci_epc_notify_pending_init()
878 			epf->event_ops->epc_init(epf);  in pci_epc_notify_pending_init()
879 		mutex_unlock(&epf->lock);  in pci_epc_notify_pending_init()
885  * pci_epc_deinit_notify() - Notify the EPF device about EPC deinitialization
897 	mutex_lock(&epc->list_lock);  in pci_epc_deinit_notify()
898 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_deinit_notify()
899 		mutex_lock(&epf->lock);  in pci_epc_deinit_notify()
900 		if (epf->event_ops && epf->event_ops->epc_deinit)  in pci_epc_deinit_notify()
901 			epf->event_ops->epc_deinit(epf);  in pci_epc_deinit_notify()
902 		mutex_unlock(&epf->lock);  in pci_epc_deinit_notify()
904 	epc->init_complete = false;  in pci_epc_deinit_notify()
905 	mutex_unlock(&epc->list_lock);  in pci_epc_deinit_notify()
910  * pci_epc_bus_master_enable_notify() - Notify the EPF device that the EPC
925 	mutex_lock(&epc->list_lock);  in pci_epc_bus_master_enable_notify()
926 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_bus_master_enable_notify()
927 		mutex_lock(&epf->lock);  in pci_epc_bus_master_enable_notify()
928 		if (epf->event_ops && epf->event_ops->bus_master_enable)  in pci_epc_bus_master_enable_notify()
929 			epf->event_ops->bus_master_enable(epf);  in pci_epc_bus_master_enable_notify()
930 		mutex_unlock(&epf->lock);  in pci_epc_bus_master_enable_notify()
932 	mutex_unlock(&epc->list_lock);  in pci_epc_bus_master_enable_notify()
937  * pci_epc_destroy() - destroy the EPC device
944 	pci_ep_cfs_remove_epc_group(epc->group);  in pci_epc_destroy()
946 	pci_bus_release_domain_nr(epc->dev.parent, epc->domain_nr);  in pci_epc_destroy()
948 	device_unregister(&epc->dev);  in pci_epc_destroy()
958  * __pci_epc_create() - create a new endpoint controller (EPC) device
973 		ret = -EINVAL;  in __pci_epc_create()
979 		ret = -ENOMEM;  in __pci_epc_create()
983 	mutex_init(&epc->lock);  in __pci_epc_create()
984 	mutex_init(&epc->list_lock);  in __pci_epc_create()
985 	INIT_LIST_HEAD(&epc->pci_epf);  in __pci_epc_create()
987 	device_initialize(&epc->dev);  in __pci_epc_create()
988 	epc->dev.class = &pci_epc_class;  in __pci_epc_create()
989 	epc->dev.parent = dev;  in __pci_epc_create()
990 	epc->dev.release = pci_epc_release;  in __pci_epc_create()
991 	epc->ops = ops;  in __pci_epc_create()
994 	epc->domain_nr = pci_bus_find_domain_nr(NULL, dev);  in __pci_epc_create()
1003 	ret = dev_set_name(&epc->dev, "%s", dev_name(dev));  in __pci_epc_create()
1007 	ret = device_add(&epc->dev);  in __pci_epc_create()
1011 	epc->group = pci_ep_cfs_add_epc_group(dev_name(dev));  in __pci_epc_create()
1016 	put_device(&epc->dev);  in __pci_epc_create()
1024  * __devm_pci_epc_create() - create a new endpoint controller (EPC) device
1042 		return ERR_PTR(-ENOMEM);  in __devm_pci_epc_create()