1 // SPDX-License-Identifier: GPL-2.0+
6 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)  in ocxl_fn_get()
8 	return (get_device(&fn->dev) == NULL) ? NULL : fn;  in ocxl_fn_get()
11 static void ocxl_fn_put(struct ocxl_fn *fn)  in ocxl_fn_put()
13 	put_device(&fn->dev);  in ocxl_fn_put()
16 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)  in alloc_afu()
24 	kref_init(&afu->kref);  in alloc_afu()
25 	mutex_init(&afu->contexts_lock);  in alloc_afu()
26 	mutex_init(&afu->afu_control_lock);  in alloc_afu()
27 	idr_init(&afu->contexts_idr);  in alloc_afu()
28 	afu->fn = fn;  in alloc_afu()
33 static void free_afu(struct kref *kref)  in free_afu()
37 	idr_destroy(&afu->contexts_idr);  in free_afu()
38 	ocxl_fn_put(afu->fn);  in free_afu()
44 	kref_get(&afu->kref);  in ocxl_afu_get()
50 	kref_put(&afu->kref, free_afu);  in ocxl_afu_put()
54 static int assign_afu_actag(struct ocxl_afu *afu)  in assign_afu_actag()
56 	struct ocxl_fn *fn = afu->fn;  in assign_afu_actag()
58 	struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);  in assign_afu_actag()
64 	actag_count = afu->config.actag_supported *  in assign_afu_actag()
65 		fn->actag_enabled / fn->actag_supported;  in assign_afu_actag()
68 		dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n",  in assign_afu_actag()
72 	afu->actag_base = fn->actag_base + actag_offset;  in assign_afu_actag()
73 	afu->actag_enabled = actag_count;  in assign_afu_actag()
75 	ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos,  in assign_afu_actag()
76 				afu->actag_base, afu->actag_enabled);  in assign_afu_actag()
77 	dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n",  in assign_afu_actag()
78 		afu->actag_base, afu->actag_enabled);  in assign_afu_actag()
82 static void reclaim_afu_actag(struct ocxl_afu *afu)  in reclaim_afu_actag()
84 	struct ocxl_fn *fn = afu->fn;  in reclaim_afu_actag()
87 	start_offset = afu->actag_base - fn->actag_base;  in reclaim_afu_actag()
88 	size = afu->actag_enabled;  in reclaim_afu_actag()
89 	ocxl_actag_afu_free(afu->fn, start_offset, size);  in reclaim_afu_actag()
92 static int assign_afu_pasid(struct ocxl_afu *afu)  in assign_afu_pasid()
94 	struct ocxl_fn *fn = afu->fn;  in assign_afu_pasid()
96 	struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);  in assign_afu_pasid()
102 	pasid_count = 1 << afu->config.pasid_supported_log;  in assign_afu_pasid()
105 		dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n",  in assign_afu_pasid()
109 	afu->pasid_base = fn->pasid_base + pasid_offset;  in assign_afu_pasid()
110 	afu->pasid_count = 0;  in assign_afu_pasid()
111 	afu->pasid_max = pasid_count;  in assign_afu_pasid()
113 	ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos,  in assign_afu_pasid()
114 				afu->pasid_base,  in assign_afu_pasid()
115 				afu->config.pasid_supported_log);  in assign_afu_pasid()
116 	dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n",  in assign_afu_pasid()
117 		afu->pasid_base, pasid_count);  in assign_afu_pasid()
121 static void reclaim_afu_pasid(struct ocxl_afu *afu)  in reclaim_afu_pasid()
123 	struct ocxl_fn *fn = afu->fn;  in reclaim_afu_pasid()
126 	start_offset = afu->pasid_base - fn->pasid_base;  in reclaim_afu_pasid()
127 	size = 1 << afu->config.pasid_supported_log;  in reclaim_afu_pasid()
128 	ocxl_pasid_afu_free(afu->fn, start_offset, size);  in reclaim_afu_pasid()
131 static int reserve_fn_bar(struct ocxl_fn *fn, int bar)  in reserve_fn_bar()
133 	struct pci_dev *dev = to_pci_dev(fn->dev.parent);  in reserve_fn_bar()
137 		return -EINVAL;  in reserve_fn_bar()
140 	if (fn->bar_used[idx]++ == 0) {  in reserve_fn_bar()
148 static void release_fn_bar(struct ocxl_fn *fn, int bar)  in release_fn_bar()
150 	struct pci_dev *dev = to_pci_dev(fn->dev.parent);  in release_fn_bar()
157 	if (--fn->bar_used[idx] == 0)  in release_fn_bar()
159 	WARN_ON(fn->bar_used[idx] < 0);  in release_fn_bar()
162 static int map_mmio_areas(struct ocxl_afu *afu)  in map_mmio_areas()
165 	struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent);  in map_mmio_areas()
167 	rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);  in map_mmio_areas()
171 	rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);  in map_mmio_areas()
173 		release_fn_bar(afu->fn, afu->config.global_mmio_bar);  in map_mmio_areas()
177 	afu->global_mmio_start =  in map_mmio_areas()
178 		pci_resource_start(pci_dev, afu->config.global_mmio_bar) +  in map_mmio_areas()
179 		afu->config.global_mmio_offset;  in map_mmio_areas()
180 	afu->pp_mmio_start =  in map_mmio_areas()
181 		pci_resource_start(pci_dev, afu->config.pp_mmio_bar) +  in map_mmio_areas()
182 		afu->config.pp_mmio_offset;  in map_mmio_areas()
184 	afu->global_mmio_ptr = ioremap(afu->global_mmio_start,  in map_mmio_areas()
185 				afu->config.global_mmio_size);  in map_mmio_areas()
186 	if (!afu->global_mmio_ptr) {  in map_mmio_areas()
187 		release_fn_bar(afu->fn, afu->config.pp_mmio_bar);  in map_mmio_areas()
188 		release_fn_bar(afu->fn, afu->config.global_mmio_bar);  in map_mmio_areas()
189 		dev_err(&pci_dev->dev, "Error mapping global mmio area\n");  in map_mmio_areas()
190 		return -ENOMEM;  in map_mmio_areas()
194 	 * Leave an empty page between the per-process mmio area and  in map_mmio_areas()
197 	afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;  in map_mmio_areas()
201 static void unmap_mmio_areas(struct ocxl_afu *afu)  in unmap_mmio_areas()
203 	if (afu->global_mmio_ptr) {  in unmap_mmio_areas()
204 		iounmap(afu->global_mmio_ptr);  in unmap_mmio_areas()
205 		afu->global_mmio_ptr = NULL;  in unmap_mmio_areas()
207 	afu->global_mmio_start = 0;  in unmap_mmio_areas()
208 	afu->pp_mmio_start = 0;  in unmap_mmio_areas()
209 	release_fn_bar(afu->fn, afu->config.pp_mmio_bar);  in unmap_mmio_areas()
210 	release_fn_bar(afu->fn, afu->config.global_mmio_bar);  in unmap_mmio_areas()
213 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)  in configure_afu()
217 	rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);  in configure_afu()
242 static void deconfigure_afu(struct ocxl_afu *afu)  in deconfigure_afu()
249 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)  in activate_afu()
251 	ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);  in activate_afu()
256 static void deactivate_afu(struct ocxl_afu *afu)  in deactivate_afu()
258 	struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);  in deactivate_afu()
260 	ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);  in deactivate_afu()
263 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)  in init_afu()
270 		return -ENOMEM;  in init_afu()
285 	list_add_tail(&afu->list, &fn->afu_list);  in init_afu()
290 static void remove_afu(struct ocxl_afu *afu)  in remove_afu()
292 	list_del(&afu->list);  in remove_afu()
299 static struct ocxl_fn *alloc_function(void)  in alloc_function()
307 	INIT_LIST_HEAD(&fn->afu_list);  in alloc_function()
308 	INIT_LIST_HEAD(&fn->pasid_list);  in alloc_function()
309 	INIT_LIST_HEAD(&fn->actag_list);  in alloc_function()
314 static void free_function(struct ocxl_fn *fn)  in free_function()
316 	WARN_ON(!list_empty(&fn->afu_list));  in free_function()
317 	WARN_ON(!list_empty(&fn->pasid_list));  in free_function()
321 static void free_function_dev(struct device *dev)  in free_function_dev()
328 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)  in set_function_device()
330 	fn->dev.parent = &dev->dev;  in set_function_device()
331 	fn->dev.release = free_function_dev;  in set_function_device()
332 	return dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));  in set_function_device()
335 static int assign_function_actag(struct ocxl_fn *fn)  in assign_function_actag()
337 	struct pci_dev *dev = to_pci_dev(fn->dev.parent);  in assign_function_actag()
345 	fn->actag_base = base;  in assign_function_actag()
346 	fn->actag_enabled = enabled;  in assign_function_actag()
347 	fn->actag_supported = supported;  in assign_function_actag()
349 	ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,  in assign_function_actag()
350 			fn->actag_base,	fn->actag_enabled);  in assign_function_actag()
351 	dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",  in assign_function_actag()
352 		fn->actag_base, fn->actag_enabled);  in assign_function_actag()
356 static int set_function_pasid(struct ocxl_fn *fn)  in set_function_pasid()
358 	struct pci_dev *dev = to_pci_dev(fn->dev.parent);  in set_function_pasid()
362 	if (fn->config.max_pasid_log < 0)  in set_function_pasid()
369 	desired_count = 1 << fn->config.max_pasid_log;  in set_function_pasid()
372 		dev_err(&fn->dev,  in set_function_pasid()
375 		return -ENOSPC;  in set_function_pasid()
378 	fn->pasid_base = 0;  in set_function_pasid()
382 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)  in configure_function()
388 		dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);  in configure_function()
400 	 * - there's not status bit to know when the reset is done. We  in configure_function()
401 	 *   should try reading the config space to know when it's  in configure_function()
403 	 * - probably something like:  in configure_function()
406 	 *	issue config read  in configure_function()
407 	 *	allow device up to 1 sec to return success on config  in configure_function()
413 	rc = ocxl_config_read_function(dev, &fn->config);  in configure_function()
429 	rc = ocxl_link_setup(dev, 0, &fn->link);  in configure_function()
433 	rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);  in configure_function()
435 		ocxl_link_release(dev, fn->link);  in configure_function()
441 static void deconfigure_function(struct ocxl_fn *fn)  in deconfigure_function()
443 	struct pci_dev *dev = to_pci_dev(fn->dev.parent);  in deconfigure_function()
445 	ocxl_link_release(dev, fn->link);  in deconfigure_function()
449 static struct ocxl_fn *init_function(struct pci_dev *dev)  in init_function()
456 		return ERR_PTR(-ENOMEM);  in init_function()
464 	rc = device_register(&fn->dev);  in init_function()
467 		put_device(&fn->dev);  in init_function()
482 		dev_err(&dev->dev, "Unsupported memory model (hash)\n");  in ocxl_function_open()
483 		return ERR_PTR(-ENODEV);  in ocxl_function_open()
488 		dev_err(&dev->dev, "function init failed: %li\n",  in ocxl_function_open()
493 	for (afu = 0; afu <= fn->config.max_afu_index; afu++) {  in ocxl_function_open()
494 		rc = ocxl_config_check_afu_index(dev, &fn->config, afu);  in ocxl_function_open()
498 				dev_err(&dev->dev,  in ocxl_function_open()
505 	dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);  in ocxl_function_open()
512 	return &fn->afu_list;  in ocxl_function_afu_list()
520 	list_for_each_entry(afu, &fn->afu_list, list) {  in ocxl_function_fetch_afu()
521 		if (afu->config.idx == afu_idx)  in ocxl_function_fetch_afu()
531 	return &fn->config;  in ocxl_function_config()
539 	list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {  in ocxl_function_close()
544 	device_unregister(&fn->dev);  in ocxl_function_close()
552 	return &afu->config;  in ocxl_afu_config()
558 	afu->private = private;  in ocxl_afu_set_private()
565 		return afu->private;  in ocxl_afu_get_private()