1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic support for Memory System Cache Maintenance operations. 4 * 5 * Coherency maintenance drivers register with this simple framework that will 6 * iterate over each registered instance to first kick off invalidation and 7 * then to wait until it is complete. 8 * 9 * If no implementations are registered yet cpu_cache_has_invalidate_memregion() 10 * will return false. If this runs concurrently with unregistration then a 11 * race exists but this is no worse than the case where the operations instance 12 * responsible for a given memory region has not yet registered. 13 */ 14 #include <linux/cache_coherency.h> 15 #include <linux/cleanup.h> 16 #include <linux/container_of.h> 17 #include <linux/export.h> 18 #include <linux/kref.h> 19 #include <linux/list.h> 20 #include <linux/memregion.h> 21 #include <linux/module.h> 22 #include <linux/rwsem.h> 23 #include <linux/slab.h> 24 25 static LIST_HEAD(cache_ops_instance_list); 26 static DECLARE_RWSEM(cache_ops_instance_list_lock); 27 28 static void __cache_coherency_ops_instance_free(struct kref *kref) 29 { 30 struct cache_coherency_ops_inst *cci = 31 container_of(kref, struct cache_coherency_ops_inst, kref); 32 kfree(cci); 33 } 34 35 void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci) 36 { 37 kref_put(&cci->kref, __cache_coherency_ops_instance_free); 38 } 39 EXPORT_SYMBOL_GPL(cache_coherency_ops_instance_put); 40 41 static int cache_inval_one(struct cache_coherency_ops_inst *cci, void *data) 42 { 43 if (!cci->ops) 44 return -EINVAL; 45 46 return cci->ops->wbinv(cci, data); 47 } 48 49 static int cache_inval_done_one(struct cache_coherency_ops_inst *cci) 50 { 51 if (!cci->ops) 52 return -EINVAL; 53 54 if (!cci->ops->done) 55 return 0; 56 57 return cci->ops->done(cci); 58 } 59 60 static int cache_invalidate_memregion(phys_addr_t addr, size_t size) 61 { 62 int ret; 63 struct cache_coherency_ops_inst *cci; 64 struct cc_inval_params params = { 65 .addr = addr, 66 .size = size, 67 }; 68 69 guard(rwsem_read)(&cache_ops_instance_list_lock); 70 list_for_each_entry(cci, &cache_ops_instance_list, node) { 71 ret = cache_inval_one(cci, ¶ms); 72 if (ret) 73 return ret; 74 } 75 list_for_each_entry(cci, &cache_ops_instance_list, node) { 76 ret = cache_inval_done_one(cci); 77 if (ret) 78 return ret; 79 } 80 81 return 0; 82 } 83 84 struct cache_coherency_ops_inst * 85 _cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops, 86 size_t size) 87 { 88 struct cache_coherency_ops_inst *cci; 89 90 if (!ops || !ops->wbinv) 91 return NULL; 92 93 cci = kzalloc(size, GFP_KERNEL); 94 if (!cci) 95 return NULL; 96 97 cci->ops = ops; 98 INIT_LIST_HEAD(&cci->node); 99 kref_init(&cci->kref); 100 101 return cci; 102 } 103 EXPORT_SYMBOL_NS_GPL(_cache_coherency_ops_instance_alloc, "CACHE_COHERENCY"); 104 105 int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci) 106 { 107 guard(rwsem_write)(&cache_ops_instance_list_lock); 108 list_add(&cci->node, &cache_ops_instance_list); 109 110 return 0; 111 } 112 EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_register, "CACHE_COHERENCY"); 113 114 void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci) 115 { 116 guard(rwsem_write)(&cache_ops_instance_list_lock); 117 list_del(&cci->node); 118 } 119 EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_unregister, "CACHE_COHERENCY"); 120 121 int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len) 122 { 123 return cache_invalidate_memregion(start, len); 124 } 125 EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, "DEVMEM"); 126 127 /* 128 * Used for optimization / debug purposes only as removal can race 129 * 130 * Machines that do not support invalidation, e.g. VMs, will not have any 131 * operations instance to register and so this will always return false. 132 */ 133 bool cpu_cache_has_invalidate_memregion(void) 134 { 135 guard(rwsem_read)(&cache_ops_instance_list_lock); 136 return !list_empty(&cache_ops_instance_list); 137 } 138 EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM"); 139