1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Red Hat, Inc. 3 * Author: Jason Wang <jasowang@redhat.com> 4 * 5 * IOTLB implementation for vhost. 6 */ 7 #include <linux/slab.h> 8 #include <linux/vhost_iotlb.h> 9 #include <linux/module.h> 10 11 #define MOD_VERSION "0.1" 12 #define MOD_DESC "VHOST IOTLB" 13 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>" 14 #define MOD_LICENSE "GPL v2" 15 16 #define START(map) ((map)->start) 17 #define LAST(map) ((map)->last) 18 19 INTERVAL_TREE_DEFINE(struct vhost_iotlb_map, 20 rb, __u64, __subtree_last, 21 START, LAST, static inline, vhost_iotlb_itree); 22 23 static void vhost_iotlb_map_unlink(struct vhost_iotlb *iotlb, 24 struct vhost_iotlb_map *map) 25 { 26 vhost_iotlb_itree_remove(map, &iotlb->root); 27 list_del(&map->link); 28 iotlb->nmaps--; 29 } 30 31 /** 32 * vhost_iotlb_map_free - remove a map node and free it 33 * @iotlb: the IOTLB 34 * @map: the map that want to be remove and freed 35 */ 36 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, 37 struct vhost_iotlb_map *map) 38 { 39 vhost_iotlb_map_unlink(iotlb, map); 40 kfree(map); 41 } 42 EXPORT_SYMBOL_GPL(vhost_iotlb_map_free); 43 44 /** 45 * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB 46 * @iotlb: the IOTLB 47 * @start: start of the IOVA range 48 * @last: last of IOVA range 49 * @addr: the address that is mapped to @start 50 * @perm: access permission of this range 51 * @opaque: the opaque pointer for the new mapping 52 * 53 * Returns an error last is smaller than start or memory allocation 54 * fails 55 */ 56 int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb, 57 u64 start, u64 last, 58 u64 addr, unsigned int perm, 59 void *opaque) 60 { 61 struct vhost_iotlb_map *map; 62 63 if (last < start) 64 return -EFAULT; 65 66 if (!iotlb->limit) 67 return -EINVAL; 68 69 /* If the range being mapped is [0, ULONG_MAX], split it into two entries 70 * otherwise its size would overflow u64. 71 */ 72 if (start == 0 && last == ULONG_MAX) { 73 u64 mid = last / 2; 74 int err; 75 76 if (iotlb->limit < 2) 77 return -ENOSPC; 78 79 if (!(iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) && 80 iotlb->nmaps > iotlb->limit - 2) 81 return -ENOSPC; 82 83 err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, 84 perm, opaque); 85 if (err) 86 return err; 87 88 addr += mid + 1; 89 start = mid + 1; 90 } 91 92 if (iotlb->nmaps >= iotlb->limit) { 93 if (iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) { 94 map = list_first_entry(&iotlb->list, typeof(*map), link); 95 vhost_iotlb_map_unlink(iotlb, map); 96 } else { 97 return -ENOSPC; 98 } 99 } else { 100 map = kmalloc_obj(*map, GFP_ATOMIC); 101 if (!map) 102 return -ENOMEM; 103 } 104 105 map->start = start; 106 map->size = last - start + 1; 107 map->last = last; 108 map->addr = addr; 109 map->perm = perm; 110 map->opaque = opaque; 111 112 iotlb->nmaps++; 113 vhost_iotlb_itree_insert(map, &iotlb->root); 114 115 INIT_LIST_HEAD(&map->link); 116 list_add_tail(&map->link, &iotlb->list); 117 118 return 0; 119 } 120 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx); 121 122 int vhost_iotlb_add_range(struct vhost_iotlb *iotlb, 123 u64 start, u64 last, 124 u64 addr, unsigned int perm) 125 { 126 return vhost_iotlb_add_range_ctx(iotlb, start, last, 127 addr, perm, NULL); 128 } 129 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range); 130 131 /** 132 * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB 133 * @iotlb: the IOTLB 134 * @start: start of the IOVA range 135 * @last: last of IOVA range 136 */ 137 void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last) 138 { 139 struct vhost_iotlb_map *map; 140 141 while ((map = vhost_iotlb_itree_iter_first(&iotlb->root, 142 start, last))) 143 vhost_iotlb_map_free(iotlb, map); 144 } 145 EXPORT_SYMBOL_GPL(vhost_iotlb_del_range); 146 147 /** 148 * vhost_iotlb_init - initialize a vhost IOTLB 149 * @iotlb: the IOTLB that needs to be initialized 150 * @limit: maximum number of IOTLB entries 151 * @flags: VHOST_IOTLB_FLAG_XXX 152 */ 153 void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit, 154 unsigned int flags) 155 { 156 iotlb->root = RB_ROOT_CACHED; 157 iotlb->limit = limit; 158 iotlb->nmaps = 0; 159 iotlb->flags = flags; 160 INIT_LIST_HEAD(&iotlb->list); 161 } 162 EXPORT_SYMBOL_GPL(vhost_iotlb_init); 163 164 /** 165 * vhost_iotlb_alloc - add a new vhost IOTLB 166 * @limit: maximum number of IOTLB entries 167 * @flags: VHOST_IOTLB_FLAG_XXX 168 * 169 * Returns an error is memory allocation fails 170 */ 171 struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags) 172 { 173 struct vhost_iotlb *iotlb = kzalloc_obj(*iotlb); 174 175 if (!iotlb) 176 return NULL; 177 178 vhost_iotlb_init(iotlb, limit, flags); 179 180 return iotlb; 181 } 182 EXPORT_SYMBOL_GPL(vhost_iotlb_alloc); 183 184 /** 185 * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries) 186 * @iotlb: the IOTLB to be reset 187 */ 188 void vhost_iotlb_reset(struct vhost_iotlb *iotlb) 189 { 190 vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1); 191 } 192 EXPORT_SYMBOL_GPL(vhost_iotlb_reset); 193 194 /** 195 * vhost_iotlb_free - reset and free vhost IOTLB 196 * @iotlb: the IOTLB to be freed 197 */ 198 void vhost_iotlb_free(struct vhost_iotlb *iotlb) 199 { 200 if (iotlb) { 201 vhost_iotlb_reset(iotlb); 202 kfree(iotlb); 203 } 204 } 205 EXPORT_SYMBOL_GPL(vhost_iotlb_free); 206 207 /** 208 * vhost_iotlb_itree_first - return the first overlapped range 209 * @iotlb: the IOTLB 210 * @start: start of IOVA range 211 * @last: last byte in IOVA range 212 */ 213 struct vhost_iotlb_map * 214 vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last) 215 { 216 return vhost_iotlb_itree_iter_first(&iotlb->root, start, last); 217 } 218 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first); 219 220 /** 221 * vhost_iotlb_itree_next - return the next overlapped range 222 * @map: the starting map node 223 * @start: start of IOVA range 224 * @last: last byte IOVA range 225 */ 226 struct vhost_iotlb_map * 227 vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last) 228 { 229 return vhost_iotlb_itree_iter_next(map, start, last); 230 } 231 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next); 232 233 MODULE_VERSION(MOD_VERSION); 234 MODULE_DESCRIPTION(MOD_DESC); 235 MODULE_AUTHOR(MOD_AUTHOR); 236 MODULE_LICENSE(MOD_LICENSE); 237