1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Red Hat, Inc.
3 * Author: Jason Wang <jasowang@redhat.com>
4 *
5 * IOTLB implementation for vhost.
6 */
7 #include <linux/slab.h>
8 #include <linux/vhost_iotlb.h>
9 #include <linux/module.h>
10
11 #define MOD_VERSION "0.1"
12 #define MOD_DESC "VHOST IOTLB"
13 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
14 #define MOD_LICENSE "GPL v2"
15
16 #define START(map) ((map)->start)
17 #define LAST(map) ((map)->last)
18
19 INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
20 rb, __u64, __subtree_last,
21 START, LAST, static inline, vhost_iotlb_itree);
22
vhost_iotlb_map_unlink(struct vhost_iotlb * iotlb,struct vhost_iotlb_map * map)23 static void vhost_iotlb_map_unlink(struct vhost_iotlb *iotlb,
24 struct vhost_iotlb_map *map)
25 {
26 vhost_iotlb_itree_remove(map, &iotlb->root);
27 list_del(&map->link);
28 iotlb->nmaps--;
29 }
30
31 /**
32 * vhost_iotlb_map_free - remove a map node and free it
33 * @iotlb: the IOTLB
34 * @map: the map that want to be remove and freed
35 */
vhost_iotlb_map_free(struct vhost_iotlb * iotlb,struct vhost_iotlb_map * map)36 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
37 struct vhost_iotlb_map *map)
38 {
39 vhost_iotlb_map_unlink(iotlb, map);
40 kfree(map);
41 }
42 EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
43
44 /**
45 * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB
46 * @iotlb: the IOTLB
47 * @start: start of the IOVA range
48 * @last: last of IOVA range
49 * @addr: the address that is mapped to @start
50 * @perm: access permission of this range
51 * @opaque: the opaque pointer for the new mapping
52 *
53 * Returns an error if last is smaller than start or memory allocation
54 * fails
55 */
vhost_iotlb_add_range_ctx(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm,void * opaque)56 int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
57 u64 start, u64 last,
58 u64 addr, unsigned int perm,
59 void *opaque)
60 {
61 struct vhost_iotlb_map *map;
62
63 if (last < start)
64 return -EFAULT;
65
66 if (!iotlb->limit)
67 return -EINVAL;
68
69 /* If the range being mapped is [0, ULONG_MAX], split it into two entries
70 * otherwise its size would overflow u64.
71 */
72 if (start == 0 && last == ULONG_MAX) {
73 u64 mid = last / 2;
74 int err;
75
76 if (iotlb->limit < 2)
77 return -ENOSPC;
78
79 if (!(iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) &&
80 iotlb->nmaps > iotlb->limit - 2)
81 return -ENOSPC;
82
83 err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
84 perm, opaque);
85 if (err)
86 return err;
87
88 addr += mid + 1;
89 start = mid + 1;
90 }
91
92 if (iotlb->nmaps >= iotlb->limit) {
93 if (iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
94 map = list_first_entry(&iotlb->list, typeof(*map), link);
95 vhost_iotlb_map_unlink(iotlb, map);
96 } else {
97 return -ENOSPC;
98 }
99 } else {
100 map = kmalloc_obj(*map, GFP_ATOMIC);
101 if (!map)
102 return -ENOMEM;
103 }
104
105 map->start = start;
106 map->size = last - start + 1;
107 map->last = last;
108 map->addr = addr;
109 map->perm = perm;
110 map->opaque = opaque;
111
112 iotlb->nmaps++;
113 vhost_iotlb_itree_insert(map, &iotlb->root);
114
115 INIT_LIST_HEAD(&map->link);
116 list_add_tail(&map->link, &iotlb->list);
117
118 return 0;
119 }
120 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx);
121
vhost_iotlb_add_range(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm)122 int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
123 u64 start, u64 last,
124 u64 addr, unsigned int perm)
125 {
126 return vhost_iotlb_add_range_ctx(iotlb, start, last,
127 addr, perm, NULL);
128 }
129 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
130
131 /**
132 * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB
133 * @iotlb: the IOTLB
134 * @start: start of the IOVA range
135 * @last: last of IOVA range
136 */
vhost_iotlb_del_range(struct vhost_iotlb * iotlb,u64 start,u64 last)137 void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
138 {
139 struct vhost_iotlb_map *map;
140
141 while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
142 start, last)))
143 vhost_iotlb_map_free(iotlb, map);
144 }
145 EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
146
147 /**
148 * vhost_iotlb_init - initialize a vhost IOTLB
149 * @iotlb: the IOTLB that needs to be initialized
150 * @limit: maximum number of IOTLB entries
151 * @flags: VHOST_IOTLB_FLAG_XXX
152 */
vhost_iotlb_init(struct vhost_iotlb * iotlb,unsigned int limit,unsigned int flags)153 void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit,
154 unsigned int flags)
155 {
156 iotlb->root = RB_ROOT_CACHED;
157 iotlb->limit = limit;
158 iotlb->nmaps = 0;
159 iotlb->flags = flags;
160 INIT_LIST_HEAD(&iotlb->list);
161 }
162 EXPORT_SYMBOL_GPL(vhost_iotlb_init);
163
164 /**
165 * vhost_iotlb_alloc - allocate a new vhost IOTLB
166 * @limit: maximum number of IOTLB entries
167 * @flags: VHOST_IOTLB_FLAG_XXX
168 *
169 * Returns NULL if memory allocation fails
170 */
vhost_iotlb_alloc(unsigned int limit,unsigned int flags)171 struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
172 {
173 struct vhost_iotlb *iotlb = kzalloc_obj(*iotlb);
174
175 if (!iotlb)
176 return NULL;
177
178 vhost_iotlb_init(iotlb, limit, flags);
179
180 return iotlb;
181 }
182 EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
183
184 /**
185 * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
186 * @iotlb: the IOTLB to be reset
187 */
vhost_iotlb_reset(struct vhost_iotlb * iotlb)188 void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
189 {
190 vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
191 }
192 EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
193
194 /**
195 * vhost_iotlb_free - reset and free vhost IOTLB
196 * @iotlb: the IOTLB to be freed
197 */
vhost_iotlb_free(struct vhost_iotlb * iotlb)198 void vhost_iotlb_free(struct vhost_iotlb *iotlb)
199 {
200 if (iotlb) {
201 vhost_iotlb_reset(iotlb);
202 kfree(iotlb);
203 }
204 }
205 EXPORT_SYMBOL_GPL(vhost_iotlb_free);
206
207 /**
208 * vhost_iotlb_itree_first - return the first overlapped range
209 * @iotlb: the IOTLB
210 * @start: start of IOVA range
211 * @last: last byte in IOVA range
212 */
213 struct vhost_iotlb_map *
vhost_iotlb_itree_first(struct vhost_iotlb * iotlb,u64 start,u64 last)214 vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
215 {
216 return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
217 }
218 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
219
220 /**
221 * vhost_iotlb_itree_next - return the next overlapped range
222 * @map: the starting map node
223 * @start: start of IOVA range
224 * @last: last byte IOVA range
225 */
226 struct vhost_iotlb_map *
vhost_iotlb_itree_next(struct vhost_iotlb_map * map,u64 start,u64 last)227 vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
228 {
229 return vhost_iotlb_itree_iter_next(map, start, last);
230 }
231 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
232
233 MODULE_VERSION(MOD_VERSION);
234 MODULE_DESCRIPTION(MOD_DESC);
235 MODULE_AUTHOR(MOD_AUTHOR);
236 MODULE_LICENSE(MOD_LICENSE);
237