1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Ram backed block device driver.
4 *
5 * Copyright (C) 2007 Nick Piggin
6 * Copyright (C) 2007 Novell Inc.
7 *
8 * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright
9 * of their respective owners.
10 */
11
12 #include <linux/init.h>
13 #include <linux/initrd.h>
14 #include <linux/module.h>
15 #include <linux/moduleparam.h>
16 #include <linux/major.h>
17 #include <linux/blkdev.h>
18 #include <linux/bio.h>
19 #include <linux/highmem.h>
20 #include <linux/mutex.h>
21 #include <linux/pagemap.h>
22 #include <linux/xarray.h>
23 #include <linux/fs.h>
24 #include <linux/slab.h>
25 #include <linux/backing-dev.h>
26 #include <linux/debugfs.h>
27
28 #include <linux/uaccess.h>
29
30 /*
31 * Each block ramdisk device has a xarray brd_pages of pages that stores
32 * the pages containing the block device's contents.
33 */
34 struct brd_device {
35 int brd_number;
36 struct gendisk *brd_disk;
37 struct list_head brd_list;
38
39 /*
40 * Backing store of pages. This is the contents of the block device.
41 */
42 struct xarray brd_pages;
43 u64 brd_nr_pages;
44 };
45
46 /*
47 * Look up and return a brd's page for a given sector.
48 */
brd_lookup_page(struct brd_device * brd,sector_t sector)49 static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
50 {
51 return xa_load(&brd->brd_pages, sector >> PAGE_SECTORS_SHIFT);
52 }
53
54 /*
55 * Insert a new page for a given sector, if one does not already exist.
56 */
brd_insert_page(struct brd_device * brd,sector_t sector,blk_opf_t opf)57 static struct page *brd_insert_page(struct brd_device *brd, sector_t sector,
58 blk_opf_t opf)
59 __releases(rcu)
60 __acquires(rcu)
61 {
62 gfp_t gfp = (opf & REQ_NOWAIT) ? GFP_NOWAIT : GFP_NOIO;
63 struct page *page, *ret;
64
65 rcu_read_unlock();
66 page = alloc_page(gfp | __GFP_ZERO | __GFP_HIGHMEM);
67 if (!page) {
68 rcu_read_lock();
69 return ERR_PTR(-ENOMEM);
70 }
71
72 xa_lock(&brd->brd_pages);
73 ret = __xa_cmpxchg(&brd->brd_pages, sector >> PAGE_SECTORS_SHIFT, NULL,
74 page, gfp);
75 rcu_read_lock();
76 if (ret) {
77 xa_unlock(&brd->brd_pages);
78 __free_page(page);
79 if (xa_is_err(ret))
80 return ERR_PTR(xa_err(ret));
81 return ret;
82 }
83 brd->brd_nr_pages++;
84 xa_unlock(&brd->brd_pages);
85 return page;
86 }
87
88 /*
89 * Free all backing store pages and xarray. This must only be called when
90 * there are no other users of the device.
91 */
brd_free_pages(struct brd_device * brd)92 static void brd_free_pages(struct brd_device *brd)
93 {
94 struct page *page;
95 pgoff_t idx;
96
97 xa_for_each(&brd->brd_pages, idx, page) {
98 __free_page(page);
99 cond_resched();
100 }
101
102 xa_destroy(&brd->brd_pages);
103 }
104
105 /*
106 * Process a single segment. The segment is capped to not cross page boundaries
107 * in both the bio and the brd backing memory.
108 */
brd_rw_bvec(struct brd_device * brd,struct bio * bio)109 static bool brd_rw_bvec(struct brd_device *brd, struct bio *bio)
110 {
111 struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
112 sector_t sector = bio->bi_iter.bi_sector;
113 u32 offset = (sector & (PAGE_SECTORS - 1)) << SECTOR_SHIFT;
114 blk_opf_t opf = bio->bi_opf;
115 struct page *page;
116 void *kaddr;
117
118 bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset);
119
120 rcu_read_lock();
121 page = brd_lookup_page(brd, sector);
122 if (!page && op_is_write(opf)) {
123 page = brd_insert_page(brd, sector, opf);
124 if (IS_ERR(page))
125 goto out_error;
126 }
127
128 kaddr = bvec_kmap_local(&bv);
129 if (op_is_write(opf)) {
130 memcpy_to_page(page, offset, kaddr, bv.bv_len);
131 } else {
132 if (page)
133 memcpy_from_page(kaddr, page, offset, bv.bv_len);
134 else
135 memset(kaddr, 0, bv.bv_len);
136 }
137 kunmap_local(kaddr);
138 rcu_read_unlock();
139
140 bio_advance_iter_single(bio, &bio->bi_iter, bv.bv_len);
141 return true;
142
143 out_error:
144 rcu_read_unlock();
145 if (PTR_ERR(page) == -ENOMEM && (opf & REQ_NOWAIT))
146 bio_wouldblock_error(bio);
147 else
148 bio_io_error(bio);
149 return false;
150 }
151
brd_free_one_page(struct rcu_head * head)152 static void brd_free_one_page(struct rcu_head *head)
153 {
154 struct page *page = container_of(head, struct page, rcu_head);
155
156 __free_page(page);
157 }
158
brd_do_discard(struct brd_device * brd,sector_t sector,u32 size)159 static void brd_do_discard(struct brd_device *brd, sector_t sector, u32 size)
160 {
161 sector_t aligned_sector = round_up(sector, PAGE_SECTORS);
162 sector_t aligned_end = round_down(
163 sector + (size >> SECTOR_SHIFT), PAGE_SECTORS);
164 struct page *page;
165
166 if (aligned_end <= aligned_sector)
167 return;
168
169 xa_lock(&brd->brd_pages);
170 while (aligned_sector < aligned_end && aligned_sector < rd_size * 2) {
171 page = __xa_erase(&brd->brd_pages, aligned_sector >> PAGE_SECTORS_SHIFT);
172 if (page) {
173 call_rcu(&page->rcu_head, brd_free_one_page);
174 brd->brd_nr_pages--;
175 }
176 aligned_sector += PAGE_SECTORS;
177 }
178 xa_unlock(&brd->brd_pages);
179 }
180
brd_submit_bio(struct bio * bio)181 static void brd_submit_bio(struct bio *bio)
182 {
183 struct brd_device *brd = bio->bi_bdev->bd_disk->private_data;
184
185 if (unlikely(op_is_discard(bio->bi_opf))) {
186 brd_do_discard(brd, bio->bi_iter.bi_sector,
187 bio->bi_iter.bi_size);
188 bio_endio(bio);
189 return;
190 }
191
192 do {
193 if (!brd_rw_bvec(brd, bio))
194 return;
195 } while (bio->bi_iter.bi_size);
196
197 bio_endio(bio);
198 }
199
200 static const struct block_device_operations brd_fops = {
201 .owner = THIS_MODULE,
202 .submit_bio = brd_submit_bio,
203 };
204
205 /*
206 * And now the modules code and kernel interface.
207 */
208 static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
209 module_param(rd_nr, int, 0444);
210 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
211
212 unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
213 module_param(rd_size, ulong, 0444);
214 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
215
216 static int max_part = 1;
217 module_param(max_part, int, 0444);
218 MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
219
220 MODULE_DESCRIPTION("Ram backed block device driver");
221 MODULE_LICENSE("GPL");
222 MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR);
223 MODULE_ALIAS("rd");
224
225 #ifndef MODULE
226 /* Legacy boot options - nonmodular */
ramdisk_size(char * str)227 static int __init ramdisk_size(char *str)
228 {
229 rd_size = simple_strtol(str, NULL, 0);
230 return 1;
231 }
232 __setup("ramdisk_size=", ramdisk_size);
233 #endif
234
235 /*
236 * The device scheme is derived from loop.c. Keep them in synch where possible
237 * (should share code eventually).
238 */
239 static LIST_HEAD(brd_devices);
240 static DEFINE_MUTEX(brd_devices_mutex);
241 static struct dentry *brd_debugfs_dir;
242
brd_find_or_alloc_device(int i)243 static struct brd_device *brd_find_or_alloc_device(int i)
244 {
245 struct brd_device *brd;
246
247 mutex_lock(&brd_devices_mutex);
248 list_for_each_entry(brd, &brd_devices, brd_list) {
249 if (brd->brd_number == i) {
250 mutex_unlock(&brd_devices_mutex);
251 return ERR_PTR(-EEXIST);
252 }
253 }
254
255 brd = kzalloc(sizeof(*brd), GFP_KERNEL);
256 if (!brd) {
257 mutex_unlock(&brd_devices_mutex);
258 return ERR_PTR(-ENOMEM);
259 }
260 brd->brd_number = i;
261 list_add_tail(&brd->brd_list, &brd_devices);
262 mutex_unlock(&brd_devices_mutex);
263 return brd;
264 }
265
brd_free_device(struct brd_device * brd)266 static void brd_free_device(struct brd_device *brd)
267 {
268 mutex_lock(&brd_devices_mutex);
269 list_del(&brd->brd_list);
270 mutex_unlock(&brd_devices_mutex);
271 kfree(brd);
272 }
273
brd_alloc(int i)274 static int brd_alloc(int i)
275 {
276 struct brd_device *brd;
277 struct gendisk *disk;
278 char buf[DISK_NAME_LEN];
279 int err = -ENOMEM;
280 struct queue_limits lim = {
281 /*
282 * This is so fdisk will align partitions on 4k, because of
283 * direct_access API needing 4k alignment, returning a PFN
284 * (This is only a problem on very small devices <= 4M,
285 * otherwise fdisk will align on 1M. Regardless this call
286 * is harmless)
287 */
288 .physical_block_size = PAGE_SIZE,
289 .max_hw_discard_sectors = UINT_MAX,
290 .max_discard_segments = 1,
291 .discard_granularity = PAGE_SIZE,
292 .features = BLK_FEAT_SYNCHRONOUS |
293 BLK_FEAT_NOWAIT,
294 };
295
296 brd = brd_find_or_alloc_device(i);
297 if (IS_ERR(brd))
298 return PTR_ERR(brd);
299
300 xa_init(&brd->brd_pages);
301
302 snprintf(buf, DISK_NAME_LEN, "ram%d", i);
303 if (!IS_ERR_OR_NULL(brd_debugfs_dir))
304 debugfs_create_u64(buf, 0444, brd_debugfs_dir,
305 &brd->brd_nr_pages);
306
307 disk = brd->brd_disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
308 if (IS_ERR(disk)) {
309 err = PTR_ERR(disk);
310 goto out_free_dev;
311 }
312 disk->major = RAMDISK_MAJOR;
313 disk->first_minor = i * max_part;
314 disk->minors = max_part;
315 disk->fops = &brd_fops;
316 disk->private_data = brd;
317 strscpy(disk->disk_name, buf, DISK_NAME_LEN);
318 set_capacity(disk, rd_size * 2);
319
320 err = add_disk(disk);
321 if (err)
322 goto out_cleanup_disk;
323
324 return 0;
325
326 out_cleanup_disk:
327 put_disk(disk);
328 out_free_dev:
329 brd_free_device(brd);
330 return err;
331 }
332
brd_probe(dev_t dev)333 static void brd_probe(dev_t dev)
334 {
335 brd_alloc(MINOR(dev) / max_part);
336 }
337
brd_cleanup(void)338 static void brd_cleanup(void)
339 {
340 struct brd_device *brd, *next;
341
342 debugfs_remove_recursive(brd_debugfs_dir);
343
344 list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
345 del_gendisk(brd->brd_disk);
346 put_disk(brd->brd_disk);
347 brd_free_pages(brd);
348 brd_free_device(brd);
349 }
350 }
351
brd_check_and_reset_par(void)352 static inline void brd_check_and_reset_par(void)
353 {
354 if (unlikely(!max_part))
355 max_part = 1;
356
357 /*
358 * make sure 'max_part' can be divided exactly by (1U << MINORBITS),
359 * otherwise, it is possiable to get same dev_t when adding partitions.
360 */
361 if ((1U << MINORBITS) % max_part != 0)
362 max_part = 1UL << fls(max_part);
363
364 if (max_part > DISK_MAX_PARTS) {
365 pr_info("brd: max_part can't be larger than %d, reset max_part = %d.\n",
366 DISK_MAX_PARTS, DISK_MAX_PARTS);
367 max_part = DISK_MAX_PARTS;
368 }
369 }
370
brd_init(void)371 static int __init brd_init(void)
372 {
373 int err, i;
374
375 /*
376 * brd module now has a feature to instantiate underlying device
377 * structure on-demand, provided that there is an access dev node.
378 *
379 * (1) if rd_nr is specified, create that many upfront. else
380 * it defaults to CONFIG_BLK_DEV_RAM_COUNT
381 * (2) User can further extend brd devices by create dev node themselves
382 * and have kernel automatically instantiate actual device
383 * on-demand. Example:
384 * mknod /path/devnod_name b 1 X # 1 is the rd major
385 * fdisk -l /path/devnod_name
386 * If (X / max_part) was not already created it will be created
387 * dynamically.
388 */
389
390 brd_check_and_reset_par();
391
392 brd_debugfs_dir = debugfs_create_dir("ramdisk_pages", NULL);
393
394 if (__register_blkdev(RAMDISK_MAJOR, "ramdisk", brd_probe)) {
395 err = -EIO;
396 goto out_free;
397 }
398
399 for (i = 0; i < rd_nr; i++)
400 brd_alloc(i);
401
402 pr_info("brd: module loaded\n");
403 return 0;
404
405 out_free:
406 brd_cleanup();
407
408 pr_info("brd: module NOT loaded !!!\n");
409 return err;
410 }
411
brd_exit(void)412 static void __exit brd_exit(void)
413 {
414
415 unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
416 brd_cleanup();
417
418 pr_info("brd: module unloaded\n");
419 }
420
421 module_init(brd_init);
422 module_exit(brd_exit);
423
424