alloc.c (c74a7469f97c0f40b46e82ee979f9fb1bb6e847c) alloc.c (6f10f7d1b02b1bbc305f88d7696445dd38b13881)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Primary bucket allocation code
4 *
5 * Copyright 2012 Google, Inc.
6 *
7 * Allocation in bcache is done in terms of buckets:
8 *

--- 73 unchanged lines hidden (view full) ---

82
83 return ret;
84}
85
86void bch_rescale_priorities(struct cache_set *c, int sectors)
87{
88 struct cache *ca;
89 struct bucket *b;
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Primary bucket allocation code
4 *
5 * Copyright 2012 Google, Inc.
6 *
7 * Allocation in bcache is done in terms of buckets:
8 *

--- 73 unchanged lines hidden (view full) ---

82
83 return ret;
84}
85
86void bch_rescale_priorities(struct cache_set *c, int sectors)
87{
88 struct cache *ca;
89 struct bucket *b;
90 unsigned next = c->nbuckets * c->sb.bucket_size / 1024;
91 unsigned i;
90 unsigned int next = c->nbuckets * c->sb.bucket_size / 1024;
91 unsigned int i;
92 int r;
93
94 atomic_sub(sectors, &c->rescale);
95
96 do {
97 r = atomic_read(&c->rescale);
98
99 if (r >= 0)

--- 64 unchanged lines hidden (view full) ---

164 * bucket, and in order for that multiply to make sense we have to scale bucket
165 *
166 * Thus, we scale the bucket priorities so that the bucket with the smallest
167 * prio is worth 1/8th of what INITIAL_PRIO is worth.
168 */
169
170#define bucket_prio(b) \
171({ \
92 int r;
93
94 atomic_sub(sectors, &c->rescale);
95
96 do {
97 r = atomic_read(&c->rescale);
98
99 if (r >= 0)

--- 64 unchanged lines hidden (view full) ---

164 * bucket, and in order for that multiply to make sense we have to scale bucket
165 *
166 * Thus, we scale the bucket priorities so that the bucket with the smallest
167 * prio is worth 1/8th of what INITIAL_PRIO is worth.
168 */
169
170#define bucket_prio(b) \
171({ \
172 unsigned min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; \
172 unsigned int min_prio = (INITIAL_PRIO - ca->set->min_prio) / 8; \
173 \
174 (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); \
175})
176
177#define bucket_max_cmp(l, r) (bucket_prio(l) < bucket_prio(r))
178#define bucket_min_cmp(l, r) (bucket_prio(l) > bucket_prio(r))
179
180static void invalidate_buckets_lru(struct cache *ca)

--- 115 unchanged lines hidden (view full) ---

296 schedule(); \
297 mutex_lock(&(ca)->set->bucket_lock); \
298 } \
299 __set_current_state(TASK_RUNNING); \
300} while (0)
301
302static int bch_allocator_push(struct cache *ca, long bucket)
303{
173 \
174 (b->prio - ca->set->min_prio + min_prio) * GC_SECTORS_USED(b); \
175})
176
177#define bucket_max_cmp(l, r) (bucket_prio(l) < bucket_prio(r))
178#define bucket_min_cmp(l, r) (bucket_prio(l) > bucket_prio(r))
179
180static void invalidate_buckets_lru(struct cache *ca)

--- 115 unchanged lines hidden (view full) ---

296 schedule(); \
297 mutex_lock(&(ca)->set->bucket_lock); \
298 } \
299 __set_current_state(TASK_RUNNING); \
300} while (0)
301
302static int bch_allocator_push(struct cache *ca, long bucket)
303{
304 unsigned i;
304 unsigned int i;
305
306 /* Prios/gens are actually the most important reserve */
307 if (fifo_push(&ca->free[RESERVE_PRIO], bucket))
308 return true;
309
310 for (i = 0; i < RESERVE_NR; i++)
311 if (fifo_push(&ca->free[i], bucket))
312 return true;

--- 67 unchanged lines hidden (view full) ---

380 }
381out:
382 wait_for_kthread_stop();
383 return 0;
384}
385
386/* Allocation */
387
305
306 /* Prios/gens are actually the most important reserve */
307 if (fifo_push(&ca->free[RESERVE_PRIO], bucket))
308 return true;
309
310 for (i = 0; i < RESERVE_NR; i++)
311 if (fifo_push(&ca->free[i], bucket))
312 return true;

--- 67 unchanged lines hidden (view full) ---

380 }
381out:
382 wait_for_kthread_stop();
383 return 0;
384}
385
386/* Allocation */
387
388long bch_bucket_alloc(struct cache *ca, unsigned reserve, bool wait)
388long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait)
389{
390 DEFINE_WAIT(w);
391 struct bucket *b;
392 long r;
393
394 /* fastpath */
395 if (fifo_pop(&ca->free[RESERVE_NONE], r) ||
396 fifo_pop(&ca->free[reserve], r))

--- 19 unchanged lines hidden (view full) ---

416 if (ca->alloc_thread)
417 wake_up_process(ca->alloc_thread);
418
419 trace_bcache_alloc(ca, reserve);
420
421 if (expensive_debug_checks(ca->set)) {
422 size_t iter;
423 long i;
389{
390 DEFINE_WAIT(w);
391 struct bucket *b;
392 long r;
393
394 /* fastpath */
395 if (fifo_pop(&ca->free[RESERVE_NONE], r) ||
396 fifo_pop(&ca->free[reserve], r))

--- 19 unchanged lines hidden (view full) ---

416 if (ca->alloc_thread)
417 wake_up_process(ca->alloc_thread);
418
419 trace_bcache_alloc(ca, reserve);
420
421 if (expensive_debug_checks(ca->set)) {
422 size_t iter;
423 long i;
424 unsigned j;
424 unsigned int j;
425
426 for (iter = 0; iter < prio_buckets(ca) * 2; iter++)
427 BUG_ON(ca->prio_buckets[iter] == (uint64_t) r);
428
429 for (j = 0; j < RESERVE_NR; j++)
430 fifo_for_each(i, &ca->free[j], iter)
431 BUG_ON(i == r);
432 fifo_for_each(i, &ca->free_inc, iter)

--- 32 unchanged lines hidden (view full) ---

465 if (ca->set->avail_nbuckets < ca->set->nbuckets) {
466 ca->set->avail_nbuckets++;
467 bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
468 }
469}
470
471void bch_bucket_free(struct cache_set *c, struct bkey *k)
472{
425
426 for (iter = 0; iter < prio_buckets(ca) * 2; iter++)
427 BUG_ON(ca->prio_buckets[iter] == (uint64_t) r);
428
429 for (j = 0; j < RESERVE_NR; j++)
430 fifo_for_each(i, &ca->free[j], iter)
431 BUG_ON(i == r);
432 fifo_for_each(i, &ca->free_inc, iter)

--- 32 unchanged lines hidden (view full) ---

465 if (ca->set->avail_nbuckets < ca->set->nbuckets) {
466 ca->set->avail_nbuckets++;
467 bch_update_bucket_in_use(ca->set, &ca->set->gc_stats);
468 }
469}
470
471void bch_bucket_free(struct cache_set *c, struct bkey *k)
472{
473 unsigned i;
473 unsigned int i;
474
475 for (i = 0; i < KEY_PTRS(k); i++)
476 __bch_bucket_free(PTR_CACHE(c, k, i),
477 PTR_BUCKET(c, k, i));
478}
479
474
475 for (i = 0; i < KEY_PTRS(k); i++)
476 __bch_bucket_free(PTR_CACHE(c, k, i),
477 PTR_BUCKET(c, k, i));
478}
479
480int __bch_bucket_alloc_set(struct cache_set *c, unsigned reserve,
480int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
481 struct bkey *k, int n, bool wait)
482{
483 int i;
484
485 lockdep_assert_held(&c->bucket_lock);
486 BUG_ON(!n || n > c->caches_loaded || n > 8);
487
488 bkey_init(k);

--- 16 unchanged lines hidden (view full) ---

505
506 return 0;
507err:
508 bch_bucket_free(c, k);
509 bkey_put(c, k);
510 return -1;
511}
512
481 struct bkey *k, int n, bool wait)
482{
483 int i;
484
485 lockdep_assert_held(&c->bucket_lock);
486 BUG_ON(!n || n > c->caches_loaded || n > 8);
487
488 bkey_init(k);

--- 16 unchanged lines hidden (view full) ---

505
506 return 0;
507err:
508 bch_bucket_free(c, k);
509 bkey_put(c, k);
510 return -1;
511}
512
513int bch_bucket_alloc_set(struct cache_set *c, unsigned reserve,
513int bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
514 struct bkey *k, int n, bool wait)
515{
516 int ret;
517 mutex_lock(&c->bucket_lock);
518 ret = __bch_bucket_alloc_set(c, reserve, k, n, wait);
519 mutex_unlock(&c->bucket_lock);
520 return ret;
521}
522
523/* Sector allocator */
524
525struct open_bucket {
526 struct list_head list;
514 struct bkey *k, int n, bool wait)
515{
516 int ret;
517 mutex_lock(&c->bucket_lock);
518 ret = __bch_bucket_alloc_set(c, reserve, k, n, wait);
519 mutex_unlock(&c->bucket_lock);
520 return ret;
521}
522
523/* Sector allocator */
524
525struct open_bucket {
526 struct list_head list;
527 unsigned last_write_point;
528 unsigned sectors_free;
527 unsigned int last_write_point;
528 unsigned int sectors_free;
529 BKEY_PADDED(key);
530};
531
532/*
533 * We keep multiple buckets open for writes, and try to segregate different
534 * write streams for better cache utilization: first we try to segregate flash
535 * only volume write streams from cached devices, secondly we look for a bucket
536 * where the last write to it was sequential with the current write, and

--- 14 unchanged lines hidden (view full) ---

551 * data to the same buckets it'd get invalidated at the same time.
552 *
553 * Both of those tasks will be doing fairly random IO so we can't rely on
554 * detecting sequential IO to segregate their data, but going off of the task
555 * should be a sane heuristic.
556 */
557static struct open_bucket *pick_data_bucket(struct cache_set *c,
558 const struct bkey *search,
529 BKEY_PADDED(key);
530};
531
532/*
533 * We keep multiple buckets open for writes, and try to segregate different
534 * write streams for better cache utilization: first we try to segregate flash
535 * only volume write streams from cached devices, secondly we look for a bucket
536 * where the last write to it was sequential with the current write, and

--- 14 unchanged lines hidden (view full) ---

551 * data to the same buckets it'd get invalidated at the same time.
552 *
553 * Both of those tasks will be doing fairly random IO so we can't rely on
554 * detecting sequential IO to segregate their data, but going off of the task
555 * should be a sane heuristic.
556 */
557static struct open_bucket *pick_data_bucket(struct cache_set *c,
558 const struct bkey *search,
559 unsigned write_point,
559 unsigned int write_point,
560 struct bkey *alloc)
561{
562 struct open_bucket *ret, *ret_task = NULL;
563
564 list_for_each_entry_reverse(ret, &c->data_buckets, list)
565 if (UUID_FLASH_ONLY(&c->uuids[KEY_INODE(&ret->key)]) !=
566 UUID_FLASH_ONLY(&c->uuids[KEY_INODE(search)]))
567 continue;

--- 22 unchanged lines hidden (view full) ---

590 * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
591 * end of the newly allocated space).
592 *
593 * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
594 * sectors were actually allocated.
595 *
596 * If s->writeback is true, will not fail.
597 */
560 struct bkey *alloc)
561{
562 struct open_bucket *ret, *ret_task = NULL;
563
564 list_for_each_entry_reverse(ret, &c->data_buckets, list)
565 if (UUID_FLASH_ONLY(&c->uuids[KEY_INODE(&ret->key)]) !=
566 UUID_FLASH_ONLY(&c->uuids[KEY_INODE(search)]))
567 continue;

--- 22 unchanged lines hidden (view full) ---

590 * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
591 * end of the newly allocated space).
592 *
593 * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
594 * sectors were actually allocated.
595 *
596 * If s->writeback is true, will not fail.
597 */
598bool bch_alloc_sectors(struct cache_set *c, struct bkey *k, unsigned sectors,
599 unsigned write_point, unsigned write_prio, bool wait)
598bool bch_alloc_sectors(struct cache_set *c,
599 struct bkey *k,
600 unsigned int sectors,
601 unsigned int write_point,
602 unsigned int write_prio,
603 bool wait)
600{
601 struct open_bucket *b;
602 BKEY_PADDED(key) alloc;
604{
605 struct open_bucket *b;
606 BKEY_PADDED(key) alloc;
603 unsigned i;
607 unsigned int i;
604
605 /*
606 * We might have to allocate a new bucket, which we can't do with a
607 * spinlock held. So if we have to allocate, we drop the lock, allocate
608 * and then retry. KEY_PTRS() indicates whether alloc points to
609 * allocated bucket(s).
610 */
611
612 bkey_init(&alloc.key);
613 spin_lock(&c->data_bucket_lock);
614
615 while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
608
609 /*
610 * We might have to allocate a new bucket, which we can't do with a
611 * spinlock held. So if we have to allocate, we drop the lock, allocate
612 * and then retry. KEY_PTRS() indicates whether alloc points to
613 * allocated bucket(s).
614 */
615
616 bkey_init(&alloc.key);
617 spin_lock(&c->data_bucket_lock);
618
619 while (!(b = pick_data_bucket(c, k, write_point, &alloc.key))) {
616 unsigned watermark = write_prio
620 unsigned int watermark = write_prio
617 ? RESERVE_MOVINGGC
618 : RESERVE_NONE;
619
620 spin_unlock(&c->data_bucket_lock);
621
622 if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
623 return false;
624

--- 99 unchanged lines hidden --- 		621 ? RESERVE_MOVINGGC
622 : RESERVE_NONE;
623
624 spin_unlock(&c->data_bucket_lock);
625
626 if (bch_bucket_alloc_set(c, watermark, &alloc.key, 1, wait))
627 return false;
628

--- 99 unchanged lines hidden ---