Lines Matching defs:cache
42 * In the current (newest) era, block map pages are not written unless there is cache pressure. In
121 * For adjusting VDO page cache statistic fields which are only mutated on the logical zone thread.
158 struct vdo_page_cache *cache = info->cache;
160 return &cache->pages[(info - cache->infos) * VDO_BLOCK_SIZE];
177 * @cache: The page cache.
181 static int initialize_info(struct vdo_page_cache *cache)
185 INIT_LIST_HEAD(&cache->free_list);
186 for (info = cache->infos; info < cache->infos + cache->page_count; info++) {
189 info->cache = cache;
193 result = create_metadata_vio(cache->vdo, VIO_TYPE_BLOCK_MAP,
200 info->vio->completion.callback_thread_id = cache->zone->thread_id;
203 list_add_tail(&info->state_entry, &cache->free_list);
211 * allocate_cache_components() - Allocate components of the cache which require their own
213 * @cache: The page cache.
219 static int __must_check allocate_cache_components(struct vdo_page_cache *cache)
221 u64 size = cache->page_count * (u64) VDO_BLOCK_SIZE;
224 result = vdo_allocate(cache->page_count, "page infos", &cache->infos);
228 result = vdo_allocate_memory(size, VDO_BLOCK_SIZE, "cache pages", &cache->pages);
232 result = vdo_int_map_create(cache->page_count, &cache->page_map);
236 return initialize_info(cache);
240 * assert_on_cache_thread() - Assert that a function has been called on the VDO page cache's
242 * @cache: The page cache.
245 static inline void assert_on_cache_thread(struct vdo_page_cache *cache,
250 VDO_ASSERT_LOG_ONLY((thread_id == cache->zone->thread_id),
251 "%s() must only be called on cache thread %d, not thread %d",
252 function_name, cache->zone->thread_id, thread_id);
255 /** assert_io_allowed() - Assert that a page cache may issue I/O. */
256 static inline void assert_io_allowed(struct vdo_page_cache *cache)
258 VDO_ASSERT_LOG_ONLY(!vdo_is_state_quiescent(&cache->zone->state),
259 "VDO page cache may issue I/O");
262 /** report_cache_pressure() - Log and, if enabled, report cache pressure. */
263 static void report_cache_pressure(struct vdo_page_cache *cache)
265 ADD_ONCE(cache->stats.cache_pressure, 1);
266 if (cache->waiter_count > cache->page_count) {
267 if ((cache->pressure_report % LOG_INTERVAL) == 0)
268 vdo_log_info("page cache pressure %u", cache->stats.cache_pressure);
270 if (++cache->pressure_report >= DISPLAY_INTERVAL)
271 cache->pressure_report = 0;
307 struct block_map_statistics *stats = &info->cache->stats;
342 if (info->cache->lru_list.prev != &info->lru_entry)
343 list_move_tail(&info->lru_entry, &info->cache->lru_list);
364 list_move_tail(&info->state_entry, &info->cache->free_list);
368 list_move_tail(&info->state_entry, &info->cache->outgoing_list);
382 struct vdo_page_cache *cache = info->cache;
391 vdo_int_map_remove(cache->page_map, info->pbn);
396 result = vdo_int_map_put(cache->page_map, pbn, info, true, NULL);
425 * @cache: The page cache.
429 static struct page_info * __must_check find_free_page(struct vdo_page_cache *cache)
433 info = list_first_entry_or_null(&cache->free_list, struct page_info,
443 * @cache: The page cache.
448 static struct page_info * __must_check find_page(struct vdo_page_cache *cache,
451 if ((cache->last_found != NULL) && (cache->last_found->pbn == pbn))
452 return cache->last_found;
454 cache->last_found = vdo_int_map_get(cache->page_map, pbn);
455 return cache->last_found;
460 * @cache: The page cache.
469 static struct page_info * __must_check select_lru_page(struct vdo_page_cache *cache)
473 list_for_each_entry(info, &cache->lru_list, lru_entry)
494 "Requested cache page %llu in state %s is not %s",
562 * @cache: The page cache.
564 * @result: The error result to set on the cache.
569 static void set_persistent_error(struct vdo_page_cache *cache, const char *context,
574 struct vdo *vdo = cache->vdo;
582 assert_on_cache_thread(cache, __func__);
584 vdo_waitq_notify_all_waiters(&cache->free_waiters,
586 cache->waiter_count = 0;
588 for (info = cache->infos; info < cache->infos + cache->page_count; info++) {
595 * validate_completed_page() - Check that a page completion which is being freed to the cache
671 enter_zone_read_only_mode(completion->info->cache->zone, result);
683 struct vdo_page_cache *cache = info->cache;
685 assert_on_cache_thread(cache, __func__);
687 vdo_enter_read_only_mode(cache->zone->block_map->vdo, result);
688 ADD_ONCE(cache->stats.failed_reads, 1);
695 * ensure that the above work can't cause the page cache to be freed out from under us.
697 cache->outstanding_reads--;
698 check_for_drain_complete(cache->zone);
708 struct vdo_page_cache *cache = info->cache;
709 nonce_t nonce = info->cache->zone->block_map->nonce;
713 assert_on_cache_thread(cache, __func__);
737 * ensure that the above work can't cause the page cache to be freed out from under us.
739 cache->outstanding_reads--;
740 check_for_drain_complete(cache->zone);
750 struct vdo_page_cache *cache = info->cache;
752 assert_on_cache_thread(cache, __func__);
759 ADD_ONCE(cache->stats.failed_reads, 1);
770 continue_vio_after_io(vio, page_is_loaded, info->cache->zone->thread_id);
785 struct vdo_page_cache *cache = info->cache;
787 assert_io_allowed(cache);
798 cache->outstanding_reads++;
799 ADD_ONCE(cache->stats.pages_loaded, 1);
800 callback = (cache->rebuilding ? handle_rebuild_read_error : handle_load_error);
814 set_persistent_error(info->cache, "flush failed", completion->result);
823 continue_vio_after_io(vio, write_pages, info->cache->zone->thread_id);
827 static void save_pages(struct vdo_page_cache *cache)
832 if ((cache->pages_in_flush > 0) || (cache->pages_to_flush == 0))
835 assert_io_allowed(cache);
837 info = list_first_entry(&cache->outgoing_list, struct page_info, state_entry);
839 cache->pages_in_flush = cache->pages_to_flush;
840 cache->pages_to_flush = 0;
841 ADD_ONCE(cache->stats.flush_count, 1);
866 info->cache->pages_to_flush++;
867 info->cache->outstanding_writes++;
879 save_pages(info->cache);
909 struct vdo_page_cache *cache = info->cache;
911 assert_on_cache_thread(cache, __func__);
913 if (!vdo_waitq_has_waiters(&cache->free_waiters)) {
914 if (cache->stats.cache_pressure > 0) {
915 vdo_log_info("page cache pressure relieved");
916 WRITE_ONCE(cache->stats.cache_pressure, 0);
924 set_persistent_error(cache, "cannot reset page info", result);
928 oldest_waiter = vdo_waitq_get_first_waiter(&cache->free_waiters);
935 vdo_waitq_dequeue_matching_waiters(&cache->free_waiters, completion_needs_page,
937 cache->waiter_count -= vdo_waitq_num_waiters(&info->waiting);
948 * @cache: The page cache.
952 * cache is not big enough.
957 static void discard_a_page(struct vdo_page_cache *cache)
959 struct page_info *info = select_lru_page(cache);
962 report_cache_pressure(cache);
974 cache->discard_count++;
981 struct vdo_page_cache *cache = vdo_page_comp->cache;
983 cache->waiter_count++;
984 vdo_waitq_enqueue_waiter(&cache->free_waiters, &vdo_page_comp->waiter);
985 discard_a_page(cache);
989 * discard_page_if_needed() - Helper used to trigger a discard if the cache needs another free
991 * @cache: The page cache.
993 static void discard_page_if_needed(struct vdo_page_cache *cache)
995 if (cache->waiter_count > cache->discard_count)
996 discard_a_page(cache);
1000 * write_has_finished() - Inform the cache that a write has finished (possibly with an error).
1009 assert_on_cache_thread(info->cache, __func__);
1010 info->cache->outstanding_writes--;
1024 struct vdo_page_cache *cache = info->cache;
1036 ADD_ONCE(cache->stats.failed_writes, 1);
1037 set_persistent_error(cache, "cannot write page", result);
1040 discard_page_if_needed(cache);
1042 check_for_drain_complete(cache->zone);
1052 continue_vio_after_io(vio, page_is_written_out, info->cache->zone->thread_id);
1064 struct vdo_page_cache *cache = info->cache;
1077 vdo_release_recovery_journal_block_reference(cache->zone->block_map->journal,
1080 cache->zone->zone_number);
1088 ADD_ONCE(cache->stats.reclaimed, reclamations);
1091 cache->discard_count--;
1094 discard_page_if_needed(cache);
1098 check_for_drain_complete(cache->zone);
1110 struct vdo_page_cache *cache = ((struct page_info *) flush_completion->parent)->cache;
1113 * We need to cache these two values on the stack since it is possible for the last
1114 * page info to cause the page cache to get freed. Hence once we launch the last page,
1115 * it may be unsafe to dereference the cache.
1117 bool has_unflushed_pages = (cache->pages_to_flush > 0);
1118 page_count_t pages_in_flush = cache->pages_in_flush;
1120 cache->pages_in_flush = 0;
1123 list_first_entry(&cache->outgoing_list, struct page_info,
1127 if (vdo_is_read_only(info->cache->vdo)) {
1136 ADD_ONCE(info->cache->stats.pages_saved, 1);
1143 * If there are unflushed pages, the cache can't have been freed, so this call is
1146 save_pages(cache);
1162 struct vdo_page_cache *cache;
1176 cache = page_completion->cache;
1177 assert_on_cache_thread(cache, __func__);
1189 discard_page_if_needed(cache);
1219 * resident in the cache and marked busy. All callers must call vdo_release_page_completion()
1227 struct vdo_page_cache *cache = &zone->page_cache;
1231 assert_on_cache_thread(cache, __func__);
1238 .cache = cache,
1241 vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION);
1243 cache->zone->thread_id, parent);
1246 if (page_completion->writable && vdo_is_read_only(cache->vdo)) {
1252 ADD_ONCE(cache->stats.write_count, 1);
1254 ADD_ONCE(cache->stats.read_count, 1);
1256 info = find_page(cache, page_completion->pbn);
1258 /* The page is in the cache already. */
1263 ADD_ONCE(cache->stats.wait_for_page, 1);
1270 ADD_ONCE(cache->stats.found_in_cache, 1);
1272 ADD_ONCE(cache->stats.read_outgoing, 1);
1284 info = find_free_page(cache);
1286 ADD_ONCE(cache->stats.fetch_required, 1);
1292 ADD_ONCE(cache->stats.discard_required, 1);
1335 * vdo_invalidate_page_cache() - Invalidate all entries in the VDO page cache.
1336 * @cache: The page cache.
1338 * There must not be any dirty pages in the cache.
1342 int vdo_invalidate_page_cache(struct vdo_page_cache *cache)
1346 assert_on_cache_thread(cache, __func__);
1349 for (info = cache->infos; info < cache->infos + cache->page_count; info++) {
1350 int result = VDO_ASSERT(!is_dirty(info), "cache must have no dirty pages");
1357 vdo_int_map_free(vdo_forget(cache->page_map));
1358 return vdo_int_map_create(cache->page_count, &cache->page_map);
2232 * resides and cache that result in the data_vio.
2732 * @cache_size: The total block map cache size.
2841 struct vdo_page_cache *cache = &zone->page_cache;
2846 if (cache->infos != NULL) {
2849 for (info = cache->infos; info < cache->infos + cache->page_count; info++)
2853 vdo_int_map_free(vdo_forget(cache->page_map));
2854 vdo_free(vdo_forget(cache->infos));
2855 vdo_free(vdo_forget(cache->pages));
2888 result = VDO_ASSERT(cache_size > 0, "block map cache size is specified");
3278 add_to_dirty_lists(info->cache->zone, &info->state_entry,