1 // SPDX-License-Identifier: GPL-2.0-only
6 #include "index-session.h"
11 #include "memory-alloc.h"
12 #include "time-utils.h"
14 #include "funnel-requestqueue.h"
16 #include "index-layout.h"
21  * index through a set of descriptive flags. The request_mutex must be notified whenever a
22  * non-transient state flag is cleared. The request_mutex is also used to count the number of
65 	mutex_lock(&index_session->request_mutex);  in release_index_session()
66 	if (--index_session->request_count == 0)  in release_index_session()
67 		uds_broadcast_cond(&index_session->request_cond);  in release_index_session()
68 	mutex_unlock(&index_session->request_mutex);  in release_index_session()
80 	mutex_lock(&index_session->request_mutex);  in get_index_session()
81 	index_session->request_count++;  in get_index_session()
82 	state = index_session->state;  in get_index_session()
83 	mutex_unlock(&index_session->request_mutex);  in get_index_session()
92 		result = -EBUSY;  in get_index_session()
105 	if (request->callback == NULL) {  in uds_launch_request()
107 		return -EINVAL;  in uds_launch_request()
110 	switch (request->type) {  in uds_launch_request()
119 		return -EINVAL;  in uds_launch_request()
123 	memset(&request->internal, 0, sizeof(request->internal));  in uds_launch_request()
125 	result = get_index_session(request->session);  in uds_launch_request()
129 	request->found = false;  in uds_launch_request()
130 	request->unbatched = false;  in uds_launch_request()
131 	request->index = request->session->index;  in uds_launch_request()
139 	if (request->status != UDS_SUCCESS) {  in enter_callback_stage()
141 		mutex_lock(&request->session->request_mutex);  in enter_callback_stage()
142 		request->session->state |= IS_FLAG_DISABLED;  in enter_callback_stage()
143 		mutex_unlock(&request->session->request_mutex);  in enter_callback_stage()
146 	uds_request_queue_enqueue(request->session->callback_queue, request);  in enter_callback_stage()
156 	struct session_stats *session_stats = &request->session->stats;  in update_session_stats()
158 	count_once(&session_stats->requests);  in update_session_stats()
160 	switch (request->type) {  in update_session_stats()
162 		if (request->found)  in update_session_stats()
163 			count_once(&session_stats->posts_found);  in update_session_stats()
165 			count_once(&session_stats->posts_not_found);  in update_session_stats()
167 		if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER)  in update_session_stats()
168 			count_once(&session_stats->posts_found_open_chapter);  in update_session_stats()
169 		else if (request->location == UDS_LOCATION_IN_DENSE)  in update_session_stats()
170 			count_once(&session_stats->posts_found_dense);  in update_session_stats()
171 		else if (request->location == UDS_LOCATION_IN_SPARSE)  in update_session_stats()
172 			count_once(&session_stats->posts_found_sparse);  in update_session_stats()
176 		if (request->found)  in update_session_stats()
177 			count_once(&session_stats->updates_found);  in update_session_stats()
179 			count_once(&session_stats->updates_not_found);  in update_session_stats()
183 		if (request->found)  in update_session_stats()
184 			count_once(&session_stats->deletions_found);  in update_session_stats()
186 			count_once(&session_stats->deletions_not_found);  in update_session_stats()
191 		if (request->found)  in update_session_stats()
192 			count_once(&session_stats->queries_found);  in update_session_stats()
194 			count_once(&session_stats->queries_not_found);  in update_session_stats()
198 		request->status = VDO_ASSERT(false, "unknown request type: %d",  in update_session_stats()
199 					     request->type);  in update_session_stats()
205 	struct uds_index_session *index_session = request->session;  in handle_callbacks()
207 	if (request->status == UDS_SUCCESS)  in handle_callbacks()
210 	request->status = uds_status_to_errno(request->status);  in handle_callbacks()
211 	request->callback(request);  in handle_callbacks()
224 	mutex_init(&session->request_mutex);  in make_empty_index_session()
225 	uds_init_cond(&session->request_cond);  in make_empty_index_session()
226 	mutex_init(&session->load_context.mutex);  in make_empty_index_session()
227 	uds_init_cond(&session->load_context.cond);  in make_empty_index_session()
230 					&session->callback_queue);  in make_empty_index_session()
244 		return -EINVAL;  in uds_create_index_session()
254 	mutex_lock(&index_session->request_mutex);  in start_loading_index_session()
255 	if (index_session->state & IS_FLAG_SUSPENDED) {  in start_loading_index_session()
257 		result = -EBUSY;  in start_loading_index_session()
258 	} else if (index_session->state != 0) {  in start_loading_index_session()
260 		result = -EBUSY;  in start_loading_index_session()
262 		index_session->state |= IS_FLAG_LOADING;  in start_loading_index_session()
265 	mutex_unlock(&index_session->request_mutex);  in start_loading_index_session()
272 	mutex_lock(&index_session->request_mutex);  in finish_loading_index_session()
273 	index_session->state &= ~IS_FLAG_LOADING;  in finish_loading_index_session()
275 		index_session->state |= IS_FLAG_LOADED;  in finish_loading_index_session()
277 	uds_broadcast_cond(&index_session->request_cond);  in finish_loading_index_session()
278 	mutex_unlock(&index_session->request_mutex);  in finish_loading_index_session()
287 	result = uds_make_configuration(&index_session->parameters, &config);  in initialize_index_session()
293 	memset(&index_session->stats, 0, sizeof(index_session->stats));  in initialize_index_session()
294 	result = uds_make_index(config, open_type, &index_session->load_context,  in initialize_index_session()
295 				enter_callback_stage, &index_session->index);  in initialize_index_session()
332 		return -EINVAL;  in uds_open_index()
334 	if (parameters->bdev == NULL) {  in uds_open_index()
336 		return -EINVAL;  in uds_open_index()
340 		return -EINVAL;  in uds_open_index()
347 	session->parameters = *parameters;  in uds_open_index()
348 	format_dev_t(name, parameters->bdev->bd_dev);  in uds_open_index()
362 	mutex_lock(&index_session->request_mutex);  in wait_for_no_requests_in_progress()
363 	while (index_session->request_count > 0) {  in wait_for_no_requests_in_progress()
364 		uds_wait_cond(&index_session->request_cond,  in wait_for_no_requests_in_progress()
365 			      &index_session->request_mutex);  in wait_for_no_requests_in_progress()
367 	mutex_unlock(&index_session->request_mutex);  in wait_for_no_requests_in_progress()
373 	return uds_save_index(index_session->index);  in save_index()
378 	mutex_lock(&session->load_context.mutex);  in suspend_rebuild()
379 	switch (session->load_context.status) {  in suspend_rebuild()
381 		session->load_context.status = INDEX_SUSPENDING;  in suspend_rebuild()
384 		while ((session->load_context.status != INDEX_SUSPENDED) &&  in suspend_rebuild()
385 		       (session->load_context.status != INDEX_READY)) {  in suspend_rebuild()
386 			uds_wait_cond(&session->load_context.cond,  in suspend_rebuild()
387 				      &session->load_context.mutex);  in suspend_rebuild()
402 				    session->load_context.status);  in suspend_rebuild()
405 	mutex_unlock(&session->load_context.mutex);  in suspend_rebuild()
419 	mutex_lock(&session->request_mutex);  in uds_suspend_index_session()
420 	while (session->state & IS_FLAG_CLOSING)  in uds_suspend_index_session()
421 		uds_wait_cond(&session->request_cond, &session->request_mutex);  in uds_suspend_index_session()
423 	if ((session->state & IS_FLAG_WAITING) || (session->state & IS_FLAG_DESTROYING)) {  in uds_suspend_index_session()
426 		result = -EBUSY;  in uds_suspend_index_session()
427 	} else if (session->state & IS_FLAG_SUSPENDED) {  in uds_suspend_index_session()
429 	} else if (session->state & IS_FLAG_LOADING) {  in uds_suspend_index_session()
430 		session->state |= IS_FLAG_WAITING;  in uds_suspend_index_session()
432 	} else if (session->state & IS_FLAG_LOADED) {  in uds_suspend_index_session()
433 		session->state |= IS_FLAG_WAITING;  in uds_suspend_index_session()
436 		session->state |= IS_FLAG_SUSPENDED;  in uds_suspend_index_session()
437 		uds_broadcast_cond(&session->request_cond);  in uds_suspend_index_session()
439 	mutex_unlock(&session->request_mutex);  in uds_suspend_index_session()
451 	mutex_lock(&session->request_mutex);  in uds_suspend_index_session()
452 	session->state &= ~IS_FLAG_WAITING;  in uds_suspend_index_session()
453 	session->state |= IS_FLAG_SUSPENDED;  in uds_suspend_index_session()
454 	uds_broadcast_cond(&session->request_cond);  in uds_suspend_index_session()
455 	mutex_unlock(&session->request_mutex);  in uds_suspend_index_session()
463 	result = uds_replace_index_storage(session->index, bdev);  in replace_device()
467 	session->parameters.bdev = bdev;  in replace_device()
482 	mutex_lock(&session->request_mutex);  in uds_resume_index_session()
483 	if (session->state & IS_FLAG_WAITING) {  in uds_resume_index_session()
486 		result = -EBUSY;  in uds_resume_index_session()
487 	} else if (!(session->state & IS_FLAG_SUSPENDED)) {  in uds_resume_index_session()
492 		session->state |= IS_FLAG_WAITING;  in uds_resume_index_session()
493 		if (session->state & IS_FLAG_LOADING)  in uds_resume_index_session()
496 	mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
501 	if ((session->index != NULL) && (bdev != session->parameters.bdev)) {  in uds_resume_index_session()
504 			mutex_lock(&session->request_mutex);  in uds_resume_index_session()
505 			session->state &= ~IS_FLAG_WAITING;  in uds_resume_index_session()
506 			uds_broadcast_cond(&session->request_cond);  in uds_resume_index_session()
507 			mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
513 		mutex_lock(&session->load_context.mutex);  in uds_resume_index_session()
514 		switch (session->load_context.status) {  in uds_resume_index_session()
516 			session->load_context.status = INDEX_OPENING;  in uds_resume_index_session()
518 			uds_broadcast_cond(&session->load_context.cond);  in uds_resume_index_session()
531 					    session->load_context.status);  in uds_resume_index_session()
534 		mutex_unlock(&session->load_context.mutex);  in uds_resume_index_session()
537 	mutex_lock(&session->request_mutex);  in uds_resume_index_session()
538 	session->state &= ~IS_FLAG_WAITING;  in uds_resume_index_session()
539 	session->state &= ~IS_FLAG_SUSPENDED;  in uds_resume_index_session()
540 	uds_broadcast_cond(&session->request_cond);  in uds_resume_index_session()
541 	mutex_unlock(&session->request_mutex);  in uds_resume_index_session()
549 	struct uds_index *index = index_session->index;  in save_and_free_index()
554 	mutex_lock(&index_session->request_mutex);  in save_and_free_index()
555 	suspended = (index_session->state & IS_FLAG_SUSPENDED);  in save_and_free_index()
556 	mutex_unlock(&index_session->request_mutex);  in save_and_free_index()
565 	index_session->index = NULL;  in save_and_free_index()
571 	mutex_lock(&index_session->load_context.mutex);  in save_and_free_index()
572 	index_session->load_context.status = INDEX_OPENING;  in save_and_free_index()
573 	mutex_unlock(&index_session->load_context.mutex);  in save_and_free_index()
575 	mutex_lock(&index_session->request_mutex);  in save_and_free_index()
577 	index_session->state &= IS_FLAG_SUSPENDED;  in save_and_free_index()
578 	mutex_unlock(&index_session->request_mutex);  in save_and_free_index()
589 	mutex_lock(&index_session->request_mutex);  in uds_close_index()
590 	while ((index_session->state & IS_FLAG_WAITING) ||  in uds_close_index()
591 	       (index_session->state & IS_FLAG_CLOSING)) {  in uds_close_index()
592 		uds_wait_cond(&index_session->request_cond,  in uds_close_index()
593 			      &index_session->request_mutex);  in uds_close_index()
596 	if (index_session->state & IS_FLAG_SUSPENDED) {  in uds_close_index()
598 		result = -EBUSY;  in uds_close_index()
599 	} else if ((index_session->state & IS_FLAG_DESTROYING) ||  in uds_close_index()
600 		   !(index_session->state & IS_FLAG_LOADED)) {  in uds_close_index()
604 		index_session->state |= IS_FLAG_CLOSING;  in uds_close_index()
606 	mutex_unlock(&index_session->request_mutex);  in uds_close_index()
615 	mutex_lock(&index_session->request_mutex);  in uds_close_index()
616 	index_session->state &= ~IS_FLAG_CLOSING;  in uds_close_index()
617 	uds_broadcast_cond(&index_session->request_cond);  in uds_close_index()
618 	mutex_unlock(&index_session->request_mutex);  in uds_close_index()
631 	mutex_lock(&index_session->request_mutex);  in uds_destroy_index_session()
632 	while ((index_session->state & IS_FLAG_WAITING) ||  in uds_destroy_index_session()
633 	       (index_session->state & IS_FLAG_CLOSING)) {  in uds_destroy_index_session()
634 		uds_wait_cond(&index_session->request_cond,  in uds_destroy_index_session()
635 			      &index_session->request_mutex);  in uds_destroy_index_session()
638 	if (index_session->state & IS_FLAG_DESTROYING) {  in uds_destroy_index_session()
639 		mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
641 		return -EBUSY;  in uds_destroy_index_session()
644 	index_session->state |= IS_FLAG_DESTROYING;  in uds_destroy_index_session()
645 	load_pending = ((index_session->state & IS_FLAG_LOADING) &&  in uds_destroy_index_session()
646 			(index_session->state & IS_FLAG_SUSPENDED));  in uds_destroy_index_session()
647 	mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
651 		mutex_lock(&index_session->load_context.mutex);  in uds_destroy_index_session()
652 		if (index_session->load_context.status == INDEX_SUSPENDED) {  in uds_destroy_index_session()
653 			index_session->load_context.status = INDEX_FREEING;  in uds_destroy_index_session()
654 			uds_broadcast_cond(&index_session->load_context.cond);  in uds_destroy_index_session()
656 		mutex_unlock(&index_session->load_context.mutex);  in uds_destroy_index_session()
659 		mutex_lock(&index_session->request_mutex);  in uds_destroy_index_session()
660 		while (index_session->state & IS_FLAG_LOADING) {  in uds_destroy_index_session()
661 			uds_wait_cond(&index_session->request_cond,  in uds_destroy_index_session()
662 				      &index_session->request_mutex);  in uds_destroy_index_session()
664 		mutex_unlock(&index_session->request_mutex);  in uds_destroy_index_session()
669 	uds_request_queue_finish(index_session->callback_queue);  in uds_destroy_index_session()
670 	index_session->callback_queue = NULL;  in uds_destroy_index_session()
680 	uds_wait_for_idle_index(index_session->index);  in uds_flush_index_session()
684 /* Statistics collection is intended to be thread-safe. */
688 	const struct session_stats *session_stats = &index_session->stats;  in collect_stats()
690 	stats->current_time = ktime_to_seconds(current_time_ns(CLOCK_REALTIME));  in collect_stats()
691 	stats->posts_found = READ_ONCE(session_stats->posts_found);  in collect_stats()
692 	stats->in_memory_posts_found = READ_ONCE(session_stats->posts_found_open_chapter);  in collect_stats()
693 	stats->dense_posts_found = READ_ONCE(session_stats->posts_found_dense);  in collect_stats()
694 	stats->sparse_posts_found = READ_ONCE(session_stats->posts_found_sparse);  in collect_stats()
695 	stats->posts_not_found = READ_ONCE(session_stats->posts_not_found);  in collect_stats()
696 	stats->updates_found = READ_ONCE(session_stats->updates_found);  in collect_stats()
697 	stats->updates_not_found = READ_ONCE(session_stats->updates_not_found);  in collect_stats()
698 	stats->deletions_found = READ_ONCE(session_stats->deletions_found);  in collect_stats()
699 	stats->deletions_not_found = READ_ONCE(session_stats->deletions_not_found);  in collect_stats()
700 	stats->queries_found = READ_ONCE(session_stats->queries_found);  in collect_stats()
701 	stats->queries_not_found = READ_ONCE(session_stats->queries_not_found);  in collect_stats()
702 	stats->requests = READ_ONCE(session_stats->requests);  in collect_stats()
710 		return -EINVAL;  in uds_get_index_session_stats()
714 	if (index_session->index != NULL) {  in uds_get_index_session_stats()
715 		uds_get_index_stats(index_session->index, stats);  in uds_get_index_session_stats()
717 		stats->entries_indexed = 0;  in uds_get_index_session_stats()
718 		stats->memory_used = 0;  in uds_get_index_session_stats()
719 		stats->collisions = 0;  in uds_get_index_session_stats()
720 		stats->entries_discarded = 0;  in uds_get_index_session_stats()
730 	prepare_to_wait(&cv->wait_queue, &__wait, TASK_IDLE);  in uds_wait_cond()
733 	finish_wait(&cv->wait_queue, &__wait);  in uds_wait_cond()