43  * request wants to add a chapter to the sparse cache, it sends a barrier message to each zone
89 	struct uds_request *request;  in launch_zone_message()  local
91 	result = vdo_allocate(1, struct uds_request, __func__, &request);  in launch_zone_message()
95 	request->index = index;  in launch_zone_message()
96 	request->unbatched = true;  in launch_zone_message()
97 	request->zone_number = zone;  in launch_zone_message()
98 	request->zone_message = message;  in launch_zone_message()
100 	uds_enqueue_request(request, STAGE_MESSAGE);  in launch_zone_message()
120  * Determine whether this request should trigger a sparse cache barrier message to change the
124 static u64 triage_index_request(struct uds_index *index, struct uds_request *request)  in triage_index_request()  argument
130 						       &request->record_name);  in triage_index_request()
134 	zone = index->zones[request->zone_number];  in triage_index_request()
152 					       struct uds_request *request)  in simulate_index_zone_barrier_message()  argument
160 	sparse_virtual_chapter = triage_index_request(zone->index, request);  in simulate_index_zone_barrier_message()
167 /* This is the request processing function for the triage queue. */
168 static void triage_request(struct uds_request *request)  in triage_request()  argument
170 	struct uds_index *index = request->index;  in triage_request()
171 	u64 sparse_virtual_chapter = triage_index_request(index, request);  in triage_request()
176 	uds_enqueue_request(request, STAGE_INDEX);  in triage_request()
302 static int dispatch_index_zone_control_request(struct uds_request *request)  in dispatch_index_zone_control_request()  argument
304 	struct uds_zone_message *message = &request->zone_message;  in dispatch_index_zone_control_request()
305 	struct index_zone *zone = request->index->zones[request->zone_number];  in dispatch_index_zone_control_request()
320 static void set_request_location(struct uds_request *request,  in set_request_location()  argument
323 	request->location = new_location;  in set_request_location()
324 	request->found = ((new_location == UDS_LOCATION_IN_OPEN_CHAPTER) ||  in set_request_location()
329 static void set_chapter_location(struct uds_request *request,  in set_chapter_location()  argument
332 	request->found = true;  in set_chapter_location()
334 		request->location = UDS_LOCATION_IN_OPEN_CHAPTER;  in set_chapter_location()
336 		request->location = UDS_LOCATION_IN_SPARSE;  in set_chapter_location()
338 		request->location = UDS_LOCATION_IN_DENSE;  in set_chapter_location()
341 static int search_sparse_cache_in_zone(struct index_zone *zone, struct uds_request *request,  in search_sparse_cache_in_zone()  argument
349 	result = uds_search_sparse_cache(zone, &request->record_name, &virtual_chapter,  in search_sparse_cache_in_zone()
354 	request->virtual_chapter = virtual_chapter;  in search_sparse_cache_in_zone()
357 	return uds_search_cached_record_page(volume, request, chapter,  in search_sparse_cache_in_zone()
361 static int get_record_from_zone(struct index_zone *zone, struct uds_request *request,  in get_record_from_zone()  argument
366 	if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) {  in get_record_from_zone()
369 	} else if (request->location == UDS_LOCATION_UNAVAILABLE) {  in get_record_from_zone()
374 	if (request->virtual_chapter == zone->newest_virtual_chapter) {  in get_record_from_zone()
375 		uds_search_open_chapter(zone->open_chapter, &request->record_name,  in get_record_from_zone()
376 					&request->old_metadata, found);  in get_record_from_zone()
381 	    (request->virtual_chapter == (zone->newest_virtual_chapter - 1)) &&  in get_record_from_zone()
383 		uds_search_open_chapter(zone->writing_chapter, &request->record_name,  in get_record_from_zone()
384 					&request->old_metadata, found);  in get_record_from_zone()
389 	if (is_zone_chapter_sparse(zone, request->virtual_chapter) &&  in get_record_from_zone()
390 	    uds_sparse_cache_contains(volume->sparse_cache, request->virtual_chapter,  in get_record_from_zone()
391 				      request->zone_number))  in get_record_from_zone()
392 		return search_sparse_cache_in_zone(zone, request,  in get_record_from_zone()
393 						   request->virtual_chapter, found);  in get_record_from_zone()
395 	return uds_search_volume_page_cache(volume, request, found);  in get_record_from_zone()
398 static int put_record_in_zone(struct index_zone *zone, struct uds_request *request,  in put_record_in_zone()  argument
403 	remaining = uds_put_open_chapter(zone->open_chapter, &request->record_name,  in put_record_in_zone()
411 static int search_index_zone(struct index_zone *zone, struct uds_request *request)  in search_index_zone()  argument
420 					     &request->record_name, &record);  in search_index_zone()
425 		if (request->requeued && request->virtual_chapter != record.virtual_chapter)  in search_index_zone()
426 			set_request_location(request, UDS_LOCATION_UNKNOWN);  in search_index_zone()
428 		request->virtual_chapter = record.virtual_chapter;  in search_index_zone()
429 		result = get_record_from_zone(zone, request, &found);  in search_index_zone()
435 		set_chapter_location(request, zone, record.virtual_chapter);  in search_index_zone()
446 		if ((request->type == UDS_QUERY_NO_UPDATE) ||  in search_index_zone()
447 		    ((request->type == UDS_QUERY) && overflow_record)) {  in search_index_zone()
459 		} else if (request->type != UDS_UPDATE) {  in search_index_zone()
469 		if (request->location == UDS_LOCATION_RECORD_PAGE_LOOKUP) {  in search_index_zone()
471 		} else if (request->location == UDS_LOCATION_UNAVAILABLE) {  in search_index_zone()
475 						       &request->record_name)) {  in search_index_zone()
476 			result = search_sparse_cache_in_zone(zone, request, NO_CHAPTER,  in search_index_zone()
483 			set_request_location(request, UDS_LOCATION_IN_SPARSE);  in search_index_zone()
485 		if ((request->type == UDS_QUERY_NO_UPDATE) ||  in search_index_zone()
486 		    ((request->type == UDS_QUERY) && !found)) {  in search_index_zone()
509 	if (!found || (request->type == UDS_UPDATE)) {  in search_index_zone()
511 		metadata = &request->new_metadata;  in search_index_zone()
514 		metadata = &request->old_metadata;  in search_index_zone()
517 	return put_record_in_zone(zone, request, metadata);  in search_index_zone()
520 static int remove_from_index_zone(struct index_zone *zone, struct uds_request *request)  in remove_from_index_zone()  argument
526 					     &request->record_name, &record);  in remove_from_index_zone()
533 	/* If the request was requeued, check whether the saved state is still valid. */  in remove_from_index_zone()
536 		set_chapter_location(request, zone, record.virtual_chapter);  in remove_from_index_zone()
541 		if (request->requeued && request->virtual_chapter != record.virtual_chapter)  in remove_from_index_zone()
542 			set_request_location(request, UDS_LOCATION_UNKNOWN);  in remove_from_index_zone()
544 		request->virtual_chapter = record.virtual_chapter;  in remove_from_index_zone()
545 		result = get_record_from_zone(zone, request, &found);  in remove_from_index_zone()
555 	set_chapter_location(request, zone, record.virtual_chapter);  in remove_from_index_zone()
570 	if (request->location == UDS_LOCATION_IN_OPEN_CHAPTER)  in remove_from_index_zone()
571 		uds_remove_from_open_chapter(zone->open_chapter, &request->record_name);  in remove_from_index_zone()
576 static int dispatch_index_request(struct uds_index *index, struct uds_request *request)  in dispatch_index_request()  argument
579 	struct index_zone *zone = index->zones[request->zone_number];  in dispatch_index_request()
581 	if (!request->requeued) {  in dispatch_index_request()
582 		result = simulate_index_zone_barrier_message(zone, request);  in dispatch_index_request()
587 	switch (request->type) {  in dispatch_index_request()
592 		result = search_index_zone(zone, request);  in dispatch_index_request()
596 		result = remove_from_index_zone(zone, request);  in dispatch_index_request()
601 						  "invalid request type: %d",  in dispatch_index_request()
602 						  request->type);  in dispatch_index_request()
609 /* This is the request processing function invoked by each zone's thread. */
610 static void execute_zone_request(struct uds_request *request)  in execute_zone_request()  argument
613 	struct uds_index *index = request->index;  in execute_zone_request()
615 	if (request->zone_message.type != UDS_MESSAGE_NONE) {  in execute_zone_request()
616 		result = dispatch_index_zone_control_request(request);  in execute_zone_request()
619 					       request->zone_message.type);  in execute_zone_request()
623 		vdo_free(vdo_forget(request));  in execute_zone_request()
628 	if (request->requeued && (request->status != UDS_SUCCESS)) {  in execute_zone_request()
629 		set_request_location(request, UDS_LOCATION_UNAVAILABLE);  in execute_zone_request()
630 		index->callback(request);  in execute_zone_request()
634 	result = dispatch_index_request(index, request);  in execute_zone_request()
636 		/* The request has been requeued so don't let it complete. */  in execute_zone_request()
640 	if (!request->found)  in execute_zone_request()
641 		set_request_location(request, UDS_LOCATION_UNAVAILABLE);  in execute_zone_request()
643 	request->status = result;  in execute_zone_request()
644 	index->callback(request);  in execute_zone_request()
1356 void uds_enqueue_request(struct uds_request *request, enum request_stage stage)  in uds_enqueue_request()  argument
1358 	struct uds_index *index = request->index;  in uds_enqueue_request()
1371 		request->zone_number =  in uds_enqueue_request()
1372 			uds_get_volume_index_zone(index->volume_index, &request->record_name);  in uds_enqueue_request()
1376 		queue = index->zone_queues[request->zone_number];  in uds_enqueue_request()
1384 	uds_request_queue_enqueue(queue, request);  in uds_enqueue_request()