163 #define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset]))
169 #define SHADOW_STACK_TASK_VARS(ret_stack) \  argument
170 	((unsigned long *)(&(ret_stack)[SHADOW_STACK_OFFSET - FGRAPH_ARRAY_SIZE]))
251 	return t->ret_stack[offset];  in get_fgraph_entry()
257 	return __get_offset(t->ret_stack[offset]);  in get_frame_offset()
264 	return (t->ret_stack[offset] >> FGRAPH_INDEX_SHIFT) & FGRAPH_INDEX_MASK;  in get_bitmap_bits()
271 	t->ret_stack[offset] = (bitmap << FGRAPH_INDEX_SHIFT) |  in set_bitmap()
278 	unsigned long val = t->ret_stack[offset];  in get_data_type_data()
283 	return (void *)&t->ret_stack[offset];  in get_data_type_data()
309 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);  in ret_stack_set_task_var()
317 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(t->ret_stack);  in ret_stack_get_task_var()
322 static void ret_stack_init_task_vars(unsigned long *ret_stack)  in ret_stack_init_task_vars()  argument
324 	unsigned long *gvals = SHADOW_STACK_TASK_VARS(ret_stack);  in ret_stack_init_task_vars()
360 	data = &current->ret_stack[curr_ret_stack];  in fgraph_reserve_data()
369 	current->ret_stack[curr_ret_stack - 1] = val;  in fgraph_reserve_data()
375 	current->ret_stack[curr_ret_stack - 1] = val;  in fgraph_reserve_data()
466 	struct ftrace_ret_stack *ret_stack = NULL;  in fgraph_retrieve_parent_data()  local
476 		ret_stack = get_ret_stack(current, offset, &next_offset);  in fgraph_retrieve_parent_data()
477 		if (!ret_stack || --depth < 0)  in fgraph_retrieve_parent_data()
482 	if (!ret_stack)  in fgraph_retrieve_parent_data()
563 	struct ftrace_ret_stack *ret_stack;  in ftrace_push_return_trace()  local
570 	if (!current->ret_stack)  in ftrace_push_return_trace()
594 	ret_stack = RET_STACK(current, offset);  in ftrace_push_return_trace()
598 	current->ret_stack[offset] = val;  in ftrace_push_return_trace()
599 	ret_stack->ret = ret;  in ftrace_push_return_trace()
622 	current->ret_stack[offset] = val;  in ftrace_push_return_trace()
624 	ret_stack->ret = ret;  in ftrace_push_return_trace()
625 	ret_stack->func = func;  in ftrace_push_return_trace()
627 	ret_stack->fp = frame_pointer;  in ftrace_push_return_trace()
629 	ret_stack->retp = retp;  in ftrace_push_return_trace()
722 	struct ftrace_ret_stack *ret_stack;  in ftrace_pop_return_trace()  local
724 	ret_stack = get_ret_stack(current, current->curr_ret_stack, offset);  in ftrace_pop_return_trace()
726 	if (unlikely(!ret_stack)) {  in ftrace_pop_return_trace()
750 	if (unlikely(ret_stack->fp != frame_pointer)) {  in ftrace_pop_return_trace()
754 		     ret_stack->fp,  in ftrace_pop_return_trace()
756 		     (void *)ret_stack->func,  in ftrace_pop_return_trace()
757 		     ret_stack->ret);  in ftrace_pop_return_trace()
764 	*ret = ret_stack->ret;  in ftrace_pop_return_trace()
765 	trace->func = ret_stack->func;  in ftrace_pop_return_trace()
775 	return ret_stack;  in ftrace_pop_return_trace()
810 	struct ftrace_ret_stack *ret_stack;  in __ftrace_return_to_handler()  local
818 	ret_stack = ftrace_pop_return_trace(&trace, &ret, frame_pointer, &offset);  in __ftrace_return_to_handler()
820 	if (unlikely(!ret_stack)) {  in __ftrace_return_to_handler()
907 	struct ftrace_ret_stack *ret_stack = NULL;  in ftrace_graph_get_ret_stack()  local
914 		ret_stack = get_ret_stack(task, offset, &offset);  in ftrace_graph_get_ret_stack()
915 	} while (ret_stack && --idx >= 0);  in ftrace_graph_get_ret_stack()
917 	return ret_stack;  in ftrace_graph_get_ret_stack()
930 	struct ftrace_ret_stack *ret_stack = NULL;  in ftrace_graph_top_ret_addr()  local
937 		ret_stack = get_ret_stack(task, offset, &offset);  in ftrace_graph_top_ret_addr()
938 	} while (ret_stack && ret_stack->ret == return_handler);  in ftrace_graph_top_ret_addr()
940 	return ret_stack ? ret_stack->ret : 0;  in ftrace_graph_top_ret_addr()
966 	struct ftrace_ret_stack *ret_stack;  in ftrace_graph_ret_addr()  local
978 		ret_stack = get_ret_stack(task, i, &i);  in ftrace_graph_ret_addr()
979 		if (!ret_stack)  in ftrace_graph_ret_addr()
989 		if (ret_stack->retp == retp &&  in ftrace_graph_ret_addr()
990 		    ret_stack->ret != return_handler) {  in ftrace_graph_ret_addr()
992 			return ret_stack->ret;  in ftrace_graph_ret_addr()
1063 		if (t->ret_stack == NULL) {  in alloc_retstack_tasklist()
1070 			t->ret_stack = ret_stack_list[start++];  in alloc_retstack_tasklist()
1111 graph_init_task(struct task_struct *t, unsigned long *ret_stack)  in graph_init_task()  argument
1114 	ret_stack_init_task_vars(ret_stack);  in graph_init_task()
1120 	t->ret_stack = ret_stack;  in graph_init_task()
1135 	if (t->ret_stack)  in ftrace_graph_init_idle_task()
1136 		WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));  in ftrace_graph_init_idle_task()
1139 		unsigned long *ret_stack;  in ftrace_graph_init_idle_task()  local
1144 		ret_stack = per_cpu(idle_ret_stack, cpu);  in ftrace_graph_init_idle_task()
1145 		if (!ret_stack) {  in ftrace_graph_init_idle_task()
1146 			ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);  in ftrace_graph_init_idle_task()
1147 			if (!ret_stack)  in ftrace_graph_init_idle_task()
1149 			per_cpu(idle_ret_stack, cpu) = ret_stack;  in ftrace_graph_init_idle_task()
1151 		graph_init_task(t, ret_stack);  in ftrace_graph_init_idle_task()
1159 	t->ret_stack = NULL;  in ftrace_graph_init_task()
1164 		unsigned long *ret_stack;  in ftrace_graph_init_task()  local
1169 		ret_stack = kmem_cache_alloc(fgraph_stack_cachep, GFP_KERNEL);  in ftrace_graph_init_task()
1170 		if (!ret_stack)  in ftrace_graph_init_task()
1172 		graph_init_task(t, ret_stack);  in ftrace_graph_init_task()
1178 	unsigned long *ret_stack = t->ret_stack;  in ftrace_graph_exit_task()  local
1180 	t->ret_stack = NULL;  in ftrace_graph_exit_task()
1184 	if (ret_stack) {  in ftrace_graph_exit_task()
1187 		kmem_cache_free(fgraph_stack_cachep, ret_stack);  in ftrace_graph_exit_task()
1245 		if (!idle_task(cpu)->ret_stack)  in start_graph_tracing()
1270 		if (idle_task(cpu)->ret_stack)  in init_task_vars()
1276 		if (t->ret_stack)  in init_task_vars()
1321 	if (!idle_task(cpu)->ret_stack)  in fgraph_cpu_init()