42 static inline void trace_lock_elapsed_time_start(struct f2fs_rwsem *sem,  in trace_lock_elapsed_time_start()  argument
52 static inline void trace_lock_elapsed_time_end(struct f2fs_rwsem *sem,  in trace_lock_elapsed_time_end()  argument
66 	if (time_to_inject(sem->sbi, FAULT_LOCK_TIMEOUT))  in trace_lock_elapsed_time_end()
72 	if (total_time <= sem->sbi->max_lock_elapsed_time)  in trace_lock_elapsed_time_end()
88 	trace_f2fs_lock_elapsed_time(sem->sbi, sem->name, is_write, current,  in trace_lock_elapsed_time_end()
93 static bool need_uplift_priority(struct f2fs_rwsem *sem, bool is_write)  in need_uplift_priority()  argument
95 	if (!(sem->sbi->adjust_lock_priority & BIT(sem->name - 1)))  in need_uplift_priority()
98 	switch (sem->name) {  in need_uplift_priority()
113 		f2fs_bug_on(sem->sbi, 1);  in need_uplift_priority()
118 static void uplift_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc,  in uplift_priority()  argument
122 	if (!sem->sbi->adjust_lock_priority)  in uplift_priority()
126 	if (!need_uplift_priority(sem, is_write))  in uplift_priority()
129 	lc->new_nice = PRIO_TO_NICE(sem->sbi->lock_duration_priority);  in uplift_priority()
135 	trace_f2fs_priority_uplift(sem->sbi, sem->name, is_write, current,  in uplift_priority()
139 static void restore_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc,  in restore_priority()  argument
149 	trace_f2fs_priority_restore(sem->sbi, sem->name, is_write, current,  in restore_priority()
153 void f2fs_down_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_down_read_trace()  argument
155 	uplift_priority(sem, lc, false);  in f2fs_down_read_trace()
156 	f2fs_down_read(sem);  in f2fs_down_read_trace()
157 	trace_lock_elapsed_time_start(sem, lc);  in f2fs_down_read_trace()
160 int f2fs_down_read_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_down_read_trylock_trace()  argument
162 	uplift_priority(sem, lc, false);  in f2fs_down_read_trylock_trace()
163 	if (!f2fs_down_read_trylock(sem)) {  in f2fs_down_read_trylock_trace()
164 		restore_priority(sem, lc, false);  in f2fs_down_read_trylock_trace()
167 	trace_lock_elapsed_time_start(sem, lc);  in f2fs_down_read_trylock_trace()
171 void f2fs_up_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_up_read_trace()  argument
173 	f2fs_up_read(sem);  in f2fs_up_read_trace()
174 	restore_priority(sem, lc, false);  in f2fs_up_read_trace()
175 	trace_lock_elapsed_time_end(sem, lc, false);  in f2fs_up_read_trace()
178 void f2fs_down_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_down_write_trace()  argument
180 	uplift_priority(sem, lc, true);  in f2fs_down_write_trace()
181 	f2fs_down_write(sem);  in f2fs_down_write_trace()
182 	trace_lock_elapsed_time_start(sem, lc);  in f2fs_down_write_trace()
185 int f2fs_down_write_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_down_write_trylock_trace()  argument
187 	uplift_priority(sem, lc, true);  in f2fs_down_write_trylock_trace()
188 	if (!f2fs_down_write_trylock(sem)) {  in f2fs_down_write_trylock_trace()
189 		restore_priority(sem, lc, true);  in f2fs_down_write_trylock_trace()
192 	trace_lock_elapsed_time_start(sem, lc);  in f2fs_down_write_trylock_trace()
196 void f2fs_up_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc)  in f2fs_up_write_trace()  argument
198 	f2fs_up_write(sem);  in f2fs_up_write_trace()
199 	restore_priority(sem, lc, true);  in f2fs_up_write_trace()
200 	trace_lock_elapsed_time_end(sem, lc, true);  in f2fs_up_write_trace()