1 // SPDX-License-Identifier: LGPL-2.1
44 static __thread __attribute__((tls_model("initial-exec")))
49 static __thread __attribute__((tls_model("initial-exec"), unused))
66 	"mov asm_loop_cnt_" #n ", %%" INJECT_ASM_REG "\n\t" \
67 	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
68 	"jz 333f\n\t" \
69 	"222:\n\t" \
70 	"dec %%" INJECT_ASM_REG "\n\t" \
71 	"jnz 222b\n\t" \
72 	"333:\n\t"
84 	"lea asm_loop_cnt_" #n "(%%rip), %%" INJECT_ASM_REG_P "\n\t" \
85 	"mov (%%" INJECT_ASM_REG_P "), %%" INJECT_ASM_REG "\n\t" \
86 	"test %%" INJECT_ASM_REG ",%%" INJECT_ASM_REG "\n\t" \
87 	"jz 333f\n\t" \
88 	"222:\n\t" \
89 	"dec %%" INJECT_ASM_REG "\n\t" \
90 	"jnz 222b\n\t" \
91 	"333:\n\t"
109 	"l %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
110 	"ltr %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG "\n\t" \
111 	"je 333f\n\t" \
112 	"222:\n\t" \
113 	"ahi %%" INJECT_ASM_REG ", -1\n\t" \
114 	"jnz 222b\n\t" \
115 	"333:\n\t"
133 	"ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
134 	"cmp " INJECT_ASM_REG ", #0\n\t" \
135 	"beq 333f\n\t" \
136 	"222:\n\t" \
137 	"subs " INJECT_ASM_REG ", #1\n\t" \
138 	"bne 222b\n\t" \
139 	"333:\n\t"
177 	"lwz %%" INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
178 	"cmpwi %%" INJECT_ASM_REG ", 0\n\t" \
179 	"beq 333f\n\t" \
180 	"222:\n\t" \
181 	"subic. %%" INJECT_ASM_REG ", %%" INJECT_ASM_REG ", 1\n\t" \
182 	"bne 222b\n\t" \
183 	"333:\n\t"
201 	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
202 	"beqz " INJECT_ASM_REG ", 333f\n\t" \
203 	"222:\n\t" \
204 	"addiu " INJECT_ASM_REG ", -1\n\t" \
205 	"bnez " INJECT_ASM_REG ", 222b\n\t" \
206 	"333:\n\t"
223 	"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t"		\
224 	"beqz " INJECT_ASM_REG ", 333f\n\t"			\
225 	"222:\n\t"						\
226 	"addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\
227 	"bnez " INJECT_ASM_REG ", 222b\n\t"			\
228 	"333:\n\t"
245 	"l.lwz   " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t"	\
246 	"l.sfeqi " INJECT_ASM_REG ", 0\n\t"			\
247 	"l.bf 333f\n\t"						\
248 	" l.nop\n\t"						\
249 	"222:\n\t"						\
250 	"l.addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\
251 	"l.sfeqi " INJECT_ASM_REG ", 0\n\t"			\
252 	"l.bf 222f\n\t"						\
253 	" l.nop\n\t"						\
254 	"333:\n\t"
269 	if (loc_nr_loops == -1 && opt_modulo) { \
270 		if (yield_mod_cnt == opt_modulo - 1) { \
401 	struct percpu_list_node *head;  member
456 					 &lock->c[cpu].v,  in rseq_this_cpu_lock()
472 	assert(lock->c[cpu].v == 1);  in rseq_percpu_unlock()
477 	rseq_smp_store_release(&lock->c[cpu].v, 0);  in rseq_percpu_unlock()
483 	struct spinlock_test_data *data = thread_data->data;  in test_percpu_spinlock_thread()
486 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_spinlock_thread()
489 	reps = thread_data->reps;  in test_percpu_spinlock_thread()
491 		int cpu = rseq_this_cpu_lock(&data->lock);  in test_percpu_spinlock_thread()
492 		data->c[cpu].count++;  in test_percpu_spinlock_thread()
493 		rseq_percpu_unlock(&data->lock, cpu);  in test_percpu_spinlock_thread()
502 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_spinlock_thread()
509  * A simple test which implements a sharded counter using a per-cpu
511  * per-cpu increment; however, this is reasonable for a test and the
560 	struct inc_test_data *data = thread_data->data;  in test_percpu_inc_thread()
563 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_inc_thread()
566 	reps = thread_data->reps;  in test_percpu_inc_thread()
575 					&data->c[cpu].count, 1, cpu);  in test_percpu_inc_thread()
585 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_inc_thread()
645 		/* Load list->c[cpu].head with single-copy atomicity. */  in this_cpu_list_push()
646 		expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);  in this_cpu_list_push()
648 		targetptr = (intptr_t *)&list->c[cpu].head;  in this_cpu_list_push()
649 		node->next = (struct percpu_list_node *)expect;  in this_cpu_list_push()
661  * Unlike a traditional lock-less linked list; the availability of a
663  * ABA-type races.
672 		struct percpu_list_node *head;  in this_cpu_list_pop()  local
678 		targetptr = (intptr_t *)&list->c[cpu].head;  in this_cpu_list_pop()
681 		load = (intptr_t *)&head;  in this_cpu_list_pop()
686 			node = head;  in this_cpu_list_pop()
706 	node = list->c[cpu].head;  in __percpu_list_pop()
709 	list->c[cpu].head = node->next;  in __percpu_list_pop()
740 /* Simultaneous modification to a per-cpu linked list from many threads.  */
764 			node->data = j;  in test_percpu_list()
765 			node->next = list.c[i].head;  in test_percpu_list()
766 			list.c[i].head = node;  in test_percpu_list()
796 			sum += node->data;  in test_percpu_list()
823 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_push()
824 		if (offset == buffer->c[cpu].buflen)  in this_cpu_buffer_push()
827 		targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];  in this_cpu_buffer_push()
829 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_buffer_push()
847 	struct percpu_buffer_node *head;  in this_cpu_buffer_pop()  local
856 		/* Load offset with single-copy atomicity. */  in this_cpu_buffer_pop()
857 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_pop()
859 			head = NULL;  in this_cpu_buffer_pop()
862 		head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);  in this_cpu_buffer_pop()
863 		newval = offset - 1;  in this_cpu_buffer_pop()
864 		targetptr = (intptr_t *)&buffer->c[cpu].offset;  in this_cpu_buffer_pop()
867 			(intptr_t *)&buffer->c[cpu].array[offset - 1],  in this_cpu_buffer_pop()
868 			(intptr_t)head, newval, cpu);  in this_cpu_buffer_pop()
875 	return head;  in this_cpu_buffer_pop()
885 	struct percpu_buffer_node *head;  in __percpu_buffer_pop()  local
888 	offset = buffer->c[cpu].offset;  in __percpu_buffer_pop()
891 	head = buffer->c[cpu].array[offset - 1];  in __percpu_buffer_pop()
892 	buffer->c[cpu].offset = offset - 1;  in __percpu_buffer_pop()
893 	return head;  in __percpu_buffer_pop()
927 /* Simultaneous modification to a per-cpu buffer from many threads.  */
944 		/* Worse-case is every item in same CPU. */  in test_percpu_buffer()
956 			 * We could theoretically put the word-sized  in test_percpu_buffer()
964 			node->data = j;  in test_percpu_buffer()
965 			buffer.c[i].array[j - 1] = node;  in test_percpu_buffer()
996 			sum += node->data;  in test_percpu_buffer()
1024 		/* Load offset with single-copy atomicity. */  in this_cpu_memcpy_buffer_push()
1025 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_push()
1026 		if (offset == buffer->c[cpu].buflen)  in this_cpu_memcpy_buffer_push()
1028 		destptr = (char *)&buffer->c[cpu].array[offset];  in this_cpu_memcpy_buffer_push()
1033 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_push()
1064 		/* Load offset with single-copy atomicity. */  in this_cpu_memcpy_buffer_pop()
1065 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_pop()
1069 		srcptr = (char *)&buffer->c[cpu].array[offset - 1];  in this_cpu_memcpy_buffer_pop()
1072 		newval_final = offset - 1;  in this_cpu_memcpy_buffer_pop()
1073 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_pop()
1098 	offset = buffer->c[cpu].offset;  in __percpu_memcpy_buffer_pop()
1101 	memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));  in __percpu_memcpy_buffer_pop()
1102 	buffer->c[cpu].offset = offset - 1;  in __percpu_memcpy_buffer_pop()
1138 /* Simultaneous modification to a per-cpu buffer from many threads.  */
1155 		/* Worse-case is every item in same CPU. */  in test_percpu_memcpy_buffer()
1165 			 * We could theoretically put the word-sized  in test_percpu_memcpy_buffer()
1171 			buffer.c[i].array[j - 1].data1 = j;  in test_percpu_memcpy_buffer()
1172 			buffer.c[i].array[j - 1].data2 = j + 1;  in test_percpu_memcpy_buffer()
1271 	while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {}  in test_membarrier_worker_thread()
1280 				&args->percpu_list_ptr,  in test_membarrier_worker_thread()
1303 		node->data = 0;  in test_membarrier_init_percpu_list()
1304 		node->next = NULL;  in test_membarrier_init_percpu_list()
1305 		list->c[i].head = node;  in test_membarrier_init_percpu_list()
1314 		free(list->c[i].head);  in test_membarrier_free_percpu_list()
1318  * The manager thread swaps per-cpu lists that worker threads see,
1339 	__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1341 	while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1348 		if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1354 		__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1364 		expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE);  in test_membarrier_manager_thread()
1371 		if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1377 		__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1384 		expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE);  in test_membarrier_manager_thread()
1462 	printf("	[-1 loops] Number of loops for delay injection 1\n");  in show_usage()
1463 	printf("	[-2 loops] Number of loops for delay injection 2\n");  in show_usage()
1464 	printf("	[-3 loops] Number of loops for delay injection 3\n");  in show_usage()
1465 	printf("	[-4 loops] Number of loops for delay injection 4\n");  in show_usage()
1466 	printf("	[-5 loops] Number of loops for delay injection 5\n");  in show_usage()
1467 	printf("	[-6 loops] Number of loops for delay injection 6\n");  in show_usage()
1468 	printf("	[-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");  in show_usage()
1469 	printf("	[-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");  in show_usage()
1470 	printf("	[-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");  in show_usage()
1471 	printf("	[-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");  in show_usage()
1472 	printf("	[-y] Yield\n");  in show_usage()
1473 	printf("	[-k] Kill thread with signal\n");  in show_usage()
1474 	printf("	[-s S] S: =0: disabled (default), >0: sleep time (ms)\n");  in show_usage()
1475 	printf("	[-t N] Number of threads (default 200)\n");  in show_usage()
1476 	printf("	[-r N] Number of repetitions per thread (default 5000)\n");  in show_usage()
1477 	printf("	[-d] Disable rseq system call (no initialization)\n");  in show_usage()
1478 	printf("	[-D M] Disable rseq for each M threads\n");  in show_usage()
1479 …printf("	[-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\…  in show_usage()
1480 	printf("	[-M] Push into buffer and memcpy buffer with memory barriers.\n");  in show_usage()
1481 	printf("	[-O] Test with optimized RSEQ\n");  in show_usage()
1482 	printf("	[-v] Verbose output.\n");  in show_usage()
1483 	printf("	[-h] Show this help.\n");  in show_usage()
1492 		if (argv[i][0] != '-')  in main()
1508 			loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);  in main()
1556 		case 't':  in main()
1583 		case 'T':  in main()
1666 	return -1;  in main()