Lines Matching +full:cpu +full:- +full:offset
1 // SPDX-License-Identifier: LGPL-2.1
44 static __thread __attribute__((tls_model("initial-exec")))
49 static __thread __attribute__((tls_model("initial-exec"), unused))
113 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
204 "addiu " INJECT_ASM_REG ", -1\n\t" \
226 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
250 "l.addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
269 if (loc_nr_loops == -1 && opt_modulo) { \
270 if (yield_mod_cnt == opt_modulo - 1) { \
326 int rseq_membarrier_expedited(int cpu)
351 int rseq_membarrier_expedited(int cpu)
354 MEMBARRIER_CMD_FLAG_CPU, cpu);
412 intptr_t offset;
429 intptr_t offset;
438 /* A simple percpu spinlock. Grabs lock on current cpu. */
441 int cpu;
446 cpu = get_current_cpu_id();
447 if (cpu < 0) {
448 fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n",
449 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu);
453 &lock->c[cpu].v,
454 0, 1, cpu);
464 return cpu;
467 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
469 assert(lock->c[cpu].v == 1);
474 rseq_smp_store_release(&lock->c[cpu].v, 0);
480 struct spinlock_test_data *data = thread_data->data;
483 if (!opt_disable_rseq && thread_data->reg &&
486 reps = thread_data->reps;
488 int cpu = rseq_this_cpu_lock(&data->lock);
489 data->c[cpu].count++;
490 rseq_percpu_unlock(&data->lock, cpu);
499 if (!opt_disable_rseq && thread_data->reg &&
506 * A simple test which implements a sharded counter using a per-cpu
508 * per-cpu increment; however, this is reasonable for a test and the
557 struct inc_test_data *data = thread_data->data;
560 if (!opt_disable_rseq && thread_data->reg &&
563 reps = thread_data->reps;
568 int cpu;
570 cpu = get_current_cpu_id();
572 &data->c[cpu].count, 1, cpu);
582 if (!opt_disable_rseq && thread_data->reg &&
635 int cpu;
641 cpu = get_current_cpu_id();
642 /* Load list->c[cpu].head with single-copy atomicity. */
643 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);
645 targetptr = (intptr_t *)&list->c[cpu].head;
646 node->next = (struct percpu_list_node *)expect;
648 targetptr, expect, newval, cpu);
654 *_cpu = cpu;
658 * Unlike a traditional lock-less linked list; the availability of a
660 * ABA-type races.
666 int cpu;
671 long offset;
674 cpu = get_current_cpu_id();
675 targetptr = (intptr_t *)&list->c[cpu].head;
677 offset = offsetof(struct percpu_list_node, next);
681 offset, load, cpu);
691 *_cpu = cpu;
699 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)
703 node = list->c[cpu].head;
706 list->c[cpu].head = node->next;
737 /* Simultaneous modification to a per-cpu linked list from many threads. */
749 /* Generate list entries for every usable cpu. */
761 node->data = j;
762 node->next = list.c[i].head;
793 sum += node->data;
811 int cpu;
816 intptr_t offset;
819 cpu = get_current_cpu_id();
820 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
821 if (offset == buffer->c[cpu].buflen)
824 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];
825 newval_final = offset + 1;
826 targetptr_final = &buffer->c[cpu].offset;
828 targetptr_final, offset, targetptr_spec,
829 newval_spec, newval_final, cpu);
837 *_cpu = cpu;
845 int cpu;
849 intptr_t offset;
852 cpu = get_current_cpu_id();
853 /* Load offset with single-copy atomicity. */
854 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
855 if (offset == 0) {
859 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);
860 newval = offset - 1;
861 targetptr = (intptr_t *)&buffer->c[cpu].offset;
863 targetptr, offset,
864 (intptr_t *)&buffer->c[cpu].array[offset - 1],
865 (intptr_t)head, newval, cpu);
871 *_cpu = cpu;
880 int cpu)
883 intptr_t offset;
885 offset = buffer->c[cpu].offset;
886 if (offset == 0)
888 head = buffer->c[cpu].array[offset - 1];
889 buffer->c[cpu].offset = offset - 1;
924 /* Simultaneous modification to a per-cpu buffer from many threads. */
936 /* Generate list entries for every usable cpu. */
941 /* Worse-case is every item in same CPU. */
953 * We could theoretically put the word-sized
961 node->data = j;
962 buffer.c[i].array[j - 1] = node;
963 buffer.c[i].offset++;
993 sum += node->data;
1012 int cpu;
1015 intptr_t *targetptr_final, newval_final, offset;
1020 cpu = get_current_cpu_id();
1021 /* Load offset with single-copy atomicity. */
1022 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
1023 if (offset == buffer->c[cpu].buflen)
1025 destptr = (char *)&buffer->c[cpu].array[offset];
1029 newval_final = offset + 1;
1030 targetptr_final = &buffer->c[cpu].offset;
1033 targetptr_final, offset,
1035 newval_final, cpu);
1043 *_cpu = cpu;
1052 int cpu;
1055 intptr_t *targetptr_final, newval_final, offset;
1060 cpu = get_current_cpu_id();
1061 /* Load offset with single-copy atomicity. */
1062 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);
1063 if (offset == 0)
1066 srcptr = (char *)&buffer->c[cpu].array[offset - 1];
1069 newval_final = offset - 1;
1070 targetptr_final = &buffer->c[cpu].offset;
1072 targetptr_final, offset, destptr, srcptr, copylen,
1073 newval_final, cpu);
1081 *_cpu = cpu;
1091 int cpu)
1093 intptr_t offset;
1095 offset = buffer->c[cpu].offset;
1096 if (offset == 0)
1098 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));
1099 buffer->c[cpu].offset = offset - 1;
1135 /* Simultaneous modification to a per-cpu buffer from many threads. */
1147 /* Generate list entries for every usable cpu. */
1152 /* Worse-case is every item in same CPU. */
1162 * We could theoretically put the word-sized
1168 buffer.c[i].array[j - 1].data1 = j;
1169 buffer.c[i].array[j - 1].data2 = j + 1;
1170 buffer.c[i].offset++;
1268 while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {}
1274 int cpu = get_current_cpu_id();
1277 &args->percpu_list_ptr,
1278 sizeof(struct percpu_list_entry) * cpu, 1, cpu);
1300 node->data = 0;
1301 node->next = NULL;
1302 list->c[i].head = node;
1311 free(list->c[i].head);
1315 * The manager thread swaps per-cpu lists that worker threads see,
1336 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);
1338 while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) {
1345 if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) {
1351 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE);
1353 errno != ENXIO /* missing CPU */) {
1358 * Cpu A should now only modify list_b, so the values
1361 expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE);
1368 if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) {
1374 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);
1376 errno != ENXIO /* missing CPU*/) {
1381 expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE);
1459 printf(" [-1 loops] Number of loops for delay injection 1\n");
1460 printf(" [-2 loops] Number of loops for delay injection 2\n");
1461 printf(" [-3 loops] Number of loops for delay injection 3\n");
1462 printf(" [-4 loops] Number of loops for delay injection 4\n");
1463 printf(" [-5 loops] Number of loops for delay injection 5\n");
1464 printf(" [-6 loops] Number of loops for delay injection 6\n");
1465 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");
1466 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");
1467 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");
1468 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");
1469 printf(" [-y] Yield\n");
1470 printf(" [-k] Kill thread with signal\n");
1471 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n");
1472 printf(" [-t N] Number of threads (default 200)\n");
1473 printf(" [-r N] Number of repetitions per thread (default 5000)\n");
1474 printf(" [-d] Disable rseq system call (no initialization)\n");
1475 printf(" [-D M] Disable rseq for each M threads\n");
1476 printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n");
1477 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n");
1478 printf(" [-v] Verbose output.\n");
1479 printf(" [-h] Show this help.\n");
1488 if (argv[i][0] != '-')
1504 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);
1624 fprintf(stderr, "Error: cpu id getter unavailable\n");
1659 return -1;