xref: /freebsd/sys/contrib/openzfs/cmd/zstream/zstream_selftest_queue.c (revision 2f10ffc003be396f3fc23cd2888023896560252b)
1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3  * This file and its contents are supplied under the terms of the
4  * Common Development and Distribution License ("CDDL"), version 1.0.
5  * You may only use this file in accordance with the terms of version
6  * 1.0 of the CDDL.
7  *
8  * A full copy of the text of the CDDL should have accompanied this
9  * source.  A copy of the CDDL is also available via the Internet at
10  * https://opensource.org/license/CDDL-1.0.
11  */
12 
13 /*
14  * Copyright (c) 2026 by Garth Snyder. All rights reserved.
15  */
16 
17 /*
18  * Selftests for the zstream_queue multithreaded FIFO queue API.
19  *
20  * All tests are built on one generic workload runner. A workload is
21  * described by a qtest_config_t: some number of producer threads each
22  * enqueue a stream of self-describing items with randomized costs,
23  * payloads, and processing delays, while one consumer thread per queue
24  * dequeues and verifies. Several workloads can run concurrently on separate
25  * queues to exercise the shared thread pool.
26  *
27  * Every item carries enough information to be verified independently:
28  *
29  * - The tuple (qi_producer, qi_seq) identifies each item; the consumer
30  * checks that each producer's items arrive in the same order they were
31  * enqueued.
32  *
33  * - qi_check is a hash of (qi_seed, qi_producer, qi_seq). The processing
34  * function verifies it and then XORs in TRANSFORM_MAGIC. The consumer
35  * checks that the transform happened iff cost > 0.
36  *
37  * - qi_pattern[] is filled from qi_check and verified both by the process
38  * function and the consumer, to catch any corruption of the shallow copies
39  * in and out of the ring buffer.
40  *
41  * - qi_process_count counts invocations of the process function, which must
42  * be exactly one for cost > 0 items and zero for cost == 0 items.
43  *
44  * Global conservation checks: the number of items dequeued must equal the
45  * number enqueued, and the total number of process-function invocations
46  * must equal the number of nonzero-cost items enqueued.
47  *
48  * Costs are not passive values. Each queue measures how long its own work
49  * takes per unit of cost and sizes its batches accordingly. So, a
50  * workload's cost distribution decides how the implementation will behave.
51  * Configs can set qc_ns_per_cost to make processing time genuinely
52  * proportional to cost (the relationship the queue assumes), or leave it at
53  * zero to get delays unrelated to cost. Both are worth testing; see
54  * queue_batch_tuning().
55  */
56 
57 #include <assert.h>
58 #include <atomic.h>
59 #include <err.h>
60 #include <pthread.h>
61 #include <stdalign.h>
62 #include <stdint.h>
63 #include <stdio.h>
64 #include <string.h>
65 #include <sys/param.h>
66 #include <unistd.h>
67 
68 #include "zstream_queue.h"
69 #include "zstream_selftest.h"
70 
71 #define	TRANSFORM_MAGIC	0xf00dfeedbeefcafeULL
72 
73 /*
74  * Number of times per 1000 processing function invocations to use an
75  * extra-long "outlier" processing delay to force overtly out-of-order
76  * completion.
77  */
78 #define	LONG_DELAYS_PER_THOUSAND	3
79 #define	LONG_DELAY_MULTIPLIER		20
80 
81 typedef struct {
82 	uint32_t	qi_producer;
83 	uint32_t	qi_delay_us;
84 	uint64_t	qi_seq;
85 	uint64_t	qi_check;
86 	size_t		qi_cost;
87 	uint32_t	qi_process_count;
88 	uint8_t		qi_pattern[];
89 } qtest_item_t;
90 
91 typedef struct {
92 	uint32_t	qc_producers;		/* Number of producers */
93 	uint64_t	qc_items;		/* Items per producer */
94 	size_t		qc_pattern_len;		/* Extra payload bytes */
95 	uint32_t	qc_zero_cost_pct;	/* % of items fast-tracked */
96 	size_t		qc_max_cost;		/* Nonzero costs are 1..max */
97 	uint32_t	qc_delay_pct;		/* % of items slept on */
98 	uint32_t	qc_max_delay_us;
99 	uint32_t	qc_ns_per_cost;		/* Delay of cost * this, ns */
100 	uint32_t	qc_producer_stall_pct;	/* % chance producer naps */
101 	uint32_t	qc_consumer_stall_pct;	/* % chance consumer naps */
102 	uint32_t	qc_stall_max_us;
103 	uint64_t	qc_rng_stream;		/* base PRNG stream number */
104 } qtest_config_t;
105 
106 typedef struct {
107 	const qtest_config_t	*qr_cfg;
108 	zstream_queue_t		*qr_queue;
109 	uint32_t		qr_producers_left;
110 	uint64_t		qr_expect_processed;	/* Atomic */
111 	uint64_t		qr_processed;		/* Atomic */
112 	uint64_t		qr_dequeued;
113 } qtest_run_t;
114 
115 typedef struct {
116 	qtest_run_t	*qp_run;
117 	uint32_t	qp_id;
118 } qtest_producer_arg_t;
119 
120 static uint64_t
item_check_value(uint32_t producer,uint64_t seq)121 item_check_value(uint32_t producer, uint64_t seq)
122 {
123 	return (selftest_mix64(selftest_seed ^
124 	    (((uint64_t)producer << 40) + seq)));
125 }
126 
127 static void
fill_pattern(uint8_t * pattern,size_t len,uint64_t check)128 fill_pattern(uint8_t *pattern, size_t len, uint64_t check)
129 {
130 	for (size_t i = 0; i < len; i++)
131 		pattern[i] = (uint8_t)(check >> ((i & 7) << 3)) ^ (uint8_t)i;
132 }
133 
134 static void
verify_pattern(const uint8_t * pattern,size_t len,uint64_t check,const char * who)135 verify_pattern(const uint8_t *pattern, size_t len, uint64_t check,
136     const char *who)
137 {
138 	for (size_t i = 0; i < len; i++) {
139 		uint8_t expect =
140 		    (uint8_t)(check >> ((i & 7) << 3)) ^ (uint8_t)i;
141 		if (pattern[i] != expect) {
142 			errx(1, "%s: payload corrupted at byte %zu "
143 			    "(0x%02x != 0x%02x)", who, i, pattern[i], expect);
144 		}
145 	}
146 }
147 
148 static size_t
qtest_cost(void * item_in,void * context)149 qtest_cost(void *item_in, void *context)
150 {
151 	(void) context;
152 	qtest_item_t *item = item_in;
153 	return (item->qi_cost);
154 }
155 
156 static void
qtest_process(void * item_in,void * context)157 qtest_process(void *item_in, void *context)
158 {
159 	qtest_run_t *run = context;
160 	qtest_item_t *item = item_in;
161 
162 	/* Cost-0 items should never reach the process function */
163 	VERIFY3U(item->qi_cost, >, 0);
164 	VERIFY3U(item->qi_check, ==,
165 	    item_check_value(item->qi_producer, item->qi_seq));
166 	verify_pattern(item->qi_pattern, run->qr_cfg->qc_pattern_len,
167 	    item->qi_check, "process");
168 	VERIFY3U(atomic_add_32_nv(&item->qi_process_count, 1), ==, 1);
169 
170 	if (item->qi_delay_us > 0)
171 		(void) usleep(item->qi_delay_us);
172 
173 	item->qi_check ^= TRANSFORM_MAGIC;
174 	atomic_add_64(&run->qr_processed, 1);
175 }
176 
177 /*
178  * Pthreads worker function for enqueuers
179  */
180 static void *
qtest_producer(void * arg)181 qtest_producer(void *arg)
182 {
183 	qtest_producer_arg_t *pa = arg;
184 	qtest_run_t *run = pa->qp_run;
185 	const qtest_config_t *cfg = run->qr_cfg;
186 	uint64_t local_expect = 0;
187 	selftest_rng_t rng;
188 	alignas(__alignof__(uint64_t)) uint8_t item_buffer[
189 	    sizeof (qtest_item_t) + cfg->qc_pattern_len];
190 	qtest_item_t *item = (qtest_item_t *)item_buffer;
191 
192 	selftest_rng_init(&rng, cfg->qc_rng_stream + 1000 + pa->qp_id);
193 
194 	for (uint64_t seq = 0; seq < cfg->qc_items; seq++) {
195 
196 		qtest_item_t item_xfer = {
197 			.qi_producer = pa->qp_id,
198 			.qi_seq = seq,
199 			.qi_process_count = 0,
200 			.qi_check = item_check_value(pa->qp_id, seq)
201 		};
202 		*item = item_xfer;
203 		fill_pattern(item->qi_pattern, cfg->qc_pattern_len,
204 		    item->qi_check);
205 
206 		if (selftest_rng_below(&rng, 100) < cfg->qc_zero_cost_pct) {
207 			item->qi_cost = 0;
208 		} else {
209 			item->qi_cost =
210 			    1 + selftest_rng_below(&rng, cfg->qc_max_cost);
211 			local_expect++;
212 		}
213 
214 		if (item->qi_cost > 0 && cfg->qc_ns_per_cost > 0) {
215 			/*
216 			 * Make processing time proportional to cost, which
217 			 * is the relationship the queue's batch sizing
218 			 * assumes. Costs here are small enough that the
219 			 * product can't overflow, but clamp anyway so that
220 			 * a future config can't turn a tuning test into a
221 			 * multi-second sleep.
222 			 */
223 			uint64_t ns = (uint64_t)item->qi_cost *
224 			    cfg->qc_ns_per_cost;
225 			item->qi_delay_us = MIN(ns / 1000, 100000);
226 		} else if (item->qi_cost > 0 && cfg->qc_max_delay_us > 0) {
227 			if (selftest_rng_below(&rng, 1000) <
228 			    LONG_DELAYS_PER_THOUSAND) {
229 				item->qi_delay_us = cfg->qc_max_delay_us *
230 				    LONG_DELAY_MULTIPLIER;
231 			} else if (selftest_rng_below(&rng, 100) <
232 			    cfg->qc_delay_pct) {
233 				item->qi_delay_us = selftest_rng_below(&rng,
234 				    cfg->qc_max_delay_us);
235 			}
236 		}
237 
238 		if (cfg->qc_producer_stall_pct > 0 &&
239 		    selftest_rng_below(&rng, 100) < cfg->qc_producer_stall_pct)
240 			(void) usleep(selftest_rng_below(&rng,
241 			    cfg->qc_stall_max_us));
242 
243 		zstream_enqueue(run->qr_queue, item);
244 	}
245 
246 	atomic_add_64(&run->qr_expect_processed, local_expect);
247 	if (atomic_add_32_nv(&run->qr_producers_left, -1) == 0)
248 		zstream_queue_fini(run->qr_queue);
249 	return (NULL);
250 }
251 
252 /*
253  * Pthreads worker function for dequeuers
254  */
255 static void *
qtest_consumer(void * arg)256 qtest_consumer(void *arg)
257 {
258 	qtest_run_t *run = arg;
259 	const qtest_config_t *cfg = run->qr_cfg;
260 	selftest_rng_t rng;
261 	uint64_t expected_seq[cfg->qc_producers];
262 	alignas(__alignof__(uint64_t)) uint8_t item_buffer[
263 	    sizeof (qtest_item_t) + cfg->qc_pattern_len];
264 	qtest_item_t *item = (qtest_item_t *)item_buffer;
265 
266 	memset(expected_seq, 0, sizeof (expected_seq));
267 	selftest_rng_init(&rng, cfg->qc_rng_stream + 999);
268 
269 	while (zstream_dequeue(run->qr_queue, item)) {
270 		VERIFY3U(item->qi_producer, <, cfg->qc_producers);
271 		if (item->qi_seq != expected_seq[item->qi_producer]) {
272 			errx(1, "consumer: FIFO order violated: got "
273 			    "producer %u seq %ju, expected seq %ju",
274 			    item->qi_producer, (uintmax_t)item->qi_seq,
275 			    (uintmax_t)expected_seq[item->qi_producer]);
276 		}
277 		expected_seq[item->qi_producer]++;
278 
279 		uint64_t check =
280 		    item_check_value(item->qi_producer, item->qi_seq);
281 		if (item->qi_cost > 0) {
282 			VERIFY3U(item->qi_process_count, ==, 1);
283 			VERIFY3U(item->qi_check, ==, check ^ TRANSFORM_MAGIC);
284 		} else {
285 			VERIFY3U(item->qi_process_count, ==, 0);
286 			VERIFY3U(item->qi_check, ==, check);
287 		}
288 		verify_pattern(item->qi_pattern, cfg->qc_pattern_len, check,
289 		    "consumer");
290 		run->qr_dequeued++;
291 
292 		if (cfg->qc_consumer_stall_pct > 0 &&
293 		    selftest_rng_below(&rng, 100) < cfg->qc_consumer_stall_pct)
294 			(void) usleep(selftest_rng_below(&rng,
295 			    cfg->qc_stall_max_us));
296 	}
297 
298 	for (uint32_t p = 0; p < cfg->qc_producers; p++)
299 		VERIFY3U(expected_seq[p], ==, cfg->qc_items);
300 	VERIFY3U(run->qr_dequeued, ==,
301 	    (uint64_t)cfg->qc_producers * cfg->qc_items);
302 
303 	return (NULL);
304 }
305 
306 /*
307  * Run several workloads at once, one queue per config, with a dedicated
308  * consumer thread and qc_producers producer threads per queue. Returns
309  * after every queue has been drained to end-of-stream (and therefore
310  * destroyed) and all verification checks have passed.
311  */
312 static void
run_queue_workloads(const qtest_config_t * cfgs,int ncfg)313 run_queue_workloads(const qtest_config_t *cfgs, int ncfg)
314 {
315 	qtest_run_t runs[ncfg];
316 	pthread_t consumers[ncfg];
317 	uint32_t total_producers = 0;
318 
319 	for (int i = 0; i < ncfg; i++)
320 		total_producers += cfgs[i].qc_producers;
321 
322 	pthread_t producers[total_producers];
323 	qtest_producer_arg_t pargs[total_producers];
324 	memset(runs, 0, sizeof (runs));
325 	memset(pargs, 0, sizeof (pargs));
326 
327 	for (int i = 0; i < ncfg; i++) {
328 		runs[i].qr_cfg = &cfgs[i];
329 		runs[i].qr_producers_left = cfgs[i].qc_producers;
330 		zq_params_t params = {
331 			.qp_process = qtest_process,
332 			.qp_cost = qtest_cost,
333 			.qp_context = &runs[i],
334 			.qp_item_size =
335 			    sizeof (qtest_item_t) + cfgs[i].qc_pattern_len
336 		};
337 		runs[i].qr_queue = zstream_queue_create(&params);
338 	}
339 
340 	int p = 0;
341 	for (int i = 0; i < ncfg; i++) {
342 		VERIFY3S(pthread_create(&consumers[i], NULL, qtest_consumer,
343 		    &runs[i]), ==, 0);
344 		for (uint32_t j = 0; j < cfgs[i].qc_producers; j++, p++) {
345 			pargs[p].qp_run = &runs[i];
346 			pargs[p].qp_id = j;
347 			VERIFY3S(pthread_create(&producers[p], NULL,
348 			    qtest_producer, &pargs[p]), ==, 0);
349 		}
350 	}
351 
352 	for (uint32_t i = 0; i < total_producers; i++)
353 		VERIFY3S(pthread_join(producers[i], NULL), ==, 0);
354 	for (int i = 0; i < ncfg; i++)
355 		VERIFY3S(pthread_join(consumers[i], NULL), ==, 0);
356 
357 	for (int i = 0; i < ncfg; i++)
358 		VERIFY3U(runs[i].qr_processed, ==, runs[i].qr_expect_processed);
359 }
360 
361 static void
run_queue_workload(const qtest_config_t * cfg)362 run_queue_workload(const qtest_config_t *cfg)
363 {
364 	run_queue_workloads(cfg, 1);
365 }
366 
367 /*
368  * Basic single-producer smoke test: deterministic-ish costs, no delays.
369  */
370 static void
queue_basic(void)371 queue_basic(void)
372 {
373 	qtest_config_t cfg = {
374 		.qc_producers = 1,
375 		.qc_items = 5000,
376 		.qc_pattern_len = 32,
377 		.qc_zero_cost_pct = 20,
378 		.qc_max_cost = 64,
379 	};
380 	run_queue_workload(&cfg);
381 }
382 
383 /*
384  * A long, randomized stream with heavy-tailed processing delays, a large
385  * fraction of fast-tracked items, and a consumer that periodically stalls.
386  * The producer outruns the consumer badly enough that the queue sits at or
387  * near ZQ_SLOTS_PER_QUEUE for most of the run, so this is the main exercise
388  * of Q_FULL and of enqueuers blocking on the dequeued condition. 100000
389  * items wrap the ring indices a couple of dozen times.
390  */
391 static void
queue_torture(void)392 queue_torture(void)
393 {
394 	qtest_config_t cfg = {
395 		.qc_producers = 1,
396 		.qc_items = 100000,
397 		.qc_pattern_len = 64,
398 		.qc_zero_cost_pct = 30,
399 		.qc_max_cost = 4096,
400 		.qc_delay_pct = 5,
401 		.qc_max_delay_us = 100,
402 		.qc_consumer_stall_pct = 1,
403 		.qc_stall_max_us = 500,
404 		.qc_rng_stream = 100,
405 	};
406 	run_queue_workload(&cfg);
407 }
408 
409 /*
410  * Off-by-one hunting: sweep stream lengths that land on and adjacent to
411  * every boundary the implementation has, including a zero-item stream and
412  * zero-length payloads.
413  *
414  * Queue depth and batch size are no longer caller-supplied, so the
415  * boundaries worth probing are the implementation's own: ZQ_MAX_BATCH, the
416  * point at which claim_batch() stops filling a batch, and
417  * ZQ_SLOTS_PER_QUEUE, the point at which the ring index wraps and Q_FULL
418  * can trip. Both are exported by zstream_queue.h for exactly this purpose,
419  * so this sweep tracks them automatically if either one is retuned.
420  *
421  * A stalling consumer is used for the counts at or above
422  * ZQ_SLOTS_PER_QUEUE. Without it the consumer keeps up, the queue never
423  * approaches full, and the wrap and Q_FULL paths go untested no matter how
424  * many items are sent.
425  */
426 static void
queue_edge_cases(void)427 queue_edge_cases(void)
428 {
429 	static const uint64_t counts[] = {
430 		0, 1, 2, 3,
431 		ZQ_MAX_BATCH - 1, ZQ_MAX_BATCH, ZQ_MAX_BATCH + 1,
432 		2 * ZQ_MAX_BATCH,
433 		ZQ_SLOTS_PER_QUEUE - 1, ZQ_SLOTS_PER_QUEUE,
434 		ZQ_SLOTS_PER_QUEUE + 1, 2 * ZQ_SLOTS_PER_QUEUE + 3
435 	};
436 	const int ncounts = sizeof (counts) / sizeof (counts[0]);
437 	uint64_t stream = 200;
438 
439 	for (int n = 0; n < ncounts; n++) {
440 		boolean_t big = counts[n] >= ZQ_SLOTS_PER_QUEUE;
441 
442 		for (int p = 0; p < 2; p++) {
443 			qtest_config_t cfg = {
444 				.qc_producers = 1,
445 				.qc_items = counts[n],
446 				.qc_pattern_len = p ? 24 : 0,
447 				.qc_zero_cost_pct = 25,
448 				.qc_max_cost = 8,
449 				.qc_consumer_stall_pct = big ? 1 : 0,
450 				.qc_stall_max_us = big ? 200 : 0,
451 				.qc_rng_stream = stream++,
452 			};
453 			run_queue_workload(&cfg);
454 		}
455 	}
456 }
457 
458 /*
459  * All items cost 0, so every item takes the fast track and the process
460  * function must never run (qtest_process VERIFYs cost > 0, and the
461  * conservation check at the end of the run confirms zero invocations).
462  * This exercises the completion-index sweep for items no worker ever
463  * touches.
464  */
465 static void
queue_zero_cost(void)466 queue_zero_cost(void)
467 {
468 	qtest_config_t cfg = {
469 		.qc_producers = 1,
470 		.qc_items = 20000,
471 		.qc_pattern_len = 16,
472 		.qc_zero_cost_pct = 100,
473 		.qc_max_cost = 8,
474 		.qc_rng_stream = 300,
475 	};
476 	run_queue_workload(&cfg);
477 }
478 
479 /*
480  * Batch sizing is derived from each queue's own measured ns-per-unit-cost,
481  * so the cost values reported by a caller now steer the implementation rather
482  * than just gating the fast track. This test runs four queues at once whose
483  * cost-to-time relationships are deliberately dissimilar and checks that
484  * all of them still deliver every item exactly once and in order.
485  *
486  * The four cases, in the order configured below:
487  *
488  * - Faithful. Processing time really is proportional to cost, which is
489  *   what the model assumes. Costs average ~2048 at 250ns each, putting a
490  *   typical item just past the 400us target on its own, so batches come
491  *   out at one or two items.
492  *
493  * - Too slow to batch. Every item costs 1 but takes 600us, so the implied
494  *   budget is a fraction of a cost unit and has to be clamped up to 1.
495  *   Batches should be single items; a rounding error that let the budget
496  *   reach 0 would spin claim_batch() without claiming anything.
497  *
498  * - Too fast to measure. Costs are enormous and the work is nothing, so
499  *   the implied budget overflows anything a size_t can hold and has to
500  *   saturate. Sums of these costs wrap inside both claim_batch() and the
501  *   queue's running total, which is allowed to produce silly batch sizes
502  *   but must not produce wrong answers.
503  *
504  * - Uniform cost. Every item costs the same, the degenerate case for a
505  *   ratio-based estimate.
506  */
507 static void
queue_batch_tuning(void)508 queue_batch_tuning(void)
509 {
510 	qtest_config_t cfgs[] = {
511 		{
512 			.qc_producers = 2,
513 			.qc_items = 1500,
514 			.qc_pattern_len = 16,
515 			.qc_zero_cost_pct = 10,
516 			.qc_max_cost = 4096,
517 			.qc_ns_per_cost = 250,
518 			.qc_rng_stream = 600,
519 		}, {
520 			.qc_producers = 1,
521 			.qc_items = 400,
522 			.qc_pattern_len = 8,
523 			.qc_zero_cost_pct = 5,
524 			.qc_max_cost = 1,
525 			.qc_ns_per_cost = 600 * 1000,
526 			.qc_rng_stream = 610,
527 		}, {
528 			.qc_producers = 2,
529 			.qc_items = 5000,
530 			.qc_pattern_len = 32,
531 			.qc_zero_cost_pct = 20,
532 			.qc_max_cost = SIZE_MAX / 2,
533 			.qc_rng_stream = 620,
534 		}, {
535 			.qc_producers = 1,
536 			.qc_items = 20000,
537 			.qc_pattern_len = 0,
538 			.qc_zero_cost_pct = 0,
539 			.qc_max_cost = 1,
540 			.qc_rng_stream = 630,
541 		}
542 	};
543 	run_queue_workloads(cfgs, sizeof (cfgs) / sizeof (cfgs[0]));
544 }
545 
546 /*
547  * Eight producer threads hammering one queue with random pacing. The
548  * consumer verifies per-producer FIFO order and exact counts.
549  */
550 static void
queue_multi_producer(void)551 queue_multi_producer(void)
552 {
553 	qtest_config_t cfg = {
554 		.qc_producers = 8,
555 		.qc_items = 15000,
556 		.qc_pattern_len = 24,
557 		.qc_zero_cost_pct = 25,
558 		.qc_max_cost = 512,
559 		.qc_delay_pct = 2,
560 		.qc_max_delay_us = 50,
561 		.qc_producer_stall_pct = 1,
562 		.qc_stall_max_us = 200,
563 		.qc_rng_stream = 400,
564 	};
565 	run_queue_workload(&cfg);
566 }
567 
568 /*
569  * Many dissimilar queues live at once, stressing worker scoring and
570  * assignment, per-queue index isolation, and destruction of queues while
571  * others remain active (which compacts the pool's queue array).
572  */
573 static void
queue_multi_queue(void)574 queue_multi_queue(void)
575 {
576 	qtest_config_t cfgs[12];
577 	for (int i = 0; i < 12; i++) {
578 		uint32_t producers = 1 + i % 3;
579 		qtest_config_t cfg = {
580 			.qc_producers = producers,
581 			.qc_items = 4000 / producers,
582 			.qc_pattern_len = 8 * (i % 5),
583 			.qc_zero_cost_pct = 10 * (i % 6),
584 			.qc_max_cost = (size_t)16 << (i % 8),
585 			.qc_delay_pct = i % 3,
586 			.qc_max_delay_us = 60,
587 			.qc_rng_stream = 500 + i * 10000,
588 		};
589 		cfgs[i] = cfg;
590 	}
591 	run_queue_workloads(cfgs, 12);
592 }
593 
594 /*
595  * Seeded chaos: randomize every workload parameter within sane bounds
596  * and run a few rounds of concurrent queues. Whatever the targeted tests
597  * miss, this net catches over many CI runs; failures replay with -s.
598  */
599 static void
queue_stress(void)600 queue_stress(void)
601 {
602 	selftest_rng_t rng;
603 	selftest_rng_init(&rng, 900);
604 
605 	for (int iter = 0; iter < 8; iter++) {
606 		int nqueues = 1 + selftest_rng_below(&rng, 4);
607 		qtest_config_t cfgs[4];
608 
609 		for (int i = 0; i < nqueues; i++) {
610 			uint32_t producers = 1 + selftest_rng_below(&rng, 4);
611 			/*
612 			 * A quarter of the queues get processing time tied
613 			 * to cost, so the batch-size estimator sees a mix of
614 			 * well-behaved and meaningless cost data.
615 			 */
616 			uint32_t ns_per_cost =
617 			    (selftest_rng_below(&rng, 4) == 0) ?
618 			    1 + selftest_rng_below(&rng, 400) : 0;
619 			qtest_config_t cfg = {
620 				.qc_producers = producers,
621 				.qc_items = (2000 +
622 				    selftest_rng_below(&rng, 8000)) /
623 				    producers,
624 				.qc_pattern_len =
625 				    selftest_rng_below(&rng, 64),
626 				.qc_zero_cost_pct =
627 				    selftest_rng_below(&rng, 101),
628 				.qc_max_cost =
629 				    1 + selftest_rng_below(&rng, 2048),
630 				.qc_delay_pct = selftest_rng_below(&rng, 4),
631 				.qc_max_delay_us =
632 				    selftest_rng_below(&rng, 120),
633 				.qc_ns_per_cost = ns_per_cost,
634 				.qc_producer_stall_pct =
635 				    selftest_rng_below(&rng, 2),
636 				.qc_consumer_stall_pct =
637 				    selftest_rng_below(&rng, 2),
638 				.qc_stall_max_us =
639 				    selftest_rng_below(&rng, 400),
640 				.qc_rng_stream = 1000000 + iter * 1000 +
641 				    i * 100,
642 			};
643 			cfgs[i] = cfg;
644 		}
645 		run_queue_workloads(cfgs, nqueues);
646 	}
647 }
648 
649 const test_case_t selftest_queue_cases[] = {
650 	{ "queue_basic",		queue_basic },
651 	{ "queue_edge_cases",		queue_edge_cases },
652 	{ "queue_zero_cost",		queue_zero_cost },
653 	{ "queue_batch_tuning",		queue_batch_tuning },
654 	{ "queue_torture",		queue_torture },
655 	{ "queue_multi_producer",	queue_multi_producer },
656 	{ "queue_multi_queue",		queue_multi_queue },
657 	{ "queue_stress",		queue_stress },
658 	{ NULL,				NULL },
659 };
660