xref: /linux/tools/testing/selftests/bpf/bench.c (revision 27b3f70553432114b3d26f4d9c72cf02f38b84ee)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2020 Facebook */
3 #define _GNU_SOURCE
4 #include <argp.h>
5 #include <linux/compiler.h>
6 #include <sys/time.h>
7 #include <sched.h>
8 #include <fcntl.h>
9 #include <pthread.h>
10 #include <sys/sysinfo.h>
11 #include <signal.h>
12 #include "bench.h"
13 #include "testing_helpers.h"
14 
15 struct env env = {
16 	.warmup_sec = 1,
17 	.duration_sec = 5,
18 	.affinity = false,
19 	.consumer_cnt = 1,
20 	.producer_cnt = 1,
21 };
22 
23 static int libbpf_print_fn(enum libbpf_print_level level,
24 		    const char *format, va_list args)
25 {
26 	if (level == LIBBPF_DEBUG && !env.verbose)
27 		return 0;
28 	return vfprintf(stderr, format, args);
29 }
30 
31 void setup_libbpf(void)
32 {
33 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
34 	libbpf_set_print(libbpf_print_fn);
35 }
36 
37 void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)
38 {
39 	long total = res->false_hits  + res->hits + res->drops;
40 
41 	printf("Iter %3d (%7.3lfus): ",
42 	       iter, (delta_ns - 1000000000) / 1000.0);
43 
44 	printf("%ld false hits of %ld total operations. Percentage = %2.2f %%\n",
45 	       res->false_hits, total, ((float)res->false_hits / total) * 100);
46 }
47 
48 void false_hits_report_final(struct bench_res res[], int res_cnt)
49 {
50 	long total_hits = 0, total_drops = 0, total_false_hits = 0, total_ops = 0;
51 	int i;
52 
53 	for (i = 0; i < res_cnt; i++) {
54 		total_hits += res[i].hits;
55 		total_false_hits += res[i].false_hits;
56 		total_drops += res[i].drops;
57 	}
58 	total_ops = total_hits + total_false_hits + total_drops;
59 
60 	printf("Summary: %ld false hits of %ld total operations. ",
61 	       total_false_hits, total_ops);
62 	printf("Percentage =  %2.2f %%\n",
63 	       ((float)total_false_hits / total_ops) * 100);
64 }
65 
66 void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns)
67 {
68 	double hits_per_sec, drops_per_sec;
69 	double hits_per_prod;
70 
71 	hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
72 	hits_per_prod = hits_per_sec / env.producer_cnt;
73 	drops_per_sec = res->drops / 1000000.0 / (delta_ns / 1000000000.0);
74 
75 	printf("Iter %3d (%7.3lfus): ",
76 	       iter, (delta_ns - 1000000000) / 1000.0);
77 
78 	printf("hits %8.3lfM/s (%7.3lfM/prod), drops %8.3lfM/s, total operations %8.3lfM/s\n",
79 	       hits_per_sec, hits_per_prod, drops_per_sec, hits_per_sec + drops_per_sec);
80 }
81 
82 void hits_drops_report_final(struct bench_res res[], int res_cnt)
83 {
84 	int i;
85 	double hits_mean = 0.0, drops_mean = 0.0, total_ops_mean = 0.0;
86 	double hits_stddev = 0.0, drops_stddev = 0.0, total_ops_stddev = 0.0;
87 	double total_ops;
88 
89 	for (i = 0; i < res_cnt; i++) {
90 		hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
91 		drops_mean += res[i].drops / 1000000.0 / (0.0 + res_cnt);
92 	}
93 	total_ops_mean = hits_mean + drops_mean;
94 
95 	if (res_cnt > 1)  {
96 		for (i = 0; i < res_cnt; i++) {
97 			hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
98 				       (hits_mean - res[i].hits / 1000000.0) /
99 				       (res_cnt - 1.0);
100 			drops_stddev += (drops_mean - res[i].drops / 1000000.0) *
101 					(drops_mean - res[i].drops / 1000000.0) /
102 					(res_cnt - 1.0);
103 			total_ops = res[i].hits + res[i].drops;
104 			total_ops_stddev += (total_ops_mean - total_ops / 1000000.0) *
105 					(total_ops_mean - total_ops / 1000000.0) /
106 					(res_cnt - 1.0);
107 		}
108 		hits_stddev = sqrt(hits_stddev);
109 		drops_stddev = sqrt(drops_stddev);
110 		total_ops_stddev = sqrt(total_ops_stddev);
111 	}
112 	printf("Summary: hits %8.3lf \u00B1 %5.3lfM/s (%7.3lfM/prod), ",
113 	       hits_mean, hits_stddev, hits_mean / env.producer_cnt);
114 	printf("drops %8.3lf \u00B1 %5.3lfM/s, ",
115 	       drops_mean, drops_stddev);
116 	printf("total operations %8.3lf \u00B1 %5.3lfM/s\n",
117 	       total_ops_mean, total_ops_stddev);
118 }
119 
120 void ops_report_progress(int iter, struct bench_res *res, long delta_ns)
121 {
122 	double hits_per_sec, hits_per_prod;
123 
124 	hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);
125 	hits_per_prod = hits_per_sec / env.producer_cnt;
126 
127 	printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
128 
129 	printf("hits %8.3lfM/s (%7.3lfM/prod)\n", hits_per_sec, hits_per_prod);
130 }
131 
132 void ops_report_final(struct bench_res res[], int res_cnt)
133 {
134 	double hits_mean = 0.0, hits_stddev = 0.0;
135 	int i;
136 
137 	for (i = 0; i < res_cnt; i++)
138 		hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
139 
140 	if (res_cnt > 1)  {
141 		for (i = 0; i < res_cnt; i++)
142 			hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
143 				       (hits_mean - res[i].hits / 1000000.0) /
144 				       (res_cnt - 1.0);
145 
146 		hits_stddev = sqrt(hits_stddev);
147 	}
148 	printf("Summary: throughput %8.3lf \u00B1 %5.3lf M ops/s (%7.3lfM ops/prod), ",
149 	       hits_mean, hits_stddev, hits_mean / env.producer_cnt);
150 	printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);
151 }
152 
153 void local_storage_report_progress(int iter, struct bench_res *res,
154 				   long delta_ns)
155 {
156 	double important_hits_per_sec, hits_per_sec;
157 	double delta_sec = delta_ns / 1000000000.0;
158 
159 	hits_per_sec = res->hits / 1000000.0 / delta_sec;
160 	important_hits_per_sec = res->important_hits / 1000000.0 / delta_sec;
161 
162 	printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);
163 
164 	printf("hits %8.3lfM/s ", hits_per_sec);
165 	printf("important_hits %8.3lfM/s\n", important_hits_per_sec);
166 }
167 
168 void local_storage_report_final(struct bench_res res[], int res_cnt)
169 {
170 	double important_hits_mean = 0.0, important_hits_stddev = 0.0;
171 	double hits_mean = 0.0, hits_stddev = 0.0;
172 	int i;
173 
174 	for (i = 0; i < res_cnt; i++) {
175 		hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);
176 		important_hits_mean += res[i].important_hits / 1000000.0 / (0.0 + res_cnt);
177 	}
178 
179 	if (res_cnt > 1)  {
180 		for (i = 0; i < res_cnt; i++) {
181 			hits_stddev += (hits_mean - res[i].hits / 1000000.0) *
182 				       (hits_mean - res[i].hits / 1000000.0) /
183 				       (res_cnt - 1.0);
184 			important_hits_stddev +=
185 				       (important_hits_mean - res[i].important_hits / 1000000.0) *
186 				       (important_hits_mean - res[i].important_hits / 1000000.0) /
187 				       (res_cnt - 1.0);
188 		}
189 
190 		hits_stddev = sqrt(hits_stddev);
191 		important_hits_stddev = sqrt(important_hits_stddev);
192 	}
193 	printf("Summary: hits throughput %8.3lf \u00B1 %5.3lf M ops/s, ",
194 	       hits_mean, hits_stddev);
195 	printf("hits latency %8.3lf ns/op, ", 1000.0 / hits_mean);
196 	printf("important_hits throughput %8.3lf \u00B1 %5.3lf M ops/s\n",
197 	       important_hits_mean, important_hits_stddev);
198 }
199 
200 const char *argp_program_version = "benchmark";
201 const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
202 const char argp_program_doc[] =
203 "benchmark    Generic benchmarking framework.\n"
204 "\n"
205 "This tool runs benchmarks.\n"
206 "\n"
207 "USAGE: benchmark <bench-name>\n"
208 "\n"
209 "EXAMPLES:\n"
210 "    # run 'count-local' benchmark with 1 producer and 1 consumer\n"
211 "    benchmark count-local\n"
212 "    # run 'count-local' with 16 producer and 8 consumer thread, pinned to CPUs\n"
213 "    benchmark -p16 -c8 -a count-local\n";
214 
215 enum {
216 	ARG_PROD_AFFINITY_SET = 1000,
217 	ARG_CONS_AFFINITY_SET = 1001,
218 };
219 
220 static const struct argp_option opts[] = {
221 	{ "list", 'l', NULL, 0, "List available benchmarks"},
222 	{ "duration", 'd', "SEC", 0, "Duration of benchmark, seconds"},
223 	{ "warmup", 'w', "SEC", 0, "Warm-up period, seconds"},
224 	{ "producers", 'p', "NUM", 0, "Number of producer threads"},
225 	{ "consumers", 'c', "NUM", 0, "Number of consumer threads"},
226 	{ "verbose", 'v', NULL, 0, "Verbose debug output"},
227 	{ "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},
228 	{ "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,
229 	  "Set of CPUs for producer threads; implies --affinity"},
230 	{ "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,
231 	  "Set of CPUs for consumer threads; implies --affinity"},
232 	{},
233 };
234 
235 extern struct argp bench_ringbufs_argp;
236 extern struct argp bench_bloom_map_argp;
237 extern struct argp bench_bpf_loop_argp;
238 extern struct argp bench_local_storage_argp;
239 extern struct argp bench_strncmp_argp;
240 
241 static const struct argp_child bench_parsers[] = {
242 	{ &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },
243 	{ &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },
244 	{ &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },
245 	{ &bench_local_storage_argp, 0, "local_storage benchmark", 0 },
246 	{ &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },
247 	{},
248 };
249 
250 static error_t parse_arg(int key, char *arg, struct argp_state *state)
251 {
252 	static int pos_args;
253 
254 	switch (key) {
255 	case 'v':
256 		env.verbose = true;
257 		break;
258 	case 'l':
259 		env.list = true;
260 		break;
261 	case 'd':
262 		env.duration_sec = strtol(arg, NULL, 10);
263 		if (env.duration_sec <= 0) {
264 			fprintf(stderr, "Invalid duration: %s\n", arg);
265 			argp_usage(state);
266 		}
267 		break;
268 	case 'w':
269 		env.warmup_sec = strtol(arg, NULL, 10);
270 		if (env.warmup_sec <= 0) {
271 			fprintf(stderr, "Invalid warm-up duration: %s\n", arg);
272 			argp_usage(state);
273 		}
274 		break;
275 	case 'p':
276 		env.producer_cnt = strtol(arg, NULL, 10);
277 		if (env.producer_cnt <= 0) {
278 			fprintf(stderr, "Invalid producer count: %s\n", arg);
279 			argp_usage(state);
280 		}
281 		break;
282 	case 'c':
283 		env.consumer_cnt = strtol(arg, NULL, 10);
284 		if (env.consumer_cnt <= 0) {
285 			fprintf(stderr, "Invalid consumer count: %s\n", arg);
286 			argp_usage(state);
287 		}
288 		break;
289 	case 'a':
290 		env.affinity = true;
291 		break;
292 	case ARG_PROD_AFFINITY_SET:
293 		env.affinity = true;
294 		if (parse_num_list(arg, &env.prod_cpus.cpus,
295 				   &env.prod_cpus.cpus_len)) {
296 			fprintf(stderr, "Invalid format of CPU set for producers.");
297 			argp_usage(state);
298 		}
299 		break;
300 	case ARG_CONS_AFFINITY_SET:
301 		env.affinity = true;
302 		if (parse_num_list(arg, &env.cons_cpus.cpus,
303 				   &env.cons_cpus.cpus_len)) {
304 			fprintf(stderr, "Invalid format of CPU set for consumers.");
305 			argp_usage(state);
306 		}
307 		break;
308 	case ARGP_KEY_ARG:
309 		if (pos_args++) {
310 			fprintf(stderr,
311 				"Unrecognized positional argument: %s\n", arg);
312 			argp_usage(state);
313 		}
314 		env.bench_name = strdup(arg);
315 		break;
316 	default:
317 		return ARGP_ERR_UNKNOWN;
318 	}
319 	return 0;
320 }
321 
322 static void parse_cmdline_args(int argc, char **argv)
323 {
324 	static const struct argp argp = {
325 		.options = opts,
326 		.parser = parse_arg,
327 		.doc = argp_program_doc,
328 		.children = bench_parsers,
329 	};
330 	if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
331 		exit(1);
332 	if (!env.list && !env.bench_name) {
333 		argp_help(&argp, stderr, ARGP_HELP_DOC, "bench");
334 		exit(1);
335 	}
336 }
337 
338 static void collect_measurements(long delta_ns);
339 
340 static __u64 last_time_ns;
341 static void sigalarm_handler(int signo)
342 {
343 	long new_time_ns = get_time_ns();
344 	long delta_ns = new_time_ns - last_time_ns;
345 
346 	collect_measurements(delta_ns);
347 
348 	last_time_ns = new_time_ns;
349 }
350 
351 /* set up periodic 1-second timer */
352 static void setup_timer()
353 {
354 	static struct sigaction sigalarm_action = {
355 		.sa_handler = sigalarm_handler,
356 	};
357 	struct itimerval timer_settings = {};
358 	int err;
359 
360 	last_time_ns = get_time_ns();
361 	err = sigaction(SIGALRM, &sigalarm_action, NULL);
362 	if (err < 0) {
363 		fprintf(stderr, "failed to install SIGALRM handler: %d\n", -errno);
364 		exit(1);
365 	}
366 	timer_settings.it_interval.tv_sec = 1;
367 	timer_settings.it_value.tv_sec = 1;
368 	err = setitimer(ITIMER_REAL, &timer_settings, NULL);
369 	if (err < 0) {
370 		fprintf(stderr, "failed to arm interval timer: %d\n", -errno);
371 		exit(1);
372 	}
373 }
374 
375 static void set_thread_affinity(pthread_t thread, int cpu)
376 {
377 	cpu_set_t cpuset;
378 
379 	CPU_ZERO(&cpuset);
380 	CPU_SET(cpu, &cpuset);
381 	if (pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset)) {
382 		fprintf(stderr, "setting affinity to CPU #%d failed: %d\n",
383 			cpu, errno);
384 		exit(1);
385 	}
386 }
387 
388 static int next_cpu(struct cpu_set *cpu_set)
389 {
390 	if (cpu_set->cpus) {
391 		int i;
392 
393 		/* find next available CPU */
394 		for (i = cpu_set->next_cpu; i < cpu_set->cpus_len; i++) {
395 			if (cpu_set->cpus[i]) {
396 				cpu_set->next_cpu = i + 1;
397 				return i;
398 			}
399 		}
400 		fprintf(stderr, "Not enough CPUs specified, need CPU #%d or higher.\n", i);
401 		exit(1);
402 	}
403 
404 	return cpu_set->next_cpu++;
405 }
406 
407 static struct bench_state {
408 	int res_cnt;
409 	struct bench_res *results;
410 	pthread_t *consumers;
411 	pthread_t *producers;
412 } state;
413 
414 const struct bench *bench = NULL;
415 
416 extern const struct bench bench_count_global;
417 extern const struct bench bench_count_local;
418 extern const struct bench bench_rename_base;
419 extern const struct bench bench_rename_kprobe;
420 extern const struct bench bench_rename_kretprobe;
421 extern const struct bench bench_rename_rawtp;
422 extern const struct bench bench_rename_fentry;
423 extern const struct bench bench_rename_fexit;
424 extern const struct bench bench_trig_base;
425 extern const struct bench bench_trig_tp;
426 extern const struct bench bench_trig_rawtp;
427 extern const struct bench bench_trig_kprobe;
428 extern const struct bench bench_trig_fentry;
429 extern const struct bench bench_trig_fentry_sleep;
430 extern const struct bench bench_trig_fmodret;
431 extern const struct bench bench_trig_uprobe_base;
432 extern const struct bench bench_trig_uprobe_with_nop;
433 extern const struct bench bench_trig_uretprobe_with_nop;
434 extern const struct bench bench_trig_uprobe_without_nop;
435 extern const struct bench bench_trig_uretprobe_without_nop;
436 extern const struct bench bench_rb_libbpf;
437 extern const struct bench bench_rb_custom;
438 extern const struct bench bench_pb_libbpf;
439 extern const struct bench bench_pb_custom;
440 extern const struct bench bench_bloom_lookup;
441 extern const struct bench bench_bloom_update;
442 extern const struct bench bench_bloom_false_positive;
443 extern const struct bench bench_hashmap_without_bloom;
444 extern const struct bench bench_hashmap_with_bloom;
445 extern const struct bench bench_bpf_loop;
446 extern const struct bench bench_strncmp_no_helper;
447 extern const struct bench bench_strncmp_helper;
448 extern const struct bench bench_bpf_hashmap_full_update;
449 extern const struct bench bench_local_storage_cache_seq_get;
450 extern const struct bench bench_local_storage_cache_interleaved_get;
451 extern const struct bench bench_local_storage_cache_hashmap_control;
452 
453 static const struct bench *benchs[] = {
454 	&bench_count_global,
455 	&bench_count_local,
456 	&bench_rename_base,
457 	&bench_rename_kprobe,
458 	&bench_rename_kretprobe,
459 	&bench_rename_rawtp,
460 	&bench_rename_fentry,
461 	&bench_rename_fexit,
462 	&bench_trig_base,
463 	&bench_trig_tp,
464 	&bench_trig_rawtp,
465 	&bench_trig_kprobe,
466 	&bench_trig_fentry,
467 	&bench_trig_fentry_sleep,
468 	&bench_trig_fmodret,
469 	&bench_trig_uprobe_base,
470 	&bench_trig_uprobe_with_nop,
471 	&bench_trig_uretprobe_with_nop,
472 	&bench_trig_uprobe_without_nop,
473 	&bench_trig_uretprobe_without_nop,
474 	&bench_rb_libbpf,
475 	&bench_rb_custom,
476 	&bench_pb_libbpf,
477 	&bench_pb_custom,
478 	&bench_bloom_lookup,
479 	&bench_bloom_update,
480 	&bench_bloom_false_positive,
481 	&bench_hashmap_without_bloom,
482 	&bench_hashmap_with_bloom,
483 	&bench_bpf_loop,
484 	&bench_strncmp_no_helper,
485 	&bench_strncmp_helper,
486 	&bench_bpf_hashmap_full_update,
487 	&bench_local_storage_cache_seq_get,
488 	&bench_local_storage_cache_interleaved_get,
489 	&bench_local_storage_cache_hashmap_control,
490 };
491 
492 static void setup_benchmark()
493 {
494 	int i, err;
495 
496 	if (!env.bench_name) {
497 		fprintf(stderr, "benchmark name is not specified\n");
498 		exit(1);
499 	}
500 
501 	for (i = 0; i < ARRAY_SIZE(benchs); i++) {
502 		if (strcmp(benchs[i]->name, env.bench_name) == 0) {
503 			bench = benchs[i];
504 			break;
505 		}
506 	}
507 	if (!bench) {
508 		fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);
509 		exit(1);
510 	}
511 
512 	printf("Setting up benchmark '%s'...\n", bench->name);
513 
514 	state.producers = calloc(env.producer_cnt, sizeof(*state.producers));
515 	state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));
516 	state.results = calloc(env.duration_sec + env.warmup_sec + 2,
517 			       sizeof(*state.results));
518 	if (!state.producers || !state.consumers || !state.results)
519 		exit(1);
520 
521 	if (bench->validate)
522 		bench->validate();
523 	if (bench->setup)
524 		bench->setup();
525 
526 	for (i = 0; i < env.consumer_cnt; i++) {
527 		err = pthread_create(&state.consumers[i], NULL,
528 				     bench->consumer_thread, (void *)(long)i);
529 		if (err) {
530 			fprintf(stderr, "failed to create consumer thread #%d: %d\n",
531 				i, -errno);
532 			exit(1);
533 		}
534 		if (env.affinity)
535 			set_thread_affinity(state.consumers[i],
536 					    next_cpu(&env.cons_cpus));
537 	}
538 
539 	/* unless explicit producer CPU list is specified, continue after
540 	 * last consumer CPU
541 	 */
542 	if (!env.prod_cpus.cpus)
543 		env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;
544 
545 	for (i = 0; i < env.producer_cnt; i++) {
546 		err = pthread_create(&state.producers[i], NULL,
547 				     bench->producer_thread, (void *)(long)i);
548 		if (err) {
549 			fprintf(stderr, "failed to create producer thread #%d: %d\n",
550 				i, -errno);
551 			exit(1);
552 		}
553 		if (env.affinity)
554 			set_thread_affinity(state.producers[i],
555 					    next_cpu(&env.prod_cpus));
556 	}
557 
558 	printf("Benchmark '%s' started.\n", bench->name);
559 }
560 
561 static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER;
562 static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER;
563 
564 static void collect_measurements(long delta_ns) {
565 	int iter = state.res_cnt++;
566 	struct bench_res *res = &state.results[iter];
567 
568 	bench->measure(res);
569 
570 	if (bench->report_progress)
571 		bench->report_progress(iter, res, delta_ns);
572 
573 	if (iter == env.duration_sec + env.warmup_sec) {
574 		pthread_mutex_lock(&bench_done_mtx);
575 		pthread_cond_signal(&bench_done);
576 		pthread_mutex_unlock(&bench_done_mtx);
577 	}
578 }
579 
580 int main(int argc, char **argv)
581 {
582 	parse_cmdline_args(argc, argv);
583 
584 	if (env.list) {
585 		int i;
586 
587 		printf("Available benchmarks:\n");
588 		for (i = 0; i < ARRAY_SIZE(benchs); i++) {
589 			printf("- %s\n", benchs[i]->name);
590 		}
591 		return 0;
592 	}
593 
594 	setup_benchmark();
595 
596 	setup_timer();
597 
598 	pthread_mutex_lock(&bench_done_mtx);
599 	pthread_cond_wait(&bench_done, &bench_done_mtx);
600 	pthread_mutex_unlock(&bench_done_mtx);
601 
602 	if (bench->report_final)
603 		/* skip first sample */
604 		bench->report_final(state.results + env.warmup_sec,
605 				    state.res_cnt - env.warmup_sec);
606 
607 	return 0;
608 }
609