1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
22 *
23 */
24
25 /*
26 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
27 * Use is subject to license terms.
28 */
29
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/conf.h>
33 #include <sys/cpuvar.h>
34 #include <sys/endian.h>
35 #include <sys/fcntl.h>
36 #include <sys/filio.h>
37 #include <sys/kdb.h>
38 #include <sys/kernel.h>
39 #include <sys/kmem.h>
40 #include <sys/kthread.h>
41 #include <sys/limits.h>
42 #include <sys/linker.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/mutex.h>
47 #include <sys/pcpu.h>
48 #include <sys/poll.h>
49 #include <sys/proc.h>
50 #include <sys/selinfo.h>
51 #include <sys/smp.h>
52 #include <sys/syscall.h>
53 #include <sys/sysent.h>
54 #include <sys/sysproto.h>
55 #include <sys/uio.h>
56 #include <sys/unistd.h>
57 #include <machine/stdarg.h>
58
59 #include <sys/dtrace.h>
60 #include <sys/dtrace_bsd.h>
61
62 #include "fbt.h"
63
64 MALLOC_DEFINE(M_FBT, "fbt", "Function Boundary Tracing");
65
66 dtrace_provider_id_t fbt_id;
67 fbt_probe_t **fbt_probetab;
68 int fbt_probetab_mask;
69
70 static int fbt_unload(void);
71 static void fbt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *);
72 static void fbt_provide_module(void *, modctl_t *);
73 static void fbt_destroy(void *, dtrace_id_t, void *);
74 static void fbt_enable(void *, dtrace_id_t, void *);
75 static void fbt_disable(void *, dtrace_id_t, void *);
76 static void fbt_load(void *);
77 static void fbt_suspend(void *, dtrace_id_t, void *);
78 static void fbt_resume(void *, dtrace_id_t, void *);
79
80 static dtrace_pattr_t fbt_attr = {
81 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
82 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
83 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
84 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
85 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
86 };
87
88 static dtrace_pops_t fbt_pops = {
89 .dtps_provide = NULL,
90 .dtps_provide_module = fbt_provide_module,
91 .dtps_enable = fbt_enable,
92 .dtps_disable = fbt_disable,
93 .dtps_suspend = fbt_suspend,
94 .dtps_resume = fbt_resume,
95 .dtps_getargdesc = fbt_getargdesc,
96 .dtps_getargval = NULL,
97 .dtps_usermode = NULL,
98 .dtps_destroy = fbt_destroy
99 };
100
101 static int fbt_probetab_size;
102 static int fbt_verbose = 0;
103
104 int
fbt_excluded(const char * name)105 fbt_excluded(const char *name)
106 {
107
108 if (strncmp(name, "dtrace_", 7) == 0 &&
109 strncmp(name, "dtrace_safe_", 12) != 0) {
110 /*
111 * Anything beginning with "dtrace_" may be called
112 * from probe context unless it explicitly indicates
113 * that it won't be called from probe context by
114 * using the prefix "dtrace_safe_".
115 */
116 return (1);
117 }
118
119 /*
120 * Omit instrumentation of functions that are probably in DDB. It
121 * makes it too hard to debug broken FBT.
122 *
123 * NB: kdb_enter() can be excluded, but its call to printf() can't be.
124 * This is generally OK since we're not yet in debugging context.
125 */
126 if (strncmp(name, "db_", 3) == 0 ||
127 strncmp(name, "kdb_", 4) == 0)
128 return (1);
129
130 /*
131 * Lock owner methods may be called from probe context.
132 */
133 if (strcmp(name, "owner_mtx") == 0 ||
134 strcmp(name, "owner_rm") == 0 ||
135 strcmp(name, "owner_rw") == 0 ||
136 strcmp(name, "owner_sx") == 0)
137 return (1);
138
139 /*
140 * The KMSAN runtime can't be instrumented safely.
141 */
142 if (strncmp(name, "__msan", 6) == 0 ||
143 strncmp(name, "kmsan_", 6) == 0)
144 return (1);
145
146 /*
147 * Stack unwinders may be called from probe context on some
148 * platforms.
149 */
150 #if defined(__aarch64__) || defined(__riscv)
151 if (strcmp(name, "unwind_frame") == 0)
152 return (1);
153 #endif
154
155 /*
156 * When DTrace is built into the kernel we need to exclude
157 * the FBT functions from instrumentation.
158 */
159 #ifndef _KLD_MODULE
160 if (strncmp(name, "fbt_", 4) == 0)
161 return (1);
162 #endif
163
164 return (0);
165 }
166
167 static void
fbt_doubletrap(void)168 fbt_doubletrap(void)
169 {
170 fbt_probe_t *fbt;
171 int i;
172
173 for (i = 0; i < fbt_probetab_size; i++) {
174 fbt = fbt_probetab[i];
175
176 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
177 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
178 }
179 }
180
181 static void
fbt_provide_module(void * arg,modctl_t * lf)182 fbt_provide_module(void *arg, modctl_t *lf)
183 {
184 char modname[MAXPATHLEN];
185 int i;
186 size_t len;
187
188 strlcpy(modname, lf->filename, sizeof(modname));
189 len = strlen(modname);
190 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0)
191 modname[len - 3] = '\0';
192
193 /*
194 * Employees of dtrace and their families are ineligible. Void
195 * where prohibited.
196 */
197 if (strcmp(modname, "dtrace") == 0)
198 return;
199
200 /*
201 * To register with DTrace, a module must list 'dtrace' as a
202 * dependency in order for the kernel linker to resolve
203 * symbols like dtrace_register(). All modules with such a
204 * dependency are ineligible for FBT tracing.
205 */
206 for (i = 0; i < lf->ndeps; i++)
207 if (strncmp(lf->deps[i]->filename, "dtrace", 6) == 0)
208 return;
209
210 if (lf->fbt_nentries) {
211 /*
212 * This module has some FBT entries allocated; we're afraid
213 * to screw with it.
214 */
215 return;
216 }
217
218 /*
219 * List the functions in the module and the symbol values.
220 */
221 (void) linker_file_function_listall(lf, fbt_provide_module_function, modname);
222 }
223
224 static void
fbt_destroy_one(fbt_probe_t * fbt)225 fbt_destroy_one(fbt_probe_t *fbt)
226 {
227 fbt_probe_t *hash, *hashprev, *next;
228 int ndx;
229
230 ndx = FBT_ADDR2NDX(fbt->fbtp_patchpoint);
231 for (hash = fbt_probetab[ndx], hashprev = NULL; hash != NULL;
232 hashprev = hash, hash = hash->fbtp_hashnext) {
233 if (hash == fbt) {
234 if ((next = fbt->fbtp_tracenext) != NULL)
235 next->fbtp_hashnext = hash->fbtp_hashnext;
236 else
237 next = hash->fbtp_hashnext;
238 if (hashprev != NULL)
239 hashprev->fbtp_hashnext = next;
240 else
241 fbt_probetab[ndx] = next;
242 goto free;
243 } else if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
244 for (next = hash; next->fbtp_tracenext != NULL;
245 next = next->fbtp_tracenext) {
246 if (fbt == next->fbtp_tracenext) {
247 next->fbtp_tracenext =
248 fbt->fbtp_tracenext;
249 goto free;
250 }
251 }
252 }
253 }
254 panic("probe %p not found in hash table", fbt);
255 free:
256 free(fbt, M_FBT);
257 }
258
259 static void
fbt_destroy(void * arg,dtrace_id_t id,void * parg)260 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
261 {
262 fbt_probe_t *fbt = parg, *next;
263 modctl_t *ctl;
264
265 do {
266 ctl = fbt->fbtp_ctl;
267 ctl->fbt_nentries--;
268
269 next = fbt->fbtp_probenext;
270 fbt_destroy_one(fbt);
271 fbt = next;
272 } while (fbt != NULL);
273 }
274
275 static void
fbt_enable(void * arg,dtrace_id_t id,void * parg)276 fbt_enable(void *arg, dtrace_id_t id, void *parg)
277 {
278 fbt_probe_t *fbt = parg;
279 modctl_t *ctl = fbt->fbtp_ctl;
280
281 ctl->nenabled++;
282
283 /*
284 * Now check that our modctl has the expected load count. If it
285 * doesn't, this module must have been unloaded and reloaded -- and
286 * we're not going to touch it.
287 */
288 if (ctl->loadcnt != fbt->fbtp_loadcnt) {
289 if (fbt_verbose) {
290 printf("fbt is failing for probe %s "
291 "(module %s reloaded)",
292 fbt->fbtp_name, ctl->filename);
293 }
294
295 return;
296 }
297
298 for (; fbt != NULL; fbt = fbt->fbtp_probenext) {
299 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval);
300 fbt->fbtp_enabled++;
301 }
302 }
303
304 static void
fbt_disable(void * arg,dtrace_id_t id,void * parg)305 fbt_disable(void *arg, dtrace_id_t id, void *parg)
306 {
307 fbt_probe_t *fbt = parg, *hash;
308 modctl_t *ctl = fbt->fbtp_ctl;
309
310 ASSERT(ctl->nenabled > 0);
311 ctl->nenabled--;
312
313 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
314 return;
315
316 for (; fbt != NULL; fbt = fbt->fbtp_probenext) {
317 fbt->fbtp_enabled--;
318
319 for (hash = fbt_probetab[FBT_ADDR2NDX(fbt->fbtp_patchpoint)];
320 hash != NULL; hash = hash->fbtp_hashnext) {
321 if (hash->fbtp_patchpoint == fbt->fbtp_patchpoint) {
322 for (; hash != NULL; hash = hash->fbtp_tracenext)
323 if (hash->fbtp_enabled > 0)
324 break;
325 break;
326 }
327 }
328 if (hash == NULL)
329 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
330 }
331 }
332
333 static void
fbt_suspend(void * arg,dtrace_id_t id,void * parg)334 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
335 {
336 fbt_probe_t *fbt = parg;
337 modctl_t *ctl = fbt->fbtp_ctl;
338
339 ASSERT(ctl->nenabled > 0);
340
341 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
342 return;
343
344 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
345 fbt_patch_tracepoint(fbt, fbt->fbtp_savedval);
346 }
347
348 static void
fbt_resume(void * arg,dtrace_id_t id,void * parg)349 fbt_resume(void *arg, dtrace_id_t id, void *parg)
350 {
351 fbt_probe_t *fbt = parg;
352 modctl_t *ctl = fbt->fbtp_ctl;
353
354 ASSERT(ctl->nenabled > 0);
355
356 if ((ctl->loadcnt != fbt->fbtp_loadcnt))
357 return;
358
359 for (; fbt != NULL; fbt = fbt->fbtp_probenext)
360 fbt_patch_tracepoint(fbt, fbt->fbtp_patchval);
361 }
362
363 static int
fbt_ctfoff_init(modctl_t * lf,linker_ctf_t * lc)364 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
365 {
366 const Elf_Sym *symp = lc->symtab;
367 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
368 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
369 size_t idwidth;
370 int i;
371 uint32_t *ctfoff;
372 uint32_t objtoff = hp->cth_objtoff;
373 uint32_t funcoff = hp->cth_funcoff;
374 uint_t kind, info, vlen;
375
376 /* Sanity check. */
377 if (hp->cth_magic != CTF_MAGIC) {
378 printf("Bad magic value in CTF data of '%s'\n",lf->pathname);
379 return (EINVAL);
380 }
381
382 if (lc->symtab == NULL) {
383 printf("No symbol table in '%s'\n",lf->pathname);
384 return (EINVAL);
385 }
386
387 ctfoff = malloc(sizeof(uint32_t) * lc->nsym, M_LINKER, M_WAITOK);
388 *lc->ctfoffp = ctfoff;
389
390 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4;
391
392 for (i = 0; i < lc->nsym; i++, ctfoff++, symp++) {
393 if (symp->st_name == 0 || symp->st_shndx == SHN_UNDEF) {
394 *ctfoff = 0xffffffff;
395 continue;
396 }
397
398 switch (ELF_ST_TYPE(symp->st_info)) {
399 case STT_OBJECT:
400 if (objtoff >= hp->cth_funcoff ||
401 (symp->st_shndx == SHN_ABS && symp->st_value == 0)) {
402 *ctfoff = 0xffffffff;
403 break;
404 }
405
406 *ctfoff = objtoff;
407 objtoff += idwidth;
408 break;
409
410 case STT_FUNC:
411 if (funcoff >= hp->cth_typeoff) {
412 *ctfoff = 0xffffffff;
413 break;
414 }
415
416 *ctfoff = funcoff;
417
418 info = 0;
419 memcpy(&info, ctfdata + funcoff, idwidth);
420 if (hp->cth_version == CTF_VERSION_2) {
421 kind = CTF_V2_INFO_KIND(info);
422 vlen = CTF_V2_INFO_VLEN(info);
423 } else {
424 kind = CTF_V3_INFO_KIND(info);
425 vlen = CTF_V3_INFO_VLEN(info);
426 }
427
428 /*
429 * If we encounter a zero pad at the end, just skip it.
430 * Otherwise skip over the function and its return type
431 * (+2) and the argument list (vlen).
432 */
433 if (kind == CTF_K_UNKNOWN && vlen == 0)
434 funcoff += idwidth;
435 else
436 funcoff += idwidth * (vlen + 2);
437 break;
438
439 default:
440 *ctfoff = 0xffffffff;
441 break;
442 }
443 }
444
445 return (0);
446 }
447
448 static void
fbt_get_ctt_index(uint8_t version,const void * v,uint_t * indexp,uint_t * typep,int * ischildp)449 fbt_get_ctt_index(uint8_t version, const void *v, uint_t *indexp,
450 uint_t *typep, int *ischildp)
451 {
452 uint_t index, type;
453 int ischild;
454
455 if (version == CTF_VERSION_2) {
456 const struct ctf_type_v2 *ctt = v;
457
458 type = ctt->ctt_type;
459 index = CTF_V2_TYPE_TO_INDEX(ctt->ctt_type);
460 ischild = CTF_V2_TYPE_ISCHILD(ctt->ctt_type);
461 } else {
462 const struct ctf_type_v3 *ctt = v;
463
464 type = ctt->ctt_type;
465 index = CTF_V3_TYPE_TO_INDEX(ctt->ctt_type);
466 ischild = CTF_V3_TYPE_ISCHILD(ctt->ctt_type);
467 }
468
469 if (indexp != NULL)
470 *indexp = index;
471 if (typep != NULL)
472 *typep = type;
473 if (ischildp != NULL)
474 *ischildp = ischild;
475 }
476
477 static ssize_t
fbt_get_ctt_size(uint8_t version,const void * tp,ssize_t * sizep,ssize_t * incrementp)478 fbt_get_ctt_size(uint8_t version, const void *tp, ssize_t *sizep,
479 ssize_t *incrementp)
480 {
481 ssize_t size, increment;
482
483 if (version == CTF_VERSION_2) {
484 const struct ctf_type_v2 *ctt = tp;
485
486 if (ctt->ctt_size == CTF_V2_LSIZE_SENT) {
487 size = CTF_TYPE_LSIZE(ctt);
488 increment = sizeof (struct ctf_type_v2);
489 } else {
490 size = ctt->ctt_size;
491 increment = sizeof (struct ctf_stype_v2);
492 }
493 } else {
494 const struct ctf_type_v3 *ctt = tp;
495
496 if (ctt->ctt_size == CTF_V3_LSIZE_SENT) {
497 size = CTF_TYPE_LSIZE(ctt);
498 increment = sizeof (struct ctf_type_v3);
499 } else {
500 size = ctt->ctt_size;
501 increment = sizeof (struct ctf_stype_v3);
502 }
503 }
504
505 if (sizep)
506 *sizep = size;
507 if (incrementp)
508 *incrementp = increment;
509
510 return (size);
511 }
512
513 static void
fbt_get_ctt_info(uint8_t version,const void * tp,uint_t * kindp,uint_t * vlenp,int * isrootp)514 fbt_get_ctt_info(uint8_t version, const void *tp, uint_t *kindp, uint_t *vlenp,
515 int *isrootp)
516 {
517 uint_t kind, vlen;
518 int isroot;
519
520 if (version == CTF_VERSION_2) {
521 const struct ctf_type_v2 *ctt = tp;
522
523 kind = CTF_V2_INFO_KIND(ctt->ctt_info);
524 vlen = CTF_V2_INFO_VLEN(ctt->ctt_info);
525 isroot = CTF_V2_INFO_ISROOT(ctt->ctt_info);
526 } else {
527 const struct ctf_type_v3 *ctt = tp;
528
529 kind = CTF_V3_INFO_KIND(ctt->ctt_info);
530 vlen = CTF_V3_INFO_VLEN(ctt->ctt_info);
531 isroot = CTF_V3_INFO_ISROOT(ctt->ctt_info);
532 }
533
534 if (kindp != NULL)
535 *kindp = kind;
536 if (vlenp != NULL)
537 *vlenp = vlen;
538 if (isrootp != NULL)
539 *isrootp = isroot;
540 }
541
542 static int
fbt_typoff_init(linker_ctf_t * lc)543 fbt_typoff_init(linker_ctf_t *lc)
544 {
545 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
546 const void *tbuf, *tend, *tp;
547 const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
548 size_t idwidth;
549 int ctf_typemax = 0;
550 uint32_t *xp;
551 ulong_t pop[CTF_K_MAX + 1] = { 0 };
552 uint8_t version;
553
554 /* Sanity check. */
555 if (hp->cth_magic != CTF_MAGIC)
556 return (EINVAL);
557
558 version = hp->cth_version;
559 idwidth = version == CTF_VERSION_2 ? 2 : 4;
560
561 tbuf = (const void *) (ctfdata + hp->cth_typeoff);
562 tend = (const void *) (ctfdata + hp->cth_stroff);
563
564 /*
565 * We make two passes through the entire type section. In this first
566 * pass, we count the number of each type and the total number of types.
567 */
568 for (tp = tbuf; tp < tend; ctf_typemax++) {
569 uint_t kind, type, vlen;
570 ssize_t size, increment;
571 size_t vbytes;
572
573 (void) fbt_get_ctt_size(version, tp, &size, &increment);
574 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL);
575 fbt_get_ctt_index(version, tp, NULL, &type, NULL);
576
577 switch (kind) {
578 case CTF_K_INTEGER:
579 case CTF_K_FLOAT:
580 vbytes = sizeof (uint_t);
581 break;
582 case CTF_K_ARRAY:
583 if (version == CTF_VERSION_2)
584 vbytes = sizeof (struct ctf_array_v2);
585 else
586 vbytes = sizeof (struct ctf_array_v3);
587 break;
588 case CTF_K_FUNCTION:
589 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t));
590 break;
591 case CTF_K_STRUCT:
592 case CTF_K_UNION:
593 if (version == CTF_VERSION_2) {
594 if (size < CTF_V2_LSTRUCT_THRESH)
595 vbytes =
596 sizeof (struct ctf_member_v2) * vlen;
597 else
598 vbytes =
599 sizeof (struct ctf_lmember_v2) * vlen;
600 } else {
601 if (size < CTF_V3_LSTRUCT_THRESH)
602 vbytes =
603 sizeof (struct ctf_member_v3) * vlen;
604 else
605 vbytes =
606 sizeof (struct ctf_lmember_v3) * vlen;
607 }
608 break;
609 case CTF_K_ENUM:
610 vbytes = sizeof (ctf_enum_t) * vlen;
611 break;
612 case CTF_K_FORWARD:
613 /*
614 * For forward declarations, ctt_type is the CTF_K_*
615 * kind for the tag, so bump that population count too.
616 * If ctt_type is unknown, treat the tag as a struct.
617 */
618 if (type == CTF_K_UNKNOWN || type >= CTF_K_MAX)
619 pop[CTF_K_STRUCT]++;
620 else
621 pop[type]++;
622 /*FALLTHRU*/
623 case CTF_K_UNKNOWN:
624 vbytes = 0;
625 break;
626 case CTF_K_POINTER:
627 case CTF_K_TYPEDEF:
628 case CTF_K_VOLATILE:
629 case CTF_K_CONST:
630 case CTF_K_RESTRICT:
631 vbytes = 0;
632 break;
633 default:
634 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
635 return (EIO);
636 }
637 tp = (const void *)((uintptr_t)tp + increment + vbytes);
638 pop[kind]++;
639 }
640
641 /* account for a sentinel value below */
642 ctf_typemax++;
643 *lc->typlenp = ctf_typemax;
644
645 xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER,
646 M_ZERO | M_WAITOK);
647
648 *lc->typoffp = xp;
649
650 /* type id 0 is used as a sentinel value */
651 *xp++ = 0;
652
653 /*
654 * In the second pass, fill in the type offset.
655 */
656 for (tp = tbuf; tp < tend; xp++) {
657 ssize_t size, increment;
658 uint_t kind, vlen;
659
660 size_t vbytes;
661
662 (void) fbt_get_ctt_size(version, tp, &size, &increment);
663 fbt_get_ctt_info(version, tp, &kind, &vlen, NULL);
664
665 switch (kind) {
666 case CTF_K_INTEGER:
667 case CTF_K_FLOAT:
668 vbytes = sizeof (uint_t);
669 break;
670 case CTF_K_ARRAY:
671 if (version == CTF_VERSION_2)
672 vbytes = sizeof (struct ctf_array_v2);
673 else
674 vbytes = sizeof (struct ctf_array_v3);
675 break;
676 case CTF_K_FUNCTION:
677 vbytes = roundup2(idwidth * vlen, sizeof(uint32_t));
678 break;
679 case CTF_K_STRUCT:
680 case CTF_K_UNION:
681 if (version == CTF_VERSION_2) {
682 if (size < CTF_V2_LSTRUCT_THRESH)
683 vbytes =
684 sizeof (struct ctf_member_v2) * vlen;
685 else
686 vbytes =
687 sizeof (struct ctf_lmember_v2) * vlen;
688 } else {
689 if (size < CTF_V3_LSTRUCT_THRESH)
690 vbytes =
691 sizeof (struct ctf_member_v3) * vlen;
692 else
693 vbytes =
694 sizeof (struct ctf_lmember_v3) * vlen;
695 }
696 break;
697 case CTF_K_ENUM:
698 vbytes = sizeof (ctf_enum_t) * vlen;
699 break;
700 case CTF_K_FORWARD:
701 case CTF_K_UNKNOWN:
702 vbytes = 0;
703 break;
704 case CTF_K_POINTER:
705 case CTF_K_TYPEDEF:
706 case CTF_K_VOLATILE:
707 case CTF_K_CONST:
708 case CTF_K_RESTRICT:
709 vbytes = 0;
710 break;
711 default:
712 printf("%s(%d): detected invalid CTF kind -- %u\n", __func__, __LINE__, kind);
713 return (EIO);
714 }
715 *xp = (uint32_t)((uintptr_t) tp - (uintptr_t) ctfdata);
716 tp = (const void *)((uintptr_t)tp + increment + vbytes);
717 }
718
719 return (0);
720 }
721
722 /*
723 * CTF Declaration Stack
724 *
725 * In order to implement ctf_type_name(), we must convert a type graph back
726 * into a C type declaration. Unfortunately, a type graph represents a storage
727 * class ordering of the type whereas a type declaration must obey the C rules
728 * for operator precedence, and the two orderings are frequently in conflict.
729 * For example, consider these CTF type graphs and their C declarations:
730 *
731 * CTF_K_POINTER -> CTF_K_FUNCTION -> CTF_K_INTEGER : int (*)()
732 * CTF_K_POINTER -> CTF_K_ARRAY -> CTF_K_INTEGER : int (*)[]
733 *
734 * In each case, parentheses are used to raise operator * to higher lexical
735 * precedence, so the string form of the C declaration cannot be constructed by
736 * walking the type graph links and forming the string from left to right.
737 *
738 * The functions in this file build a set of stacks from the type graph nodes
739 * corresponding to the C operator precedence levels in the appropriate order.
740 * The code in ctf_type_name() can then iterate over the levels and nodes in
741 * lexical precedence order and construct the final C declaration string.
742 */
743 typedef struct ctf_list {
744 struct ctf_list *l_prev; /* previous pointer or tail pointer */
745 struct ctf_list *l_next; /* next pointer or head pointer */
746 } ctf_list_t;
747
748 #define ctf_list_prev(elem) ((void *)(((ctf_list_t *)(elem))->l_prev))
749 #define ctf_list_next(elem) ((void *)(((ctf_list_t *)(elem))->l_next))
750
751 typedef enum {
752 CTF_PREC_BASE,
753 CTF_PREC_POINTER,
754 CTF_PREC_ARRAY,
755 CTF_PREC_FUNCTION,
756 CTF_PREC_MAX
757 } ctf_decl_prec_t;
758
759 typedef struct ctf_decl_node {
760 ctf_list_t cd_list; /* linked list pointers */
761 ctf_id_t cd_type; /* type identifier */
762 uint_t cd_kind; /* type kind */
763 uint_t cd_n; /* type dimension if array */
764 } ctf_decl_node_t;
765
766 typedef struct ctf_decl {
767 ctf_list_t cd_nodes[CTF_PREC_MAX]; /* declaration node stacks */
768 int cd_order[CTF_PREC_MAX]; /* storage order of decls */
769 ctf_decl_prec_t cd_qualp; /* qualifier precision */
770 ctf_decl_prec_t cd_ordp; /* ordered precision */
771 char *cd_buf; /* buffer for output */
772 char *cd_ptr; /* buffer location */
773 char *cd_end; /* buffer limit */
774 size_t cd_len; /* buffer space required */
775 int cd_err; /* saved error value */
776 } ctf_decl_t;
777
778 /*
779 * Simple doubly-linked list append routine. This implementation assumes that
780 * each list element contains an embedded ctf_list_t as the first member.
781 * An additional ctf_list_t is used to store the head (l_next) and tail
782 * (l_prev) pointers. The current head and tail list elements have their
783 * previous and next pointers set to NULL, respectively.
784 */
785 static void
ctf_list_append(ctf_list_t * lp,void * new)786 ctf_list_append(ctf_list_t *lp, void *new)
787 {
788 ctf_list_t *p = lp->l_prev; /* p = tail list element */
789 ctf_list_t *q = new; /* q = new list element */
790
791 lp->l_prev = q;
792 q->l_prev = p;
793 q->l_next = NULL;
794
795 if (p != NULL)
796 p->l_next = q;
797 else
798 lp->l_next = q;
799 }
800
801 /*
802 * Prepend the specified existing element to the given ctf_list_t. The
803 * existing pointer should be pointing at a struct with embedded ctf_list_t.
804 */
805 static void
ctf_list_prepend(ctf_list_t * lp,void * new)806 ctf_list_prepend(ctf_list_t *lp, void *new)
807 {
808 ctf_list_t *p = new; /* p = new list element */
809 ctf_list_t *q = lp->l_next; /* q = head list element */
810
811 lp->l_next = p;
812 p->l_prev = NULL;
813 p->l_next = q;
814
815 if (q != NULL)
816 q->l_prev = p;
817 else
818 lp->l_prev = p;
819 }
820
821 static void
ctf_decl_init(ctf_decl_t * cd,char * buf,size_t len)822 ctf_decl_init(ctf_decl_t *cd, char *buf, size_t len)
823 {
824 int i;
825
826 bzero(cd, sizeof (ctf_decl_t));
827
828 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++)
829 cd->cd_order[i] = CTF_PREC_BASE - 1;
830
831 cd->cd_qualp = CTF_PREC_BASE;
832 cd->cd_ordp = CTF_PREC_BASE;
833
834 cd->cd_buf = buf;
835 cd->cd_ptr = buf;
836 cd->cd_end = buf + len;
837 }
838
839 static void
ctf_decl_fini(ctf_decl_t * cd)840 ctf_decl_fini(ctf_decl_t *cd)
841 {
842 ctf_decl_node_t *cdp, *ndp;
843 int i;
844
845 for (i = CTF_PREC_BASE; i < CTF_PREC_MAX; i++) {
846 for (cdp = ctf_list_next(&cd->cd_nodes[i]);
847 cdp != NULL; cdp = ndp) {
848 ndp = ctf_list_next(cdp);
849 free(cdp, M_FBT);
850 }
851 }
852 }
853
854 static const void *
ctf_lookup_by_id(linker_ctf_t * lc,ctf_id_t type)855 ctf_lookup_by_id(linker_ctf_t *lc, ctf_id_t type)
856 {
857 const void *tp;
858 uint32_t offset;
859 uint32_t *typoff = *lc->typoffp;
860
861 if (type >= *lc->typlenp) {
862 printf("%s(%d): type %d exceeds max %ld\n",__func__,__LINE__,(int) type,*lc->typlenp);
863 return(NULL);
864 }
865
866 /* Check if the type isn't cross-referenced. */
867 if ((offset = typoff[type]) == 0) {
868 printf("%s(%d): type %d isn't cross referenced\n",__func__,__LINE__, (int) type);
869 return(NULL);
870 }
871
872 tp = (const void *) (lc->ctftab + offset + sizeof(ctf_header_t));
873
874 return (tp);
875 }
876
877 static void
fbt_array_info(linker_ctf_t * lc,ctf_id_t type,ctf_arinfo_t * arp)878 fbt_array_info(linker_ctf_t *lc, ctf_id_t type, ctf_arinfo_t *arp)
879 {
880 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
881 const void *tp;
882 ssize_t increment;
883 uint_t kind;
884
885 bzero(arp, sizeof(*arp));
886
887 if ((tp = ctf_lookup_by_id(lc, type)) == NULL)
888 return;
889
890 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL);
891 if (kind != CTF_K_ARRAY)
892 return;
893
894 (void) fbt_get_ctt_size(hp->cth_version, tp, NULL, &increment);
895
896 if (hp->cth_version == CTF_VERSION_2) {
897 const struct ctf_array_v2 *ap;
898
899 ap = (const struct ctf_array_v2 *)((uintptr_t)tp + increment);
900 arp->ctr_contents = ap->cta_contents;
901 arp->ctr_index = ap->cta_index;
902 arp->ctr_nelems = ap->cta_nelems;
903 } else {
904 const struct ctf_array_v3 *ap;
905
906 ap = (const struct ctf_array_v3 *)((uintptr_t)tp + increment);
907 arp->ctr_contents = ap->cta_contents;
908 arp->ctr_index = ap->cta_index;
909 arp->ctr_nelems = ap->cta_nelems;
910 }
911 }
912
913 static const char *
ctf_strptr(linker_ctf_t * lc,int name)914 ctf_strptr(linker_ctf_t *lc, int name)
915 {
916 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
917 const char *strp = "";
918
919 if (name < 0 || name >= hp->cth_strlen)
920 return(strp);
921
922 strp = (const char *)(lc->ctftab + hp->cth_stroff + name + sizeof(ctf_header_t));
923
924 return (strp);
925 }
926
927 static const char *
ctf_type_rname(linker_ctf_t * lc,const void * v)928 ctf_type_rname(linker_ctf_t *lc, const void *v)
929 {
930 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
931 uint_t name;
932
933 if (hp->cth_version == CTF_VERSION_2) {
934 const struct ctf_type_v2 *ctt = v;
935
936 name = ctt->ctt_name;
937 } else {
938 const struct ctf_type_v3 *ctt = v;
939
940 name = ctt->ctt_name;
941 }
942
943 return (ctf_strptr(lc, name));
944 }
945
946 static void
ctf_decl_push(ctf_decl_t * cd,linker_ctf_t * lc,ctf_id_t type)947 ctf_decl_push(ctf_decl_t *cd, linker_ctf_t *lc, ctf_id_t type)
948 {
949 const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
950 ctf_decl_node_t *cdp;
951 ctf_decl_prec_t prec;
952 uint_t kind, n = 1, t;
953 int is_qual = 0;
954
955 const void *tp;
956 ctf_arinfo_t ar;
957
958 if ((tp = ctf_lookup_by_id(lc, type)) == NULL) {
959 cd->cd_err = ENOENT;
960 return;
961 }
962
963 fbt_get_ctt_info(hp->cth_version, tp, &kind, NULL, NULL);
964 fbt_get_ctt_index(hp->cth_version, tp, NULL, &t, NULL);
965
966 switch (kind) {
967 case CTF_K_ARRAY:
968 fbt_array_info(lc, type, &ar);
969 ctf_decl_push(cd, lc, ar.ctr_contents);
970 n = ar.ctr_nelems;
971 prec = CTF_PREC_ARRAY;
972 break;
973
974 case CTF_K_TYPEDEF:
975 if (ctf_type_rname(lc, tp)[0] == '\0') {
976 ctf_decl_push(cd, lc, t);
977 return;
978 }
979 prec = CTF_PREC_BASE;
980 break;
981
982 case CTF_K_FUNCTION:
983 ctf_decl_push(cd, lc, t);
984 prec = CTF_PREC_FUNCTION;
985 break;
986
987 case CTF_K_POINTER:
988 ctf_decl_push(cd, lc, t);
989 prec = CTF_PREC_POINTER;
990 break;
991
992 case CTF_K_VOLATILE:
993 case CTF_K_CONST:
994 case CTF_K_RESTRICT:
995 ctf_decl_push(cd, lc, t);
996 prec = cd->cd_qualp;
997 is_qual++;
998 break;
999
1000 default:
1001 prec = CTF_PREC_BASE;
1002 }
1003
1004 cdp = malloc(sizeof(*cdp), M_FBT, M_WAITOK);
1005 cdp->cd_type = type;
1006 cdp->cd_kind = kind;
1007 cdp->cd_n = n;
1008
1009 if (ctf_list_next(&cd->cd_nodes[prec]) == NULL)
1010 cd->cd_order[prec] = cd->cd_ordp++;
1011
1012 /*
1013 * Reset cd_qualp to the highest precedence level that we've seen so
1014 * far that can be qualified (CTF_PREC_BASE or CTF_PREC_POINTER).
1015 */
1016 if (prec > cd->cd_qualp && prec < CTF_PREC_ARRAY)
1017 cd->cd_qualp = prec;
1018
1019 /*
1020 * C array declarators are ordered inside out so prepend them. Also by
1021 * convention qualifiers of base types precede the type specifier (e.g.
1022 * const int vs. int const) even though the two forms are equivalent.
1023 */
1024 if (kind == CTF_K_ARRAY || (is_qual && prec == CTF_PREC_BASE))
1025 ctf_list_prepend(&cd->cd_nodes[prec], cdp);
1026 else
1027 ctf_list_append(&cd->cd_nodes[prec], cdp);
1028 }
1029
1030 static void
ctf_decl_sprintf(ctf_decl_t * cd,const char * format,...)1031 ctf_decl_sprintf(ctf_decl_t *cd, const char *format, ...)
1032 {
1033 size_t len = (size_t)(cd->cd_end - cd->cd_ptr);
1034 va_list ap;
1035 size_t n;
1036
1037 va_start(ap, format);
1038 n = vsnprintf(cd->cd_ptr, len, format, ap);
1039 va_end(ap);
1040
1041 cd->cd_ptr += MIN(n, len);
1042 cd->cd_len += n;
1043 }
1044
1045 static ssize_t
fbt_type_name(linker_ctf_t * lc,ctf_id_t type,char * buf,size_t len)1046 fbt_type_name(linker_ctf_t *lc, ctf_id_t type, char *buf, size_t len)
1047 {
1048 ctf_decl_t cd;
1049 ctf_decl_node_t *cdp;
1050 ctf_decl_prec_t prec, lp, rp;
1051 int ptr, arr;
1052 uint_t k;
1053
1054 if (lc == NULL && type == CTF_ERR)
1055 return (-1); /* simplify caller code by permitting CTF_ERR */
1056
1057 ctf_decl_init(&cd, buf, len);
1058 ctf_decl_push(&cd, lc, type);
1059
1060 if (cd.cd_err != 0) {
1061 ctf_decl_fini(&cd);
1062 return (-1);
1063 }
1064
1065 /*
1066 * If the type graph's order conflicts with lexical precedence order
1067 * for pointers or arrays, then we need to surround the declarations at
1068 * the corresponding lexical precedence with parentheses. This can
1069 * result in either a parenthesized pointer (*) as in int (*)() or
1070 * int (*)[], or in a parenthesized pointer and array as in int (*[])().
1071 */
1072 ptr = cd.cd_order[CTF_PREC_POINTER] > CTF_PREC_POINTER;
1073 arr = cd.cd_order[CTF_PREC_ARRAY] > CTF_PREC_ARRAY;
1074
1075 rp = arr ? CTF_PREC_ARRAY : ptr ? CTF_PREC_POINTER : -1;
1076 lp = ptr ? CTF_PREC_POINTER : arr ? CTF_PREC_ARRAY : -1;
1077
1078 k = CTF_K_POINTER; /* avoid leading whitespace (see below) */
1079
1080 for (prec = CTF_PREC_BASE; prec < CTF_PREC_MAX; prec++) {
1081 for (cdp = ctf_list_next(&cd.cd_nodes[prec]);
1082 cdp != NULL; cdp = ctf_list_next(cdp)) {
1083
1084 const void *tp = ctf_lookup_by_id(lc, cdp->cd_type);
1085 const char *name = ctf_type_rname(lc, tp);
1086
1087 if (k != CTF_K_POINTER && k != CTF_K_ARRAY)
1088 ctf_decl_sprintf(&cd, " ");
1089
1090 if (lp == prec) {
1091 ctf_decl_sprintf(&cd, "(");
1092 lp = -1;
1093 }
1094
1095 switch (cdp->cd_kind) {
1096 case CTF_K_INTEGER:
1097 case CTF_K_FLOAT:
1098 case CTF_K_TYPEDEF:
1099 ctf_decl_sprintf(&cd, "%s", name);
1100 break;
1101 case CTF_K_POINTER:
1102 ctf_decl_sprintf(&cd, "*");
1103 break;
1104 case CTF_K_ARRAY:
1105 ctf_decl_sprintf(&cd, "[%u]", cdp->cd_n);
1106 break;
1107 case CTF_K_FUNCTION:
1108 ctf_decl_sprintf(&cd, "()");
1109 break;
1110 case CTF_K_STRUCT:
1111 case CTF_K_FORWARD:
1112 ctf_decl_sprintf(&cd, "struct %s", name);
1113 break;
1114 case CTF_K_UNION:
1115 ctf_decl_sprintf(&cd, "union %s", name);
1116 break;
1117 case CTF_K_ENUM:
1118 ctf_decl_sprintf(&cd, "enum %s", name);
1119 break;
1120 case CTF_K_VOLATILE:
1121 ctf_decl_sprintf(&cd, "volatile");
1122 break;
1123 case CTF_K_CONST:
1124 ctf_decl_sprintf(&cd, "const");
1125 break;
1126 case CTF_K_RESTRICT:
1127 ctf_decl_sprintf(&cd, "restrict");
1128 break;
1129 }
1130
1131 k = cdp->cd_kind;
1132 }
1133
1134 if (rp == prec)
1135 ctf_decl_sprintf(&cd, ")");
1136 }
1137
1138 ctf_decl_fini(&cd);
1139 return (cd.cd_len);
1140 }
1141
1142 static void
fbt_getargdesc(void * arg __unused,dtrace_id_t id __unused,void * parg,dtrace_argdesc_t * desc)1143 fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_argdesc_t *desc)
1144 {
1145 const ctf_header_t *hp;
1146 const char *dp;
1147 fbt_probe_t *fbt = parg;
1148 linker_ctf_t lc;
1149 modctl_t *ctl = fbt->fbtp_ctl;
1150 size_t idwidth;
1151 int ndx = desc->dtargd_ndx;
1152 int symindx = fbt->fbtp_symindx;
1153 uint32_t *ctfoff;
1154 uint32_t offset, type;
1155 uint_t info, n;
1156 ushort_t kind;
1157
1158 if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) {
1159 (void) strcpy(desc->dtargd_native, "int");
1160 return;
1161 }
1162
1163 desc->dtargd_ndx = DTRACE_ARGNONE;
1164
1165 /* Get a pointer to the CTF data and it's length. */
1166 if (linker_ctf_get(ctl, &lc) != 0)
1167 /* No CTF data? Something wrong? *shrug* */
1168 return;
1169
1170 /* Check if this module hasn't been initialised yet. */
1171 if (*lc.ctfoffp == NULL) {
1172 /*
1173 * Initialise the CTF object and function symindx to
1174 * byte offset array.
1175 */
1176 if (fbt_ctfoff_init(ctl, &lc) != 0)
1177 return;
1178
1179 /* Initialise the CTF type to byte offset array. */
1180 if (fbt_typoff_init(&lc) != 0)
1181 return;
1182 }
1183
1184 ctfoff = *lc.ctfoffp;
1185
1186 if (ctfoff == NULL || *lc.typoffp == NULL)
1187 return;
1188
1189 /* Check if the symbol index is out of range. */
1190 if (symindx >= lc.nsym)
1191 return;
1192
1193 /* Check if the symbol isn't cross-referenced. */
1194 if ((offset = ctfoff[symindx]) == 0xffffffff)
1195 return;
1196
1197 hp = (const ctf_header_t *) lc.ctftab;
1198 idwidth = hp->cth_version == CTF_VERSION_2 ? 2 : 4;
1199 dp = (const char *)(lc.ctftab + offset + sizeof(ctf_header_t));
1200
1201 info = 0;
1202 memcpy(&info, dp, idwidth);
1203 dp += idwidth;
1204 if (hp->cth_version == CTF_VERSION_2) {
1205 kind = CTF_V2_INFO_KIND(info);
1206 n = CTF_V2_INFO_VLEN(info);
1207 } else {
1208 kind = CTF_V3_INFO_KIND(info);
1209 n = CTF_V3_INFO_VLEN(info);
1210 }
1211
1212 if (kind == CTF_K_UNKNOWN && n == 0) {
1213 printf("%s(%d): Unknown function!\n",__func__,__LINE__);
1214 return;
1215 }
1216
1217 if (kind != CTF_K_FUNCTION) {
1218 printf("%s(%d): Expected a function!\n",__func__,__LINE__);
1219 return;
1220 }
1221
1222 if (fbt->fbtp_roffset != 0) {
1223 /* Only return type is available for args[1] in return probe. */
1224 if (ndx > 1)
1225 return;
1226 ASSERT(ndx == 1);
1227 } else {
1228 /* Check if the requested argument doesn't exist. */
1229 if (ndx >= n)
1230 return;
1231
1232 /* Skip the return type and arguments up to the one requested. */
1233 dp += idwidth * (ndx + 1);
1234 }
1235
1236 type = 0;
1237 memcpy(&type, dp, idwidth);
1238 if (fbt_type_name(&lc, type, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0)
1239 desc->dtargd_ndx = ndx;
1240 }
1241
1242 static int
fbt_linker_file_cb(linker_file_t lf,void * arg)1243 fbt_linker_file_cb(linker_file_t lf, void *arg)
1244 {
1245
1246 fbt_provide_module(arg, lf);
1247
1248 return (0);
1249 }
1250
1251 static void
fbt_load(void * dummy)1252 fbt_load(void *dummy)
1253 {
1254 /* Default the probe table size if not specified. */
1255 if (fbt_probetab_size == 0)
1256 fbt_probetab_size = FBT_PROBETAB_SIZE;
1257
1258 /* Choose the hash mask for the probe table. */
1259 fbt_probetab_mask = fbt_probetab_size - 1;
1260
1261 /* Allocate memory for the probe table. */
1262 fbt_probetab =
1263 malloc(fbt_probetab_size * sizeof (fbt_probe_t *), M_FBT, M_WAITOK | M_ZERO);
1264
1265 dtrace_doubletrap_func = fbt_doubletrap;
1266 dtrace_invop_add(fbt_invop);
1267
1268 if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
1269 NULL, &fbt_pops, NULL, &fbt_id) != 0)
1270 return;
1271
1272 /* Create probes for the kernel and already-loaded modules. */
1273 linker_file_foreach(fbt_linker_file_cb, NULL);
1274 }
1275
1276 static int
fbt_unload(void)1277 fbt_unload(void)
1278 {
1279 int error = 0;
1280
1281 /* De-register the invalid opcode handler. */
1282 dtrace_invop_remove(fbt_invop);
1283
1284 dtrace_doubletrap_func = NULL;
1285
1286 /* De-register this DTrace provider. */
1287 if ((error = dtrace_unregister(fbt_id)) != 0)
1288 return (error);
1289
1290 /* Free the probe table. */
1291 free(fbt_probetab, M_FBT);
1292 fbt_probetab = NULL;
1293 fbt_probetab_mask = 0;
1294
1295 return (error);
1296 }
1297
1298 static int
fbt_modevent(module_t mod __unused,int type,void * data __unused)1299 fbt_modevent(module_t mod __unused, int type, void *data __unused)
1300 {
1301 int error = 0;
1302
1303 switch (type) {
1304 case MOD_LOAD:
1305 break;
1306
1307 case MOD_UNLOAD:
1308 break;
1309
1310 case MOD_SHUTDOWN:
1311 break;
1312
1313 default:
1314 error = EOPNOTSUPP;
1315 break;
1316
1317 }
1318
1319 return (error);
1320 }
1321
1322 SYSINIT(fbt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_load, NULL);
1323 SYSUNINIT(fbt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fbt_unload, NULL);
1324
1325 DEV_MODULE(fbt, fbt_modevent, NULL);
1326 MODULE_VERSION(fbt, 1);
1327 MODULE_DEPEND(fbt, dtrace, 1, 1, 1);
1328 MODULE_DEPEND(fbt, opensolaris, 1, 1, 1);
1329