1632     dtrace_ecb_t *ecb)  in dtrace_priv_probe()  argument
1634 	dtrace_probe_t *probe = ecb->dte_probe;  in dtrace_priv_probe()
1639 	ASSERT(ecb->dte_cond);  in dtrace_priv_probe()
1660 	if (ecb->dte_cond & DTRACE_COND_USERMODE) {  in dtrace_priv_probe()
1681 	if (ecb->dte_cond & DTRACE_COND_OWNER) {  in dtrace_priv_probe()
1713 	if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {  in dtrace_priv_probe()
6906 dtrace_action_breakpoint(dtrace_ecb_t *ecb)  argument
6908 	dtrace_probe_t *probe = ecb->dte_probe;
6914 	uintptr_t val = (uintptr_t)ecb;
6972 dtrace_action_panic(dtrace_ecb_t *ecb)  argument
6974 	dtrace_probe_t *probe = ecb->dte_probe;
6997 	    probe->dtpr_func, probe->dtpr_name, (void *)ecb);
7326 	dtrace_ecb_t *ecb;  local
7392 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
7393 		dtrace_predicate_t *pred = ecb->dte_predicate;
7394 		dtrace_state_t *state = ecb->dte_state;
7455 		if (ecb->dte_cond) {
7464 			if ((ecb->dte_cond & DTRACE_COND_USERMODE) &&
7482 			if (ecb->dte_cond & DTRACE_COND_OWNER) {
7485 				    ecb->dte_state->dts_cred.dcr_cred;
7502 			if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) {
7505 				    ecb->dte_state->dts_cred.dcr_cred;
7541 		if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
7542 		    ecb->dte_alignment, state, &mstate)) < 0)
7548 		if (ecb->dte_size != 0) {
7554 			ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t));
7555 			dtrh.dtrh_epid = ecb->dte_epid;
7561 		mstate.dtms_epid = ecb->dte_epid;
7590 		for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
7633 					dtrace_action_breakpoint(ecb);
7638 					dtrace_action_panic(ecb);
7719 				    ecb->dte_needed, ecb->dte_alignment,
7730 				if (ecb->dte_size == 0)
7733 				ASSERT3U(ecb->dte_size, >=,
7736 				dtrh->dtrh_epid = ecb->dte_epid;
7816 				if (ecb->dte_size)
7817 					buf->dtb_offset = offs + ecb->dte_size;
7992 			for (err = ecb->dte_action, ndx = 0;
7996 			dtrace_probe_error(state, ecb->dte_epid, ndx,
8005 			buf->dtb_offset = offs + ecb->dte_size;
11092 	dtrace_ecb_t *ecb;  local
11097 	ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
11098 	ecb->dte_predicate = NULL;
11099 	ecb->dte_probe = probe;
11105 	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t);
11106 	ecb->dte_alignment = sizeof (dtrace_epid_t);
11147 	ecb->dte_state = state;
11151 	state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
11153 	return (ecb);
11157 dtrace_ecb_enable(dtrace_ecb_t *ecb)  argument
11159 	dtrace_probe_t *probe = ecb->dte_probe;
11163 	ASSERT(ecb->dte_next == NULL);
11178 		probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
11180 		if (ecb->dte_predicate != NULL)
11181 			probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
11192 		probe->dtpr_ecb_last->dte_next = ecb;
11193 		probe->dtpr_ecb_last = ecb;
11201 dtrace_ecb_resize(dtrace_ecb_t *ecb)  argument
11211 	ecb->dte_size = sizeof (dtrace_rechdr_t);
11212 	ecb->dte_alignment = sizeof (dtrace_epid_t);
11214 	for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
11218 		ecb->dte_alignment = MAX(ecb->dte_alignment,
11237 			ecb->dte_needed = MAX(ecb->dte_needed, curneeded);
11251 				curneeded = P2PHASEUP(ecb->dte_size,
11268 			ecb->dte_size = P2ROUNDUP(ecb->dte_size,
11270 			rec->dtrd_offset = ecb->dte_size;
11271 			if (ecb->dte_size + rec->dtrd_size < ecb->dte_size)
11273 			ecb->dte_size += rec->dtrd_size;
11274 			ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size);
11278 	if ((act = ecb->dte_action) != NULL &&
11280 	    ecb->dte_size == sizeof (dtrace_rechdr_t)) {
11285 		ecb->dte_size = 0;
11288 	ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t));
11289 	ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t)));
11290 	ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed,
11291 	    ecb->dte_needed);
11296 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)  argument
11304 	dtrace_state_t *state = ecb->dte_state;
11307 	agg->dtag_ecb = ecb;
11401 	for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
11427 	ASSERT(ecb->dte_action_last != NULL);
11428 	act = ecb->dte_action_last;
11487 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)  argument
11490 	dtrace_state_t *state = ecb->dte_state;
11507 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)  argument
11514 	dtrace_state_t *state = ecb->dte_state;
11519 	ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
11528 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
11536 		action = dtrace_ecb_aggregation_create(ecb, desc);
11672 			if (ecb->dte_size > sizeof (dtrace_rechdr_t))
11686 			dtrace_action_t *act = ecb->dte_action;
11708 			dtrace_action_t *act = ecb->dte_action;
11742 	if ((last = ecb->dte_action_last) != NULL) {
11743 		ASSERT(ecb->dte_action != NULL);
11747 		ASSERT(ecb->dte_action == NULL);
11748 		ecb->dte_action = action;
11751 	ecb->dte_action_last = action;
11757 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)  argument
11759 	dtrace_action_t *act = ecb->dte_action, *next;
11760 	dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
11770 			ASSERT(next != NULL || act == ecb->dte_action_last);
11774 				dtrace_format_remove(ecb->dte_state, format);
11780 				dtrace_ecb_aggregation_destroy(ecb, act);
11787 	ecb->dte_action = NULL;
11788 	ecb->dte_action_last = NULL;
11789 	ecb->dte_size = 0;
11793 dtrace_ecb_disable(dtrace_ecb_t *ecb)  argument
11799 	dtrace_probe_t *probe = ecb->dte_probe;
11811 		if (pecb == ecb)
11819 		probe->dtpr_ecb = ecb->dte_next;
11821 		prev->dte_next = ecb->dte_next;
11824 	if (ecb == probe->dtpr_ecb_last) {
11825 		ASSERT(ecb->dte_next == NULL);
11843 		ASSERT(ecb->dte_next == NULL);
11867 		ecb->dte_next = NULL;
11872 dtrace_ecb_destroy(dtrace_ecb_t *ecb)  argument
11874 	dtrace_state_t *state = ecb->dte_state;
11877 	dtrace_epid_t epid = ecb->dte_epid;
11880 	ASSERT(ecb->dte_next == NULL);
11881 	ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
11883 	if ((pred = ecb->dte_predicate) != NULL)
11886 	dtrace_ecb_action_remove(ecb);
11888 	ASSERT(state->dts_ecbs[epid - 1] == ecb);
11891 	kmem_free(ecb, sizeof (dtrace_ecb_t));
11898 	dtrace_ecb_t *ecb;  local
11907 	ecb = dtrace_ecb_add(state, probe);
11908 	ecb->dte_uarg = desc->dted_uarg;
11912 		ecb->dte_predicate = pred;
11930 			ecb->dte_cond |= DTRACE_COND_OWNER;
11934 			ecb->dte_cond |= DTRACE_COND_ZONEOWNER;
11943 			ecb->dte_cond |= DTRACE_COND_USERMODE;
11957 			ecb->dte_action = act;
11958 			ecb->dte_action_last = cached->dte_action_last;
11959 			ecb->dte_needed = cached->dte_needed;
11960 			ecb->dte_size = cached->dte_size;
11961 			ecb->dte_alignment = cached->dte_alignment;
11964 		return (ecb);
11968 		if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
11969 			dtrace_ecb_destroy(ecb);
11974 	if ((enab->dten_error = dtrace_ecb_resize(ecb)) != 0) {
11975 		dtrace_ecb_destroy(ecb);
11979 	return (dtrace_ecb_create_cache = ecb);
11985 	dtrace_ecb_t *ecb;  local
12000 	if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
12003 	dtrace_ecb_enable(ecb);
12010 	dtrace_ecb_t *ecb;  local
12018 	ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
12669 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)  argument
12682 		enab->dten_desc[enab->dten_ndesc++] = ecb;
12703 	enab->dten_desc[enab->dten_ndesc++] = ecb;
12707 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,  argument
12720 	if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
12723 	for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
12726 	new->dted_action = ecb->dted_action;
12727 	new->dted_pred = ecb->dted_pred;
12729 	new->dted_uarg = ecb->dted_uarg;
13126 	dtrace_ecb_t *ecb;  local
13173 		while ((ecb = probe->dtpr_ecb) != NULL) {
13174 			dtrace_state_t *state = ecb->dte_state;
13187 			dtrace_ecb_disable(ecb);
13188 			ASSERT(probe->dtpr_ecb != ecb);
13189 			dtrace_ecb_destroy(ecb);
13956 	dof_ecbdesc_t *ecb;  local
13970 	ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
13971 	sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
13977 	ep->dted_uarg = ecb->dofe_uarg;
13983 	if (ecb->dofe_pred != DOF_SECIDX_NONE) {
13985 		    DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
13994 	if (ecb->dofe_actions != DOF_SECIDX_NONE) {
13996 		    DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
14934 	dtrace_ecb_t *ecb;  local
14949 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
14950 		if (ecb->dte_state != state)
14953 		state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
15402 	dtrace_ecb_t *ecb;  local
15451 			if ((ecb = state->dts_ecbs[i]) == NULL)
15454 			if (match && ecb->dte_probe != NULL) {
15455 				dtrace_probe_t *probe = ecb->dte_probe;
15462 			dtrace_ecb_disable(ecb);
15463 			dtrace_ecb_destroy(ecb);
17493 		dtrace_ecb_t *ecb;  local
17505 		if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
17510 		if (ecb->dte_probe == NULL) {
17515 		epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
17516 		epdesc.dtepd_uarg = ecb->dte_uarg;
17517 		epdesc.dtepd_size = ecb->dte_size;
17521 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
17543 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {