1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
8  * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
39  * In case of PCI-based devices, StreamID is a PCI rid.
42  * which contains per-device configuration.
44  * Stream table is a linear or 2-level walk table (this driver supports both).
50  * level 0 of page tables, ASID, etc.
67  * Register interface and Memory-based circular buffer queues are used
79  * in a producer-consumer fashion so that an output queue contains data
143 #define	Q_WRP(q, p)		((p) & (1 << (q)->size_log2))
144 #define	Q_IDX(q, p)		((p) & ((1 << (q)->size_log2) - 1))
184 		"The STE of a transaction marks non-substream transactions "
230 	if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||  in smmu_q_has_space()
231 	    Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))  in smmu_q_has_space()
241 	if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&  in smmu_q_empty()
242 	    Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))  in smmu_q_empty()
252 	if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&  in smmu_q_consumed()
253 	    (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))  in smmu_q_consumed()
256 	if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&  in smmu_q_consumed()
257 	    (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))  in smmu_q_consumed()
269 	cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;  in smmu_q_inc_cons()
270 	val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));  in smmu_q_inc_cons()
281 	prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;  in smmu_q_inc_prod()
282 	val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));  in smmu_q_inc_prod()
296 	bus_write_4(sc->res[0], reg, val);  in smmu_write_ack()
299 		v = bus_read_4(sc->res[0], reg_ack);  in smmu_write_ack()
302 	} while (timeout--);  in smmu_write_ack()
305 		device_printf(sc->dev, "Failed to write reg.\n");  in smmu_write_ack()
306 		return (-1);  in smmu_write_ack()
318 	sz = (1 << q->size_log2) * dwords * 8;  in smmu_init_queue()
321 	q->vaddr = contigmalloc(sz, M_SMMU,  in smmu_init_queue()
322 	    M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);  in smmu_init_queue()
323 	if (q->vaddr == NULL) {  in smmu_init_queue()
324 		device_printf(sc->dev, "failed to allocate %d bytes\n", sz);  in smmu_init_queue()
325 		return (-1);  in smmu_init_queue()
328 	q->prod_off = prod_off;  in smmu_init_queue()
329 	q->cons_off = cons_off;  in smmu_init_queue()
330 	q->paddr = vtophys(q->vaddr);  in smmu_init_queue()
332 	q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;  in smmu_init_queue()
333 	q->base |= q->paddr & Q_BASE_ADDR_M;  in smmu_init_queue()
334 	q->base |= q->size_log2 << Q_LOG2SIZE_S;  in smmu_init_queue()
345 	err = smmu_init_queue(sc, &sc->cmdq,  in smmu_init_queues()
351 	err = smmu_init_queue(sc, &sc->evtq,  in smmu_init_queues()
356 	if (!(sc->features & SMMU_FEATURE_PRI))  in smmu_init_queues()
360 	err = smmu_init_queue(sc, &sc->priq,  in smmu_init_queues()
379 	strtab = &sc->strtab;  in smmu_dump_ste()
381 	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {  in smmu_dump_ste()
383 		l1 = (void *)((uint64_t)strtab->vaddr +  in smmu_dump_ste()
385 		device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);  in smmu_dump_ste()
387 		l1_desc = &strtab->l1[i];  in smmu_dump_ste()
388 		ste = l1_desc->va;  in smmu_dump_ste()
392 		ste = (void *)((uint64_t)strtab->vaddr +  in smmu_dump_ste()
398 		device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);  in smmu_dump_ste()
407 	device_printf(sc->dev, "%s\n", __func__);  in smmu_dump_cd()
409 	vaddr = cd->vaddr;  in smmu_dump_cd()
411 		device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);  in smmu_dump_cd()
420 	evtq = &sc->evtq;  in smmu_evtq_dequeue()
422 	evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);  in smmu_evtq_dequeue()
423 	entry_addr = (void *)((uint64_t)evtq->vaddr +  in smmu_evtq_dequeue()
424 	    evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);  in smmu_evtq_dequeue()
426 	evtq->lc.cons = smmu_q_inc_cons(evtq);  in smmu_evtq_dequeue()
427 	bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);  in smmu_evtq_dequeue()
440 	dev = sc->dev;  in smmu_print_event()
462 		device_printf(sc->dev,  in smmu_print_event()
463 		    "Event %s (%s) received.\n", ev->str, ev->msg);  in smmu_print_event()
465 		device_printf(sc->dev, "Event 0x%x received\n", event_id);  in smmu_print_event()
467 	device_printf(sc->dev, "SID %x, Input Address: %jx\n",  in smmu_print_event()
471 		device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);  in smmu_print_event()
482 	cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;  in make_cmd()
484 	switch (entry->opcode) {  in make_cmd()
486 		cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;  in make_cmd()
487 		cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;  in make_cmd()
488 		if (entry->tlbi.leaf) {  in make_cmd()
498 		cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;  in make_cmd()
505 		cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);  in make_cmd()
508 		cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);  in make_cmd()
509 		cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);  in make_cmd()
516 		if (entry->sync.msiaddr) {  in make_cmd()
518 			cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);  in make_cmd()
523 		cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);  in make_cmd()
535 	cmdq = &sc->cmdq;  in smmu_cmdq_enqueue_cmd()
543 		cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);  in smmu_cmdq_enqueue_cmd()
547 	entry_addr = (void *)((uint64_t)cmdq->vaddr +  in smmu_cmdq_enqueue_cmd()
548 	    Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);  in smmu_cmdq_enqueue_cmd()
552 	cmdq->lc.prod = smmu_q_inc_prod(cmdq);  in smmu_cmdq_enqueue_cmd()
553 	bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);  in smmu_cmdq_enqueue_cmd()
563 		q->lc.val = bus_read_8(sc->res[0], q->prod_off);  in smmu_poll_until_consumed()
579 	q = &sc->cmdq;  in smmu_sync()
580 	prod = q->lc.prod;  in smmu_sync()
584 	cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;  in smmu_sync()
588 	base = (void *)((uint64_t)q->vaddr +  in smmu_sync()
603 	} while (timeout--);  in smmu_sync()
606 		device_printf(sc->dev, "Failed to sync\n");  in smmu_sync()
648 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid)  in smmu_tlbi_asid()  argument
652 	/* Invalidate TLB for an ASID. */  in smmu_tlbi_asid()
654 	cmd.tlbi.asid = asid;  in smmu_tlbi_asid()
660 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid)  in smmu_tlbi_va()  argument
666 	cmd.tlbi.asid = asid;  in smmu_tlbi_va()
746 	if (sc->features & SMMU_FEATURE_STALL &&  in smmu_init_ste_s1()
747 	    ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))  in smmu_init_ste_s1()
751 	val |= (cd->paddr & STE0_S1CONTEXTPTR_M);  in smmu_init_ste_s1()
777 	strtab = &sc->strtab;  in smmu_get_ste_addr()
779 	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {  in smmu_get_ste_addr()
780 		l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];  in smmu_get_ste_addr()
781 		addr = l1_desc->va;  in smmu_get_ste_addr()
782 		addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;  in smmu_get_ste_addr()
784 		addr = (void *)((uint64_t)strtab->vaddr +  in smmu_get_ste_addr()
835 	p = &domain->p;  in smmu_init_cd()
836 	cd = domain->cd = malloc(sizeof(struct smmu_cd),  in smmu_init_cd()
839 	cd->vaddr = contigmalloc(size, M_SMMU,  in smmu_init_cd()
842 	    (1ul << 40) - 1,	/* high */  in smmu_init_cd()
845 	if (cd->vaddr == NULL) {  in smmu_init_cd()
846 		device_printf(sc->dev, "Failed to allocate CD\n");  in smmu_init_cd()
850 	cd->paddr = vtophys(cd->vaddr);  in smmu_init_cd()
852 	ptr = cd->vaddr;  in smmu_init_cd()
859 	val |= (uint64_t)domain->asid << CD0_ASID_S;  in smmu_init_cd()
862 	val |= ((64 - sc->ias) << CD0_T0SZ_S);  in smmu_init_cd()
865 	paddr = p->sp_l0_paddr & CD1_TTB0_M;  in smmu_init_cd()
866 	KASSERT(paddr == p->sp_l0_paddr, ("bad allocation 1"));  in smmu_init_cd()
889 	strtab = &sc->strtab;  in smmu_init_strtab_linear()
890 	strtab->num_l1_entries = (1 << sc->sid_bits);  in smmu_init_strtab_linear()
892 	size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);  in smmu_init_strtab_linear()
895 		device_printf(sc->dev,  in smmu_init_strtab_linear()
897 		    __func__, size, strtab->num_l1_entries);  in smmu_init_strtab_linear()
899 	strtab->vaddr = contigmalloc(size, M_SMMU,  in smmu_init_strtab_linear()
902 	    (1ul << 48) - 1,	/* high */  in smmu_init_strtab_linear()
905 	if (strtab->vaddr == NULL) {  in smmu_init_strtab_linear()
906 		device_printf(sc->dev, "failed to allocate strtab\n");  in smmu_init_strtab_linear()
911 	reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;  in smmu_init_strtab_linear()
912 	strtab->base_cfg = (uint32_t)reg;  in smmu_init_strtab_linear()
914 	base = vtophys(strtab->vaddr);  in smmu_init_strtab_linear()
919 	strtab->base = reg;  in smmu_init_strtab_linear()
935 	strtab = &sc->strtab;  in smmu_init_strtab_2lvl()
937 	size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);  in smmu_init_strtab_2lvl()
938 	size = min(size, sc->sid_bits - STRTAB_SPLIT);  in smmu_init_strtab_2lvl()
939 	strtab->num_l1_entries = (1 << size);  in smmu_init_strtab_2lvl()
942 	l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);  in smmu_init_strtab_2lvl()
945 		device_printf(sc->dev,  in smmu_init_strtab_2lvl()
947 		    __func__, size, strtab->num_l1_entries, l1size);  in smmu_init_strtab_2lvl()
949 	strtab->vaddr = contigmalloc(l1size, M_SMMU,  in smmu_init_strtab_2lvl()
952 	    (1ul << 48) - 1,	/* high */  in smmu_init_strtab_2lvl()
955 	if (strtab->vaddr == NULL) {  in smmu_init_strtab_2lvl()
956 		device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");  in smmu_init_strtab_2lvl()
960 	sz = strtab->num_l1_entries * sizeof(struct l1_desc);  in smmu_init_strtab_2lvl()
962 	strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);  in smmu_init_strtab_2lvl()
967 	strtab->base_cfg = (uint32_t)reg;  in smmu_init_strtab_2lvl()
969 	base = vtophys(strtab->vaddr);  in smmu_init_strtab_2lvl()
974 	strtab->base = reg_base;  in smmu_init_strtab_2lvl()
984 	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)  in smmu_init_strtab()
1002 	strtab = &sc->strtab;  in smmu_init_l1_entry()
1003 	l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];  in smmu_init_l1_entry()
1004 	if (l1_desc->va) {  in smmu_init_l1_entry()
1011 	l1_desc->span = STRTAB_SPLIT + 1;  in smmu_init_l1_entry()
1012 	l1_desc->va = contigmalloc(size, M_SMMU,  in smmu_init_l1_entry()
1015 	    (1ul << 48) - 1,	/* high */  in smmu_init_l1_entry()
1018 	if (l1_desc->va == NULL) {  in smmu_init_l1_entry()
1019 		device_printf(sc->dev, "failed to allocate l2 entry\n");  in smmu_init_l1_entry()
1023 	l1_desc->pa = vtophys(l1_desc->va);  in smmu_init_l1_entry()
1026 	addr = (void *)((uint64_t)strtab->vaddr +  in smmu_init_l1_entry()
1030 	val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;  in smmu_init_l1_entry()
1031 	KASSERT(val == l1_desc->pa, ("bad allocation 4"));  in smmu_init_l1_entry()
1032 	val |= l1_desc->span;  in smmu_init_l1_entry()
1046 	strtab = &sc->strtab;  in smmu_deinit_l1_entry()
1049 	addr = (void *)((uint64_t)strtab->vaddr +  in smmu_deinit_l1_entry()
1053 	l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];  in smmu_deinit_l1_entry()
1054 	free(l1_desc->va, M_SMMU);  in smmu_deinit_l1_entry()
1064 	reg = bus_read_4(sc->res[0], SMMU_CR0);  in smmu_disable()
1068 		device_printf(sc->dev, "Could not disable SMMU.\n");  in smmu_disable()
1084 	} while (!smmu_q_empty(&sc->evtq));  in smmu_event_intr()
1096 	device_printf(sc->dev, "%s\n", __func__);  in smmu_sync_intr()
1108 	device_printf(sc->dev, "SMMU Global Error\n");  in smmu_gerr_intr()
1120 	bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);  in smmu_enable_interrupts()
1121 	bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);  in smmu_enable_interrupts()
1122 	bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);  in smmu_enable_interrupts()
1124 	bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);  in smmu_enable_interrupts()
1125 	bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);  in smmu_enable_interrupts()
1126 	bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);  in smmu_enable_interrupts()
1128 	if (sc->features & CR0_PRIQEN) {  in smmu_enable_interrupts()
1129 		bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);  in smmu_enable_interrupts()
1130 		bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);  in smmu_enable_interrupts()
1131 		bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);  in smmu_enable_interrupts()
1137 		device_printf(sc->dev, "Could not disable interrupts.\n");  in smmu_enable_interrupts()
1143 	if (sc->features & SMMU_FEATURE_PRI)  in smmu_enable_interrupts()
1148 		device_printf(sc->dev, "Could not enable interrupts.\n");  in smmu_enable_interrupts()
1165 	if (data->type == INTR_MAP_DATA_ACPI) {  in smmu_configure_intr()
1167 		ad->trig = INTR_TRIGGER_EDGE;  in smmu_configure_intr()
1168 		ad->pol = INTR_POLARITY_HIGH;  in smmu_configure_intr()
1179 	dev = sc->dev;  in smmu_setup_interrupts()
1186 	smmu_configure_intr(sc, sc->res[1]);  in smmu_setup_interrupts()
1188 	smmu_configure_intr(sc, sc->res[3]);  in smmu_setup_interrupts()
1189 	smmu_configure_intr(sc, sc->res[4]);  in smmu_setup_interrupts()
1192 	error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,  in smmu_setup_interrupts()
1193 	    smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);  in smmu_setup_interrupts()
1199 	error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC,  in smmu_setup_interrupts()
1200 	    smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);  in smmu_setup_interrupts()
1217 	reg = bus_read_4(sc->res[0], SMMU_CR0);  in smmu_reset()
1220 		device_printf(sc->dev,  in smmu_reset()
1225 		device_printf(sc->dev,  in smmu_reset()
1229 		device_printf(sc->dev, "Could not enable interrupts.\n");  in smmu_reset()
1239 	bus_write_4(sc->res[0], SMMU_CR1, reg);  in smmu_reset()
1242 	bus_write_4(sc->res[0], SMMU_CR2, reg);  in smmu_reset()
1245 	strtab = &sc->strtab;  in smmu_reset()
1246 	bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);  in smmu_reset()
1247 	bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);  in smmu_reset()
1250 	bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);  in smmu_reset()
1251 	bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);  in smmu_reset()
1252 	bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);  in smmu_reset()
1257 		device_printf(sc->dev, "Could not enable command queue\n");  in smmu_reset()
1264 	if (sc->features & SMMU_FEATURE_HYP) {  in smmu_reset()
1273 	bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);  in smmu_reset()
1274 	bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);  in smmu_reset()
1275 	bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);  in smmu_reset()
1280 		device_printf(sc->dev, "Could not enable event queue\n");  in smmu_reset()
1284 	if (sc->features & SMMU_FEATURE_PRI) {  in smmu_reset()
1286 		bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);  in smmu_reset()
1287 		bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);  in smmu_reset()
1288 		bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);  in smmu_reset()
1293 			device_printf(sc->dev, "Could not enable PRI queue\n");  in smmu_reset()
1298 	if (sc->features & SMMU_FEATURE_ATS) {  in smmu_reset()
1302 			device_printf(sc->dev, "Could not enable ATS check.\n");  in smmu_reset()
1310 		device_printf(sc->dev, "Could not enable SMMU.\n");  in smmu_reset()
1323 	sc->features = 0;  in smmu_check_features()
1325 	reg = bus_read_4(sc->res[0], SMMU_IDR0);  in smmu_check_features()
1329 			device_printf(sc->dev,  in smmu_check_features()
1330 			    "2-level stream table supported.\n");  in smmu_check_features()
1331 		sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;  in smmu_check_features()
1336 			device_printf(sc->dev,  in smmu_check_features()
1337 			    "2-level CD table supported.\n");  in smmu_check_features()
1338 		sc->features |= SMMU_FEATURE_2_LVL_CD;  in smmu_check_features()
1344 			device_printf(sc->dev, "Mixed endianness supported.\n");  in smmu_check_features()
1345 		sc->features |= SMMU_FEATURE_TT_LE;  in smmu_check_features()
1346 		sc->features |= SMMU_FEATURE_TT_BE;  in smmu_check_features()
1350 			device_printf(sc->dev,  in smmu_check_features()
1352 		sc->features |= SMMU_FEATURE_TT_LE;  in smmu_check_features()
1356 			device_printf(sc->dev, "Big endian supported only.\n");  in smmu_check_features()
1357 		sc->features |= SMMU_FEATURE_TT_BE;  in smmu_check_features()
1360 		device_printf(sc->dev, "Unsupported endianness.\n");  in smmu_check_features()
1365 		sc->features |= SMMU_FEATURE_SEV;  in smmu_check_features()
1369 			device_printf(sc->dev, "MSI feature present.\n");  in smmu_check_features()
1370 		sc->features |= SMMU_FEATURE_MSI;  in smmu_check_features()
1375 			device_printf(sc->dev, "HYP feature present.\n");  in smmu_check_features()
1376 		sc->features |= SMMU_FEATURE_HYP;  in smmu_check_features()
1380 		sc->features |= SMMU_FEATURE_ATS;  in smmu_check_features()
1383 		sc->features |= SMMU_FEATURE_PRI;  in smmu_check_features()
1388 		sc->features |= SMMU_FEATURE_STALL_FORCE;  in smmu_check_features()
1391 		sc->features |= SMMU_FEATURE_STALL;  in smmu_check_features()
1398 			device_printf(sc->dev,  in smmu_check_features()
1400 		sc->features |= SMMU_FEATURE_S1P;  in smmu_check_features()
1404 			device_printf(sc->dev,  in smmu_check_features()
1406 		sc->features |= SMMU_FEATURE_S2P;  in smmu_check_features()
1412 		sc->ias = 40;  in smmu_check_features()
1415 		device_printf(sc->dev, "No AArch64 table format support.\n");  in smmu_check_features()
1420 		sc->asid_bits = 16;  in smmu_check_features()
1422 		sc->asid_bits = 8;  in smmu_check_features()
1425 		device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);  in smmu_check_features()
1428 		sc->vmid_bits = 16;  in smmu_check_features()
1430 		sc->vmid_bits = 8;  in smmu_check_features()
1432 	reg = bus_read_4(sc->res[0], SMMU_IDR1);  in smmu_check_features()
1435 		device_printf(sc->dev,  in smmu_check_features()
1441 	sc->cmdq.size_log2 = val;  in smmu_check_features()
1443 		device_printf(sc->dev, "CMD queue bits %d\n", val);  in smmu_check_features()
1446 	sc->evtq.size_log2 = val;  in smmu_check_features()
1448 		device_printf(sc->dev, "EVENT queue bits %d\n", val);  in smmu_check_features()
1450 	if (sc->features & SMMU_FEATURE_PRI) {  in smmu_check_features()
1452 		sc->priq.size_log2 = val;  in smmu_check_features()
1454 			device_printf(sc->dev, "PRI queue bits %d\n", val);  in smmu_check_features()
1457 	sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;  in smmu_check_features()
1458 	sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;  in smmu_check_features()
1460 	if (sc->sid_bits <= STRTAB_SPLIT)  in smmu_check_features()
1461 		sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;  in smmu_check_features()
1464 		device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);  in smmu_check_features()
1465 		device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);  in smmu_check_features()
1469 	reg = bus_read_4(sc->res[0], SMMU_IDR3);  in smmu_check_features()
1471 		sc->features |= SMMU_FEATURE_RANGE_INV;  in smmu_check_features()
1474 	reg = bus_read_4(sc->res[0], SMMU_IDR5);  in smmu_check_features()
1478 		sc->oas = 32;  in smmu_check_features()
1481 		sc->oas = 36;  in smmu_check_features()
1484 		sc->oas = 40;  in smmu_check_features()
1487 		sc->oas = 42;  in smmu_check_features()
1490 		sc->oas = 44;  in smmu_check_features()
1493 		sc->oas = 48;  in smmu_check_features()
1496 		sc->oas = 52;  in smmu_check_features()
1500 	sc->pgsizes = 0;  in smmu_check_features()
1502 		sc->pgsizes |= 64 * 1024;  in smmu_check_features()
1504 		sc->pgsizes |= 16 * 1024;  in smmu_check_features()
1506 		sc->pgsizes |= 4 * 1024;  in smmu_check_features()
1509 		sc->features |= SMMU_FEATURE_VAX;  in smmu_check_features()
1518 	sc->asid_set_size = (1 << sc->asid_bits);  in smmu_init_asids()
1519 	sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);  in smmu_init_asids()
1520 	mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);  in smmu_init_asids()
1527 	mtx_lock_spin(&sc->asid_set_mutex);  in smmu_asid_alloc()
1528 	bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);  in smmu_asid_alloc()
1529 	if (*new_asid == -1) {  in smmu_asid_alloc()
1530 		mtx_unlock_spin(&sc->asid_set_mutex);  in smmu_asid_alloc()
1533 	bit_set(sc->asid_set, *new_asid);  in smmu_asid_alloc()
1534 	mtx_unlock_spin(&sc->asid_set_mutex);  in smmu_asid_alloc()
1540 smmu_asid_free(struct smmu_softc *sc, int asid)  in smmu_asid_free()  argument
1543 	mtx_lock_spin(&sc->asid_set_mutex);  in smmu_asid_free()
1544 	bit_clear(sc->asid_set, asid);  in smmu_asid_free()
1545 	mtx_unlock_spin(&sc->asid_set_mutex);  in smmu_asid_free()
1558 	sc->dev = dev;  in smmu_attach()
1560 	mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);  in smmu_attach()
1564 		bus_release_resources(dev, smmu_spec, sc->res);  in smmu_attach()
1605 	bus_release_resources(dev, smmu_spec, sc->res);  in smmu_detach()
1617 	device_printf(sc->dev, "%s\n", __func__);  in smmu_read_ivar()
1637 	dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);  in smmu_unmap()
1640 		if (smmu_pmap_remove(&domain->p, va) == 0) {  in smmu_unmap()
1642 			smmu_tlbi_va(sc, va, domain->asid);  in smmu_unmap()
1670 	dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,  in smmu_map()
1671 	    domain->asid);  in smmu_map()
1673 	for (i = 0; size > 0; size -= PAGE_SIZE) {  in smmu_map()
1675 		error = smmu_pmap_enter(&domain->p, va, pa, prot, 0);  in smmu_map()
1678 		smmu_tlbi_va(sc, va, domain->asid);  in smmu_map()
1706 		device_printf(sc->dev,  in smmu_domain_alloc()
1707 		    "Could not allocate ASID for a new domain.\n");  in smmu_domain_alloc()
1711 	domain->asid = (uint16_t)new_asid;  in smmu_domain_alloc()
1713 	smmu_pmap_pinit(&domain->p);  in smmu_domain_alloc()
1718 		device_printf(sc->dev, "Could not initialize CD\n");  in smmu_domain_alloc()
1722 	smmu_tlbi_asid(sc, domain->asid);  in smmu_domain_alloc()
1724 	LIST_INIT(&domain->ctx_list);  in smmu_domain_alloc()
1727 	LIST_INSERT_HEAD(&unit->domain_list, domain, next);  in smmu_domain_alloc()
1730 	iodom = &domain->iodom;  in smmu_domain_alloc()
1733 	 * Use 48-bit address space regardless of VAX bit  in smmu_domain_alloc()
1734 	 * as we need 64k IOMMU_PAGE_SIZE for 52-bit space.  in smmu_domain_alloc()
1736 	iodom->end = MAXADDR_48BIT;  in smmu_domain_alloc()
1754 	cd = domain->cd;  in smmu_domain_free()
1756 	smmu_pmap_remove_pages(&domain->p);  in smmu_domain_free()
1757 	smmu_pmap_release(&domain->p);  in smmu_domain_free()
1759 	smmu_tlbi_asid(sc, domain->asid);  in smmu_domain_free()
1760 	smmu_asid_free(sc, domain->asid);  in smmu_domain_free()
1762 	free(cd->vaddr, M_SMMU);  in smmu_domain_free()
1770     struct smmu_ctx *ctx)  in smmu_set_buswide()  argument
1778 		smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass);  in smmu_set_buswide()
1805 	struct smmu_ctx *ctx;  in smmu_ctx_alloc()  local
1809 	ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO);  in smmu_ctx_alloc()
1810 	ctx->dev = child;  in smmu_ctx_alloc()
1811 	ctx->domain = domain;  in smmu_ctx_alloc()
1813 		ctx->bypass = true;  in smmu_ctx_alloc()
1816 	LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);  in smmu_ctx_alloc()
1819 	return (&ctx->ioctx);  in smmu_ctx_alloc()
1828 	struct smmu_ctx *ctx;  in smmu_ctx_init()  local
1833 	ctx = (struct smmu_ctx *)ioctx;  in smmu_ctx_init()
1837 	domain = ctx->domain;  in smmu_ctx_init()
1841 	if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) {  in smmu_ctx_init()
1842 		err = smmu_pci_get_sid(ctx->dev, NULL, &sid);  in smmu_ctx_init()
1846 		ioctx->rid = pci_get_rid(dev);  in smmu_ctx_init()
1847 		ctx->sid = sid;  in smmu_ctx_init()
1848 		ctx->vendor = pci_get_vendor(ctx->dev);  in smmu_ctx_init()
1849 		ctx->device = pci_get_device(ctx->dev);  in smmu_ctx_init()
1852 	if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {  in smmu_ctx_init()
1853 		err = smmu_init_l1_entry(sc, ctx->sid);  in smmu_ctx_init()
1865 	smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);  in smmu_ctx_init()
1867 	if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class)  in smmu_ctx_init()
1868 		if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))  in smmu_ctx_init()
1869 			smmu_set_buswide(dev, domain, ctx);  in smmu_ctx_init()
1878 	struct smmu_ctx *ctx;  in smmu_ctx_free()  local
1880 	IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);  in smmu_ctx_free()
1883 	ctx = (struct smmu_ctx *)ioctx;  in smmu_ctx_free()
1885 	smmu_deinit_ste(sc, ctx->sid);  in smmu_ctx_free()
1887 	LIST_REMOVE(ctx, next);  in smmu_ctx_free()
1889 	free(ctx, M_SMMU);  in smmu_ctx_free()
1898 	struct smmu_ctx *ctx;  in smmu_ctx_lookup_by_sid()  local
1902 	unit = &sc->unit;  in smmu_ctx_lookup_by_sid()
1904 	LIST_FOREACH(domain, &unit->domain_list, next) {  in smmu_ctx_lookup_by_sid()
1905 		LIST_FOREACH(ctx, &domain->ctx_list, next) {  in smmu_ctx_lookup_by_sid()
1906 			if (ctx->sid == sid)  in smmu_ctx_lookup_by_sid()
1907 				return (ctx);  in smmu_ctx_lookup_by_sid()
1921 	struct smmu_ctx *ctx;  in smmu_ctx_lookup()  local
1925 	unit = &sc->unit;  in smmu_ctx_lookup()
1926 	iommu = &unit->iommu;  in smmu_ctx_lookup()
1930 	LIST_FOREACH(domain, &unit->domain_list, next) {  in smmu_ctx_lookup()
1931 		IOMMU_DOMAIN_LOCK(&domain->iodom);  in smmu_ctx_lookup()
1932 		LIST_FOREACH(ctx, &domain->ctx_list, next) {  in smmu_ctx_lookup()
1933 			if (ctx->dev == child) {  in smmu_ctx_lookup()
1934 				IOMMU_DOMAIN_UNLOCK(&domain->iodom);  in smmu_ctx_lookup()
1935 				return (&ctx->ioctx);  in smmu_ctx_lookup()
1938 		IOMMU_DOMAIN_UNLOCK(&domain->iodom);  in smmu_ctx_lookup()
1958 	if (xref != sc->xref)  in smmu_find()
1969 	struct smmu_ctx *ctx;  in smmu_ofw_md_data()  local
1971 	ctx = (struct smmu_ctx *)ioctx;  in smmu_ofw_md_data()
1974 		return (-1);  in smmu_ofw_md_data()
1976 	ctx->sid = cells[0];  in smmu_ofw_md_data()