299 static void idt_nt_write(struct idt_ntb_dev *ndev,  in idt_nt_write()  argument
310 	iowrite32(data, ndev->cfgspc + (ptrdiff_t)reg);  in idt_nt_write()
322 static u32 idt_nt_read(struct idt_ntb_dev *ndev, const unsigned int reg)  in idt_nt_read()  argument
332 	return ioread32(ndev->cfgspc + (ptrdiff_t)reg);  in idt_nt_read()
343 static void idt_sw_write(struct idt_ntb_dev *ndev,  in idt_sw_write()  argument
356 	spin_lock_irqsave(&ndev->gasa_lock, irqflags);  in idt_sw_write()
358 	iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);  in idt_sw_write()
360 	iowrite32(data, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);  in idt_sw_write()
362 	spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);  in idt_sw_write()
374 static u32 idt_sw_read(struct idt_ntb_dev *ndev, const unsigned int reg)  in idt_sw_read()  argument
387 	spin_lock_irqsave(&ndev->gasa_lock, irqflags);  in idt_sw_read()
389 	iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);  in idt_sw_read()
391 	data = ioread32(ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);  in idt_sw_read()
393 	spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);  in idt_sw_read()
414 static inline int idt_reg_set_bits(struct idt_ntb_dev *ndev, unsigned int reg,  in idt_reg_set_bits()  argument
426 	data = idt_nt_read(ndev, reg) | (u32)set_bits;  in idt_reg_set_bits()
427 	idt_nt_write(ndev, reg, data);  in idt_reg_set_bits()
450 static inline void idt_reg_clear_bits(struct idt_ntb_dev *ndev,  in idt_reg_clear_bits()  argument
459 	data = idt_nt_read(ndev, reg) & ~(u32)clear_bits;  in idt_reg_clear_bits()
460 	idt_nt_write(ndev, reg, data);  in idt_reg_clear_bits()
480 static int idt_scan_ports(struct idt_ntb_dev *ndev)  in idt_scan_ports()  argument
486 	data = idt_nt_read(ndev, IDT_NT_PCIELCAP);  in idt_scan_ports()
487 	ndev->port = GET_FIELD(PCIELCAP_PORTNUM, data);  in idt_scan_ports()
490 	portsts = idt_sw_read(ndev, portdata_tbl[ndev->port].sts);  in idt_scan_ports()
491 	ndev->part = GET_FIELD(SWPORTxSTS_SWPART, portsts);  in idt_scan_ports()
494 	memset(ndev->port_idx_map, -EINVAL, sizeof(ndev->port_idx_map));  in idt_scan_ports()
495 	memset(ndev->part_idx_map, -EINVAL, sizeof(ndev->part_idx_map));  in idt_scan_ports()
501 	ndev->peer_cnt = 0;  in idt_scan_ports()
502 	for (pidx = 0; pidx < ndev->swcfg->port_cnt; pidx++) {  in idt_scan_ports()
503 		port = ndev->swcfg->ports[pidx];  in idt_scan_ports()
505 		if (port == ndev->port)  in idt_scan_ports()
509 		portsts = idt_sw_read(ndev, portdata_tbl[port].sts);  in idt_scan_ports()
513 		partsts = idt_sw_read(ndev, partdata_tbl[part].sts);  in idt_scan_ports()
521 			ndev->peers[ndev->peer_cnt].port = port;  in idt_scan_ports()
522 			ndev->peers[ndev->peer_cnt].part = part;  in idt_scan_ports()
524 			ndev->port_idx_map[port] = ndev->peer_cnt;  in idt_scan_ports()
525 			ndev->part_idx_map[part] = ndev->peer_cnt;  in idt_scan_ports()
526 			ndev->peer_cnt++;  in idt_scan_ports()
530 	dev_dbg(&ndev->ntb.pdev->dev, "Local port: %hhu, num of peers: %hhu\n",  in idt_scan_ports()
531 		ndev->port, ndev->peer_cnt);  in idt_scan_ports()
534 	if (ndev->peer_cnt == 0) {  in idt_scan_ports()
535 		dev_warn(&ndev->ntb.pdev->dev, "No active peer found\n");  in idt_scan_ports()
550 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_port_number()  local
552 	return ndev->port;  in idt_ntb_port_number()
565 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_port_count()  local
567 	return ndev->peer_cnt;  in idt_ntb_peer_port_count()
579 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_port_number()  local
581 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_peer_port_number()
585 	return ndev->peers[pidx].port;  in idt_ntb_peer_port_number()
600 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_port_idx()  local
605 	return ndev->port_idx_map[port];  in idt_ntb_peer_port_idx()
621 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev);
635 static void idt_init_link(struct idt_ntb_dev *ndev)  in idt_init_link()  argument
641 	spin_lock_init(&ndev->mtbl_lock);  in idt_init_link()
644 	port_mask = ~BIT(ndev->port);  in idt_init_link()
645 	part_mask = ~BIT(ndev->part);  in idt_init_link()
646 	for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {  in idt_init_link()
647 		port_mask &= ~BIT(ndev->peers[pidx].port);  in idt_init_link()
648 		part_mask &= ~BIT(ndev->peers[pidx].part);  in idt_init_link()
652 	idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);  in idt_init_link()
653 	idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);  in idt_init_link()
654 	idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);  in idt_init_link()
657 	idt_sw_write(ndev, IDT_SW_SEPMSK, part_mask);  in idt_init_link()
660 	idt_sw_write(ndev, IDT_SW_SELINKUPMSK, port_mask);  in idt_init_link()
663 	idt_sw_write(ndev, IDT_SW_SELINKDNMSK, port_mask);  in idt_init_link()
666 	idt_sw_write(ndev, IDT_SW_SEGSIGMSK, part_mask);  in idt_init_link()
670 	idt_sw_write(ndev, IDT_SW_SEMSK, se_mask);  in idt_init_link()
672 	dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events initialized");  in idt_init_link()
681 static void idt_deinit_link(struct idt_ntb_dev *ndev)  in idt_deinit_link()  argument
684 	idt_ntb_local_link_disable(ndev);  in idt_deinit_link()
686 	dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events deinitialized");  in idt_deinit_link()
699 static void idt_se_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)  in idt_se_isr()  argument
704 	sests = idt_sw_read(ndev, IDT_SW_SESTS);  in idt_se_isr()
707 	idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);  in idt_se_isr()
708 	idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);  in idt_se_isr()
709 	idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);  in idt_se_isr()
712 	idt_nt_write(ndev, IDT_NT_NTINTSTS, IDT_NTINTSTS_SEVENT);  in idt_se_isr()
714 	dev_dbg(&ndev->ntb.pdev->dev, "SE IRQ detected %#08x (SESTS %#08x)",  in idt_se_isr()
718 	ntb_link_event(&ndev->ntb);  in idt_se_isr()
730 static void idt_ntb_local_link_enable(struct idt_ntb_dev *ndev)  in idt_ntb_local_link_enable()  argument
736 	idt_nt_write(ndev, IDT_NT_NTCTL, IDT_NTCTL_CPEN);  in idt_ntb_local_link_enable()
739 	reqid = idt_nt_read(ndev, IDT_NT_REQIDCAP);  in idt_ntb_local_link_enable()
746 		   SET_FIELD(NTMTBLDATA_PART, 0, ndev->part) |  in idt_ntb_local_link_enable()
748 	spin_lock_irqsave(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_enable()
749 	idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);  in idt_ntb_local_link_enable()
750 	idt_nt_write(ndev, IDT_NT_NTMTBLDATA, mtbldata);  in idt_ntb_local_link_enable()
751 	spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_enable()
754 	idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);  in idt_ntb_local_link_enable()
755 	idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);  in idt_ntb_local_link_enable()
767 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev)  in idt_ntb_local_link_disable()  argument
772 	idt_nt_write(ndev, IDT_NT_NTCTL, 0);  in idt_ntb_local_link_disable()
775 	spin_lock_irqsave(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_disable()
776 	idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);  in idt_ntb_local_link_disable()
777 	idt_nt_write(ndev, IDT_NT_NTMTBLDATA, 0);  in idt_ntb_local_link_disable()
778 	spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_disable()
781 	idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);  in idt_ntb_local_link_disable()
782 	idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);  in idt_ntb_local_link_disable()
798 static bool idt_ntb_local_link_is_up(struct idt_ntb_dev *ndev)  in idt_ntb_local_link_is_up()  argument
804 	data = idt_nt_read(ndev, IDT_NT_PCICMDSTS);  in idt_ntb_local_link_is_up()
809 	data = idt_nt_read(ndev, IDT_NT_NTCTL);  in idt_ntb_local_link_is_up()
814 	spin_lock_irqsave(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_is_up()
815 	idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);  in idt_ntb_local_link_is_up()
816 	data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);  in idt_ntb_local_link_is_up()
817 	spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);  in idt_ntb_local_link_is_up()
835 static bool idt_ntb_peer_link_is_up(struct idt_ntb_dev *ndev, int pidx)  in idt_ntb_peer_link_is_up()  argument
842 	port = ndev->peers[pidx].port;  in idt_ntb_peer_link_is_up()
845 	data = idt_sw_read(ndev, portdata_tbl[port].sts);  in idt_ntb_peer_link_is_up()
850 	data = idt_sw_read(ndev, portdata_tbl[port].pcicmdsts);  in idt_ntb_peer_link_is_up()
855 	data = idt_sw_read(ndev, portdata_tbl[port].ntctl);  in idt_ntb_peer_link_is_up()
860 	spin_lock_irqsave(&ndev->mtbl_lock, irqflags);  in idt_ntb_peer_link_is_up()
861 	idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->peers[pidx].part);  in idt_ntb_peer_link_is_up()
862 	data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);  in idt_ntb_peer_link_is_up()
863 	spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);  in idt_ntb_peer_link_is_up()
882 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_link_is_up()  local
889 		data = idt_nt_read(ndev, IDT_NT_PCIELCTLSTS);  in idt_ntb_link_is_up()
897 	if (!idt_ntb_local_link_is_up(ndev))  in idt_ntb_link_is_up()
902 	for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {  in idt_ntb_link_is_up()
903 		if (idt_ntb_peer_link_is_up(ndev, pidx))  in idt_ntb_link_is_up()
923 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_link_enable()  local
926 	idt_ntb_local_link_enable(ndev);  in idt_ntb_link_enable()
928 	dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link enabled");  in idt_ntb_link_enable()
943 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_link_disable()  local
946 	idt_ntb_local_link_disable(ndev);  in idt_ntb_link_disable()
948 	dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link disabled");  in idt_ntb_link_disable()
1041 static struct idt_mw_cfg *idt_scan_mws(struct idt_ntb_dev *ndev, int port,  in idt_scan_mws()  argument
1052 	mws = devm_kcalloc(&ndev->ntb.pdev->dev, IDT_MAX_NR_MWS,  in idt_scan_mws()
1064 		data = idt_sw_read(ndev, bars[bidx].setup);  in idt_scan_mws()
1123 static int idt_init_mws(struct idt_ntb_dev *ndev)  in idt_init_mws()  argument
1129 	ndev->mws = idt_scan_mws(ndev, ndev->port, &ndev->mw_cnt);  in idt_init_mws()
1130 	if (IS_ERR(ndev->mws)) {  in idt_init_mws()
1131 		dev_err(&ndev->ntb.pdev->dev,  in idt_init_mws()
1132 			"Failed to scan mws of local port %hhu", ndev->port);  in idt_init_mws()
1133 		return PTR_ERR(ndev->mws);  in idt_init_mws()
1137 	for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {  in idt_init_mws()
1138 		peer = &ndev->peers[pidx];  in idt_init_mws()
1139 		peer->mws = idt_scan_mws(ndev, peer->port, &peer->mw_cnt);  in idt_init_mws()
1141 			dev_err(&ndev->ntb.pdev->dev,  in idt_init_mws()
1148 	spin_lock_init(&ndev->lut_lock);  in idt_init_mws()
1150 	dev_dbg(&ndev->ntb.pdev->dev, "Outbound and inbound MWs initialized");  in idt_init_mws()
1168 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_mw_count()  local
1170 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_mw_count()
1173 	return ndev->peers[pidx].mw_cnt;  in idt_ntb_mw_count()
1195 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_mw_get_align()  local
1198 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_mw_get_align()
1201 	peer = &ndev->peers[pidx];  in idt_ntb_mw_get_align()
1230 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_mw_count()  local
1232 	return ndev->mw_cnt;  in idt_ntb_peer_mw_count()
1251 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_mw_get_addr()  local
1253 	if (widx < 0 || ndev->mw_cnt <= widx)  in idt_ntb_peer_mw_get_addr()
1258 		*base = pci_resource_start(ntb->pdev, ndev->mws[widx].bar) +  in idt_ntb_peer_mw_get_addr()
1259 			ndev->mws[widx].idx * ndev->mws[widx].size_max;  in idt_ntb_peer_mw_get_addr()
1263 		*size = ndev->mws[widx].size_max;  in idt_ntb_peer_mw_get_addr()
1286 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_mw_set_trans()  local
1290 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_peer_mw_set_trans()
1293 	if (widx < 0 || ndev->mw_cnt <= widx)  in idt_ntb_peer_mw_set_trans()
1300 	mw_cfg = &ndev->mws[widx];  in idt_ntb_peer_mw_set_trans()
1311 		data = idt_nt_read(ndev, bar->setup);  in idt_ntb_peer_mw_set_trans()
1312 		data = SET_FIELD(BARSETUP_TPART, data, ndev->peers[pidx].part);  in idt_ntb_peer_mw_set_trans()
1313 		idt_nt_write(ndev, bar->setup, data);  in idt_ntb_peer_mw_set_trans()
1315 		idt_nt_write(ndev, bar->ltbase, (u32)addr);  in idt_ntb_peer_mw_set_trans()
1316 		idt_nt_write(ndev, bar->utbase, (u32)(addr >> 32));  in idt_ntb_peer_mw_set_trans()
1319 		idt_nt_write(ndev, bar->limit, (u32)limit);  in idt_ntb_peer_mw_set_trans()
1321 			idt_nt_write(ndev, (bar + 1)->limit, (limit >> 32));  in idt_ntb_peer_mw_set_trans()
1327 		data = SET_FIELD(LUTUDATA_PART, 0, ndev->peers[pidx].part) |  in idt_ntb_peer_mw_set_trans()
1329 		spin_lock_irqsave(&ndev->lut_lock, irqflags);  in idt_ntb_peer_mw_set_trans()
1330 		idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);  in idt_ntb_peer_mw_set_trans()
1331 		idt_nt_write(ndev, IDT_NT_LUTLDATA, (u32)addr);  in idt_ntb_peer_mw_set_trans()
1332 		idt_nt_write(ndev, IDT_NT_LUTMDATA, (u32)(addr >> 32));  in idt_ntb_peer_mw_set_trans()
1333 		idt_nt_write(ndev, IDT_NT_LUTUDATA, data);  in idt_ntb_peer_mw_set_trans()
1334 		spin_unlock_irqrestore(&ndev->lut_lock, irqflags);  in idt_ntb_peer_mw_set_trans()
1355 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_mw_clear_trans()  local
1358 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_peer_mw_clear_trans()
1361 	if (widx < 0 || ndev->mw_cnt <= widx)  in idt_ntb_peer_mw_clear_trans()
1364 	mw_cfg = &ndev->mws[widx];  in idt_ntb_peer_mw_clear_trans()
1371 		data = idt_nt_read(ndev, bar->setup);  in idt_ntb_peer_mw_clear_trans()
1373 		idt_nt_write(ndev, bar->limit, 0);  in idt_ntb_peer_mw_clear_trans()
1375 			idt_nt_write(ndev, (bar + 1)->limit, 0);  in idt_ntb_peer_mw_clear_trans()
1382 		spin_lock_irqsave(&ndev->lut_lock, irqflags);  in idt_ntb_peer_mw_clear_trans()
1383 		idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);  in idt_ntb_peer_mw_clear_trans()
1384 		idt_nt_write(ndev, IDT_NT_LUTLDATA, 0);  in idt_ntb_peer_mw_clear_trans()
1385 		idt_nt_write(ndev, IDT_NT_LUTMDATA, 0);  in idt_ntb_peer_mw_clear_trans()
1386 		idt_nt_write(ndev, IDT_NT_LUTUDATA, 0);  in idt_ntb_peer_mw_clear_trans()
1387 		spin_unlock_irqrestore(&ndev->lut_lock, irqflags);  in idt_ntb_peer_mw_clear_trans()
1422 static void idt_db_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)  in idt_db_isr()  argument
1428 	dev_dbg(&ndev->ntb.pdev->dev, "DB IRQ detected %#08x", ntint_sts);  in idt_db_isr()
1431 	ntb_db_event(&ndev->ntb, 0);  in idt_db_isr()
1459 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_db_read()  local
1461 	return idt_nt_read(ndev, IDT_NT_INDBELLSTS);  in idt_ntb_db_read()
1479 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_db_clear()  local
1481 	idt_nt_write(ndev, IDT_NT_INDBELLSTS, (u32)db_bits);  in idt_ntb_db_clear()
1498 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_db_read_mask()  local
1500 	return idt_nt_read(ndev, IDT_NT_INDBELLMSK);  in idt_ntb_db_read_mask()
1516 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_db_set_mask()  local
1518 	return idt_reg_set_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,  in idt_ntb_db_set_mask()
1537 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_db_clear_mask()  local
1539 	idt_reg_clear_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,  in idt_ntb_db_clear_mask()
1558 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_db_set()  local
1563 	idt_nt_write(ndev, IDT_NT_OUTDBELLSET, (u32)db_bits);  in idt_ntb_peer_db_set()
1584 static void idt_init_msg(struct idt_ntb_dev *ndev)  in idt_init_msg()  argument
1590 		spin_lock_init(&ndev->msg_locks[midx]);  in idt_init_msg()
1592 	dev_dbg(&ndev->ntb.pdev->dev, "NTB Messaging initialized");  in idt_init_msg()
1605 static void idt_msg_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)  in idt_msg_isr()  argument
1611 	dev_dbg(&ndev->ntb.pdev->dev, "Message IRQ detected %#08x", ntint_sts);  in idt_msg_isr()
1614 	ntb_msg_event(&ndev->ntb);  in idt_msg_isr()
1671 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_msg_read_sts()  local
1673 	return idt_nt_read(ndev, IDT_NT_MSGSTS);  in idt_ntb_msg_read_sts()
1691 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_msg_clear_sts()  local
1693 	idt_nt_write(ndev, IDT_NT_MSGSTS, sts_bits);  in idt_ntb_msg_clear_sts()
1710 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_msg_set_mask()  local
1712 	return idt_reg_set_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,  in idt_ntb_msg_set_mask()
1728 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_msg_clear_mask()  local
1730 	idt_reg_clear_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,  in idt_ntb_msg_clear_mask()
1749 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_msg_read()  local
1758 		srcpart = idt_nt_read(ndev, ntdata_tbl.msgs[midx].src);  in idt_ntb_msg_read()
1759 		*pidx = ndev->part_idx_map[srcpart];  in idt_ntb_msg_read()
1767 	return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);  in idt_ntb_msg_read()
1786 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);  in idt_ntb_peer_msg_write()  local
1793 	if (pidx < 0 || ndev->peer_cnt <= pidx)  in idt_ntb_peer_msg_write()
1798 		    SET_FIELD(SWPxMSGCTL_PART, 0, ndev->peers[pidx].part);  in idt_ntb_peer_msg_write()
1801 	spin_lock_irqsave(&ndev->msg_locks[midx], irqflags);  in idt_ntb_peer_msg_write()
1803 	idt_sw_write(ndev, partdata_tbl[ndev->part].msgctl[midx], swpmsgctl);  in idt_ntb_peer_msg_write()
1804 	idt_nt_write(ndev, ntdata_tbl.msgs[midx].out, msg);  in idt_ntb_peer_msg_write()
1806 	spin_unlock_irqrestore(&ndev->msg_locks[midx], irqflags);  in idt_ntb_peer_msg_write()
1885 static void idt_read_temp(struct idt_ntb_dev *ndev,  in idt_read_temp()  argument
1894 				 idt_sw_read(ndev, IDT_SW_TMPSTS));  in idt_read_temp()
1898 				 idt_sw_read(ndev, IDT_SW_TMPSTS));  in idt_read_temp()
1902 				 idt_sw_read(ndev, IDT_SW_TMPSTS));  in idt_read_temp()
1907 				 idt_sw_read(ndev, IDT_SW_TMPADJ));  in idt_read_temp()
1912 				 idt_sw_read(ndev, IDT_SW_TMPSTS));  in idt_read_temp()
1926 static void idt_write_temp(struct idt_ntb_dev *ndev,  in idt_write_temp()  argument
1936 	mutex_lock(&ndev->hwmon_mtx);  in idt_write_temp()
1940 		data = SET_FIELD(TMPALARM_LTEMP, idt_sw_read(ndev, reg), fmt) &  in idt_write_temp()
1945 		data = SET_FIELD(TMPALARM_HTEMP, idt_sw_read(ndev, reg), fmt) &  in idt_write_temp()
1950 		data = SET_FIELD(TMPADJ_OFFSET, idt_sw_read(ndev, reg), fmt);  in idt_write_temp()
1956 	idt_sw_write(ndev, reg, data);  in idt_write_temp()
1959 	mutex_unlock(&ndev->hwmon_mtx);  in idt_write_temp()
1974 	struct idt_ntb_dev *ndev = dev_get_drvdata(dev);  in idt_sysfs_show_temp()  local
1978 	idt_read_temp(ndev, type, &mdeg);  in idt_sysfs_show_temp()
1996 	struct idt_ntb_dev *ndev = dev_get_drvdata(dev);  in idt_sysfs_set_temp()  local
2012 	idt_write_temp(ndev, type, mdeg);  in idt_sysfs_set_temp()
2030 	struct idt_ntb_dev *ndev = dev_get_drvdata(dev);  in idt_sysfs_reset_hist()  local
2035 	idt_write_temp(ndev, IDT_TEMP_LOW, IDT_TEMP_MAX_MDEG);  in idt_sysfs_reset_hist()
2036 	idt_write_temp(ndev, IDT_TEMP_HIGH, IDT_TEMP_MIN_MDEG);  in idt_sysfs_reset_hist()
2076 static void idt_init_temp(struct idt_ntb_dev *ndev)  in idt_init_temp()  argument
2081 	idt_sw_write(ndev, IDT_SW_TMPCTL, 0x0);  in idt_init_temp()
2084 	mutex_init(&ndev->hwmon_mtx);  in idt_init_temp()
2086 	hwmon = devm_hwmon_device_register_with_groups(&ndev->ntb.pdev->dev,  in idt_init_temp()
2087 		ndev->swcfg->name, ndev, idt_temp_groups);  in idt_init_temp()
2089 		dev_err(&ndev->ntb.pdev->dev, "Couldn't create hwmon device");  in idt_init_temp()
2093 	dev_dbg(&ndev->ntb.pdev->dev, "Temperature HWmon interface registered");  in idt_init_temp()
2121 static int idt_init_isr(struct idt_ntb_dev *ndev)  in idt_init_isr()  argument
2123 	struct pci_dev *pdev = ndev->ntb.pdev;  in idt_init_isr()
2143 					IRQF_ONESHOT, NTB_IRQNAME, ndev);  in idt_init_isr()
2150 	ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) & ~IDT_NTINTMSK_ALL;  in idt_init_isr()
2151 	idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);  in idt_init_isr()
2170 static void idt_deinit_isr(struct idt_ntb_dev *ndev)  in idt_deinit_isr()  argument
2172 	struct pci_dev *pdev = ndev->ntb.pdev;  in idt_deinit_isr()
2176 	ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) | IDT_NTINTMSK_ALL;  in idt_deinit_isr()
2177 	idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);  in idt_deinit_isr()
2180 	devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 0), ndev);  in idt_deinit_isr()
2199 	struct idt_ntb_dev *ndev = devid;  in idt_thread_isr()  local
2204 	ntint_sts = idt_nt_read(ndev, IDT_NT_NTINTSTS);  in idt_thread_isr()
2208 		idt_msg_isr(ndev, ntint_sts);  in idt_thread_isr()
2214 		idt_db_isr(ndev, ntint_sts);  in idt_thread_isr()
2220 		idt_se_isr(ndev, ntint_sts);  in idt_thread_isr()
2224 	dev_dbg(&ndev->ntb.pdev->dev, "IDT IRQs 0x%08x handled", ntint_sts);  in idt_thread_isr()
2275 static int idt_register_device(struct idt_ntb_dev *ndev)  in idt_register_device()  argument
2280 	ndev->ntb.ops = &idt_ntb_ops;  in idt_register_device()
2281 	ndev->ntb.topo = NTB_TOPO_SWITCH;  in idt_register_device()
2283 	ret = ntb_register_device(&ndev->ntb);  in idt_register_device()
2285 		dev_err(&ndev->ntb.pdev->dev, "Failed to register NTB device");  in idt_register_device()
2289 	dev_dbg(&ndev->ntb.pdev->dev, "NTB device successfully registered");  in idt_register_device()
2298 static void idt_unregister_device(struct idt_ntb_dev *ndev)  in idt_unregister_device()  argument
2301 	ntb_unregister_device(&ndev->ntb);  in idt_unregister_device()
2303 	dev_dbg(&ndev->ntb.pdev->dev, "NTB device unregistered");  in idt_unregister_device()
2333 	struct idt_ntb_dev *ndev = filp->private_data;  in idt_dbgfs_info_read()  local
2357 		"Local Port %hhu, Partition %hhu\n", ndev->port, ndev->part);  in idt_dbgfs_info_read()
2361 	for (idx = 0; idx < ndev->peer_cnt; idx++) {  in idt_dbgfs_info_read()
2364 			idx, ndev->peers[idx].port, ndev->peers[idx].part);  in idt_dbgfs_info_read()
2368 	data = idt_ntb_link_is_up(&ndev->ntb, &speed, &width);  in idt_dbgfs_info_read()
2377 		spin_lock_irqsave(&ndev->mtbl_lock, irqflags);  in idt_dbgfs_info_read()
2378 		idt_nt_write(ndev, IDT_NT_NTMTBLADDR, idx);  in idt_dbgfs_info_read()
2379 		data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);  in idt_dbgfs_info_read()
2380 		spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);  in idt_dbgfs_info_read()
2395 	for (idx = 0; idx < ndev->mw_cnt; idx += cnt) {  in idt_dbgfs_info_read()
2396 		data = ndev->mws[idx].type;  in idt_dbgfs_info_read()
2408 			idt_get_mw_name(data), ndev->mws[idx].bar);  in idt_dbgfs_info_read()
2411 			"Address align 0x%08llx, ", ndev->mws[idx].addr_align);  in idt_dbgfs_info_read()
2415 			ndev->mws[idx].size_align, ndev->mws[idx].size_max);  in idt_dbgfs_info_read()
2419 	for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {  in idt_dbgfs_info_read()
2422 			pidx, ndev->peers[pidx].port);  in idt_dbgfs_info_read()
2425 		for (idx = 0; idx < ndev->peers[pidx].mw_cnt; idx += cnt) {  in idt_dbgfs_info_read()
2426 			data = ndev->peers[pidx].mws[idx].type;  in idt_dbgfs_info_read()
2438 				ndev->peers[pidx].mws[idx].bar);  in idt_dbgfs_info_read()
2442 				ndev->peers[pidx].mws[idx].addr_align);  in idt_dbgfs_info_read()
2446 				ndev->peers[pidx].mws[idx].size_align,  in idt_dbgfs_info_read()
2447 				ndev->peers[pidx].mws[idx].size_max);  in idt_dbgfs_info_read()
2453 	data = idt_sw_read(ndev, IDT_SW_GDBELLSTS);  in idt_dbgfs_info_read()
2456 	data = idt_ntb_db_read(&ndev->ntb);  in idt_dbgfs_info_read()
2459 	data = idt_nt_read(ndev, IDT_NT_INDBELLMSK);  in idt_dbgfs_info_read()
2467 	data = idt_ntb_msg_read_sts(&ndev->ntb);  in idt_dbgfs_info_read()
2470 	data = idt_nt_read(ndev, IDT_NT_MSGSTSMSK);  in idt_dbgfs_info_read()
2477 		data = idt_ntb_msg_read(&ndev->ntb, &src, idx);  in idt_dbgfs_info_read()
2480 			idx, data, src, ndev->peers[src].port);  in idt_dbgfs_info_read()
2485 	idt_read_temp(ndev, IDT_TEMP_CUR, &mdeg);  in idt_dbgfs_info_read()
2503 static int idt_init_dbgfs(struct idt_ntb_dev *ndev)  in idt_init_dbgfs()  argument
2509 		dev_info(&ndev->ntb.pdev->dev, "Top DebugFS directory absent");  in idt_init_dbgfs()
2514 	snprintf(devname, 64, "info:%s", pci_name(ndev->ntb.pdev));  in idt_init_dbgfs()
2515 	ndev->dbgfs_info = debugfs_create_file(devname, 0400, dbgfs_topdir,  in idt_init_dbgfs()
2516 		ndev, &idt_dbgfs_info_ops);  in idt_init_dbgfs()
2517 	if (IS_ERR(ndev->dbgfs_info)) {  in idt_init_dbgfs()
2518 		dev_dbg(&ndev->ntb.pdev->dev, "Failed to create DebugFS node");  in idt_init_dbgfs()
2519 		return PTR_ERR(ndev->dbgfs_info);  in idt_init_dbgfs()
2522 	dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node created");  in idt_init_dbgfs()
2533 static void idt_deinit_dbgfs(struct idt_ntb_dev *ndev)  in idt_deinit_dbgfs()  argument
2535 	debugfs_remove(ndev->dbgfs_info);  in idt_deinit_dbgfs()
2537 	dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node discarded");  in idt_deinit_dbgfs()
2598 	struct idt_ntb_dev *ndev;  in idt_create_dev()  local
2601 	ndev = devm_kzalloc(&pdev->dev, sizeof(*ndev), GFP_KERNEL);  in idt_create_dev()
2602 	if (!ndev) {  in idt_create_dev()
2608 	ndev->swcfg = (struct idt_89hpes_cfg *)id->driver_data;  in idt_create_dev()
2610 	ndev->ntb.pdev = pdev;  in idt_create_dev()
2613 	spin_lock_init(&ndev->db_mask_lock);  in idt_create_dev()
2614 	spin_lock_init(&ndev->msg_mask_lock);  in idt_create_dev()
2615 	spin_lock_init(&ndev->gasa_lock);  in idt_create_dev()
2617 	dev_info(&pdev->dev, "IDT %s discovered", ndev->swcfg->name);  in idt_create_dev()
2621 	return ndev;  in idt_create_dev()
2633 static int idt_init_pci(struct idt_ntb_dev *ndev)  in idt_init_pci()  argument
2635 	struct pci_dev *pdev = ndev->ntb.pdev;  in idt_init_pci()
2678 	ndev->cfgspc = pcim_iomap(pdev, 0, 0);  in idt_init_pci()
2679 	if (!ndev->cfgspc) {  in idt_init_pci()
2686 	pci_set_drvdata(pdev, ndev);  in idt_init_pci()
2704 static void idt_deinit_pci(struct idt_ntb_dev *ndev)  in idt_deinit_pci()  argument
2706 	struct pci_dev *pdev = ndev->ntb.pdev;  in idt_deinit_pci()
2732 	struct idt_ntb_dev *ndev;  in idt_pci_probe()  local
2741 	ndev = idt_create_dev(pdev, id);  in idt_pci_probe()
2742 	if (IS_ERR(ndev))  in idt_pci_probe()
2743 		return PTR_ERR(ndev);  in idt_pci_probe()
2746 	ret = idt_init_pci(ndev);  in idt_pci_probe()
2751 	(void)idt_scan_ports(ndev);  in idt_pci_probe()
2754 	idt_init_link(ndev);  in idt_pci_probe()
2757 	ret = idt_init_mws(ndev);  in idt_pci_probe()
2762 	idt_init_msg(ndev);  in idt_pci_probe()
2765 	idt_init_temp(ndev);  in idt_pci_probe()
2768 	ret = idt_init_isr(ndev);  in idt_pci_probe()
2773 	ret = idt_register_device(ndev);  in idt_pci_probe()
2778 	(void)idt_init_dbgfs(ndev);  in idt_pci_probe()
2787 	idt_deinit_isr(ndev);  in idt_pci_probe()
2789 	idt_deinit_link(ndev);  in idt_pci_probe()
2790 	idt_deinit_pci(ndev);  in idt_pci_probe()
2801 	struct idt_ntb_dev *ndev = pci_get_drvdata(pdev);  in idt_pci_remove()  local
2804 	idt_deinit_dbgfs(ndev);  in idt_pci_remove()
2807 	idt_unregister_device(ndev);  in idt_pci_remove()
2810 	idt_deinit_isr(ndev);  in idt_pci_remove()
2813 	idt_deinit_link(ndev);  in idt_pci_remove()
2816 	idt_deinit_pci(ndev);  in idt_pci_remove()