1 /*-
7  * Copyright (c) 2019 Advanced Micro Devices, Inc. All Rights Reserved.
20  * Copyright (c) 2019 Advanced Micro Devices, Inc. All Rights Reserved.
32  *   * Neither the name of Advanced Micro Devices, Inc nor the names of its
55  * root@local# sysctl -a | grep ntb_tool
102  *-----------------------------------------------------------------------------
110  *-----------------------------------------------------------------------------
125  *-----------------------------------------------------------------------------
149  *-----------------------------------------------------------------------------
159  *-----------------------------------------------------------------------------
168  * root@local# sysctl dev.ntb_tool.0.peer0.mw0="W offset 0 nbytes 100 pattern ab"
173  * root@local# sysctl dev.ntb_tool.0.peer0.peer_mw0="W offset 0 nbytes 100 pattern ab"
175  *-----------------------------------------------------------------------------
177  *-----------------------------------------------------------------------------
298 	up = ntb_link_is_up(tc->dev, &speed, &width);  in tool_link_event()
300 		tc->link_status = 'Y';  in tool_link_event()
302 		tc->link_status = 'N';  in tool_link_event()
304 	device_printf(tc->dev, "link is %s speed %d width %d\n",  in tool_link_event()
314 	db_mask = ntb_db_vector_mask(tc->dev, vec);  in tool_db_event()
315 	db_bits = ntb_db_read(tc->dev);  in tool_db_event()
317 	device_printf(tc->dev, "doorbell vec %d mask %#llx bits %#llx\n",  in tool_db_event()
335 	val = ntb_link_is_up(tc->dev, NULL, NULL) & tc->link_mask;  in tool_link_event_handler()
337 	if (val == tc->link_bits) {  in tool_link_event_handler()
338 		device_printf(tc->dev, "link_event successful for link val="  in tool_link_event_handler()
339 		    "0x%jx\n", tc->link_bits);  in tool_link_event_handler()
340 		tc->link_bits = 0x0;  in tool_link_event_handler()
341 		tc->link_mask = 0x0;  in tool_link_event_handler()
343 		callout_reset(&tc->link_event_timer, 1, tool_link_event_handler, tc);  in tool_link_event_handler()
352 	db_bits = ntb_db_read(tc->dev);  in tool_db_event_handler()
354 	if (db_bits == tc->db_event_val) {  in tool_db_event_handler()
355 		device_printf(tc->dev, "db_event successful for db val=0x%jx\n",  in tool_db_event_handler()
356 		    tc->db_event_val);  in tool_db_event_handler()
357 		tc->db_event_val = 0x0;  in tool_db_event_handler()
359 		callout_reset(&tc->db_event_timer, 1, tool_db_event_handler, tc);  in tool_db_event_handler()
370 	if (req->newlen >= TOOL_BUF_LEN)  in get_ubuf()
373 	rc = SYSCTL_IN(req, ubuf, req->newlen);  in get_ubuf()
376 	ubuf[req->newlen] = '\0';  in get_ubuf()
399 		return read_out(req, (uint64_t)fn_read(tc->dev));  in tool_fn_read()
409 	uint64_t db_valid_mask = tc->db_valid_mask;  in tool_fn_write()
414 		device_printf(tc->dev, "ERR: Set & Clear both are not supported\n");  in tool_fn_write()
418 	if (tc->db_valid_mask == 0)  in tool_fn_write()
419 		db_valid_mask = tc->db_valid_mask = ntb_db_valid_mask(tc->dev);  in tool_fn_write()
425 			device_printf(tc->dev, "0x%jx value is not supported\n", bits);  in tool_fn_write()
429 			fn_set(tc->dev, bits);  in tool_fn_write()
436 			device_printf(tc->dev, "0x%jx value is not supported\n", bits);  in tool_fn_write()
440 			fn_clear(tc->dev, bits);  in tool_fn_write()
444 		device_printf(tc->dev, "Wrong Write\n");  in tool_fn_write()
532 		rc = sb->s_error;  in tool_mw_read_fn()
537 		sbuf_printf(sb, "\nConfigured MW size\t: %zu\n", inmw->size);  in tool_mw_read_fn()
540 		    inmw->size);  in tool_mw_read_fn()
541 	sbuf_printf(sb, "R/W size\t\t: %zi\nR/W Offset\t\t: %zi\n\nData\n----"  in tool_mw_read_fn()
542 	    "->", buf_size, buf_off);  in tool_mw_read_fn()
562 		sbuf_printf(sb, "\n\nNOTE: Truncating read size %zi->1024 "  in tool_mw_read_fn()
566 	*cmd_op -= 1;  in tool_mw_read_fn()
593 		device_printf(inmw->tc->dev, "Wrong Command \"%c\" provided\n",  in tool_mw_write_fn()
599 	if ((*buf_offset + *buf_size) > inmw->size) {  in tool_mw_write_fn()
600 		device_printf(inmw->tc->dev, "%s: configured mw size :%zi and "  in tool_mw_write_fn()
601 		    "requested size :%zi.\n", __func__, inmw->size,  in tool_mw_write_fn()
645 	peer_port = ntb_peer_port_number(tc->dev, pidx);  in sysctl_peer_port_number()
648 		device_printf(tc->dev, "Peer port sysctl set failed with err="  in sysctl_peer_port_number()
651 		tc->peers[pidx].port_no = peer_port;  in sysctl_peer_port_number()
662 	local_port = ntb_port_number(tc->dev);  in sysctl_local_port_number()
665 		device_printf(tc->dev, "Local port sysctl set failed with err="  in sysctl_local_port_number()
668 		tc->port_no = local_port;  in sysctl_local_port_number()
678 	tc->peer_cnt = ntb_peer_port_count(tc->dev);  in tool_init_peers()
679 	tc->peers = malloc(tc->peer_cnt * sizeof(*tc->peers), M_NTB_TOOL,  in tool_init_peers()
681 	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {  in tool_init_peers()
682 		tc->peers[pidx].pidx = pidx;  in tool_init_peers()
683 		tc->peers[pidx].tc = tc;  in tool_init_peers()
691 	free(tc->peers, M_NTB_TOOL);  in tool_clear_peers()
704 	if (req->newptr == NULL) {  in sysctl_link_handle()
705 		snprintf(buf, 2, "%c", tc->link_status);  in sysctl_link_handle()
715 		rc = ntb_link_enable(tc->dev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);  in sysctl_link_handle()
717 		rc = ntb_link_disable(tc->dev);  in sysctl_link_handle()
721 	sscanf(buf, "%c", &tc->link_status);  in sysctl_link_handle()
733 	if (req->newptr)  in sysctl_peer_link_handle()
736 	up = ntb_link_is_up(tc->dev, NULL, NULL);  in sysctl_peer_link_handle()
754 	if (req->newptr == NULL)  in sysctl_peer_link_event_handle()
755 		return read_out(req, tc->link_bits);  in sysctl_peer_link_event_handle()
762 	tc->link_bits = bits;  in sysctl_peer_link_event_handle()
763 	tc->link_mask = (1ULL << ((pidx) % 64));  in sysctl_peer_link_event_handle()
765 	callout_reset(&tc->link_event_timer, 1, tool_link_event_handler, tc);  in sysctl_peer_link_event_handle()
777 	if (!(cba->error = error))  in ntb_tool_load_cb()
778 		cba->addr = segs[0].ds_addr;  in ntb_tool_load_cb()
788 	if (req->newptr == NULL)  in sysctl_mw_handle()
789 		return tool_mw_read_fn(req, inmw, (char *)inmw->mm_base,  in sysctl_mw_handle()
790 		    &inmw->mw_cmd_rw, inmw->mw_buf_offset, inmw->mw_buf_size,  in sysctl_mw_handle()
795 		return tool_mw_write_fn(oidp, req, inmw, buf, inmw->mm_base,  in sysctl_mw_handle()
796 		    &inmw->mw_cmd_rw, &inmw->mw_buf_offset, &inmw->mw_buf_size);  in sysctl_mw_handle()
805 	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];  in tool_setup_mw()
810 		inmw->size = roundup(inmw->phys_size, inmw->xlat_align_size);  in tool_setup_mw()
812 		inmw->size = roundup(req_size, inmw->xlat_align_size);  in tool_setup_mw()
814 	device_printf(tc->dev, "mw_size %zi req_size %zi buff %zi\n",  in tool_setup_mw()
815 	    inmw->phys_size, req_size, inmw->size);  in tool_setup_mw()
817 	if (bus_dma_tag_create(bus_get_dma_tag(tc->dev), inmw->xlat_align, 0,  in tool_setup_mw()
818 	    inmw->addr_limit, BUS_SPACE_MAXADDR, NULL, NULL, inmw->size, 1,  in tool_setup_mw()
819 	    inmw->size, 0, NULL, NULL, &inmw->dma_tag)) {  in tool_setup_mw()
820 		device_printf(tc->dev, "Unable to create MW tag of size "  in tool_setup_mw()
821 		    "%zu/%zu\n", inmw->phys_size, inmw->size);  in tool_setup_mw()
826 	if (bus_dmamem_alloc(inmw->dma_tag, (void **)&inmw->virt_addr,  in tool_setup_mw()
827 	    BUS_DMA_WAITOK | BUS_DMA_ZERO, &inmw->dma_map)) {  in tool_setup_mw()
828 		device_printf(tc->dev, "Unable to allocate MW buffer of size "  in tool_setup_mw()
829 		    "%zu/%zu\n", inmw->phys_size, inmw->size);  in tool_setup_mw()
834 	if (bus_dmamap_load(inmw->dma_tag, inmw->dma_map, inmw->virt_addr,  in tool_setup_mw()
835 	    inmw->size, ntb_tool_load_cb, &cba, BUS_DMA_NOWAIT) || cba.error) {  in tool_setup_mw()
836 		device_printf(tc->dev, "Unable to load MW buffer of size "  in tool_setup_mw()
837 		    "%zu/%zu\n", inmw->phys_size, inmw->size);  in tool_setup_mw()
841 	inmw->dma_base = cba.addr;  in tool_setup_mw()
843 	rc = ntb_mw_set_trans(tc->dev, widx, inmw->dma_base, inmw->size);  in tool_setup_mw()
850 	bus_dmamap_unload(inmw->dma_tag, inmw->dma_map);  in tool_setup_mw()
853 	bus_dmamem_free(inmw->dma_tag, inmw->virt_addr, inmw->dma_map);  in tool_setup_mw()
856 	bus_dma_tag_destroy(inmw->dma_tag);  in tool_setup_mw()
859 	inmw->size = 0;  in tool_setup_mw()
860 	inmw->virt_addr = 0;  in tool_setup_mw()
861 	inmw->dma_base = 0;  in tool_setup_mw()
862 	inmw->dma_tag = 0;  in tool_setup_mw()
863 	inmw->dma_map = 0;  in tool_setup_mw()
871 	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];  in tool_free_mw()
873 	if (inmw->dma_base)  in tool_free_mw()
874 		ntb_mw_clear_trans(tc->dev, widx);  in tool_free_mw()
876 	if (inmw->virt_addr && inmw->dma_tag) {  in tool_free_mw()
877 		bus_dmamap_unload(inmw->dma_tag, inmw->dma_map);  in tool_free_mw()
878 		bus_dmamem_free(inmw->dma_tag, inmw->virt_addr, inmw->dma_map);  in tool_free_mw()
879 		bus_dma_tag_destroy(inmw->dma_tag);  in tool_free_mw()
882 	inmw->virt_addr = 0;  in tool_free_mw()
883 	inmw->dma_base = 0;  in tool_free_mw()
884 	inmw->dma_tag = 0;  in tool_free_mw()
885 	inmw->dma_map = 0;  in tool_free_mw()
886 	inmw->mm_base = 0;  in tool_free_mw()
887 	inmw->size = 0;  in tool_free_mw()
899 		rc = sb->s_error;  in tool_mw_trans_read()
903 	sbuf_printf(sb, "\nInbound MW     \t%d\n", inmw->widx);  in tool_mw_trans_read()
905 	    ntb_peer_port_number(inmw->tc->dev, inmw->pidx), inmw->pidx);  in tool_mw_trans_read()
906 	sbuf_printf(sb, "Window Address \t%p\n", inmw->mm_base);  in tool_mw_trans_read()
907 	sbuf_printf(sb, "DMA Address    \t0x%016llx\n", (long long)inmw->dma_base);  in tool_mw_trans_read()
908 	sbuf_printf(sb, "Window Size    \t0x%016zx[p]\n", inmw->size);  in tool_mw_trans_read()
909 	sbuf_printf(sb, "Alignment      \t0x%016zx[p]\n", inmw->xlat_align);  in tool_mw_trans_read()
911 	    inmw->xlat_align_size);  in tool_mw_trans_read()
912 	sbuf_printf(sb, "Size Max       \t0x%016zx[p]\n", inmw->phys_size);  in tool_mw_trans_read()
924 	struct tool_ctx *tc = inmw->tc;  in tool_mw_trans_write()
930 	/* No need to re-setup mw */  in tool_mw_trans_write()
931 	if (inmw->size == wsize)  in tool_mw_trans_write()
935 	if (inmw->size)  in tool_mw_trans_write()
936 		tool_free_mw(tc, inmw->pidx, inmw->widx);  in tool_mw_trans_write()
938 	rc = tool_setup_mw(tc, inmw->pidx, inmw->widx, wsize);  in tool_mw_trans_write()
951 	if (req->newptr == NULL)  in sysctl_mw_trans_handler()
970 	if (req->newptr == NULL)  in sysctl_peer_mw_handle()
971 		return tool_mw_read_fn(req, inmw, (char *)inmw->virt_addr,  in sysctl_peer_mw_handle()
972 		    &inmw->mw_peer_cmd_rw, inmw->mw_peer_buf_offset,  in sysctl_peer_mw_handle()
973 		    inmw->mw_peer_buf_size, "mw");  in sysctl_peer_mw_handle()
977 		return tool_mw_write_fn(oidp, req, inmw, buf, inmw->virt_addr,  in sysctl_peer_mw_handle()
978 		    &inmw->mw_peer_cmd_rw, &inmw->mw_peer_buf_offset,  in sysctl_peer_mw_handle()
979 		    &inmw->mw_peer_buf_size);  in sysctl_peer_mw_handle()
989 	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {  in tool_clear_mws()
990 		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)  in tool_clear_mws()
992 		free(tc->peers[pidx].inmws, M_NTB_TOOL);  in tool_clear_mws()
1003 	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {  in tool_init_mws()
1004 		tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->dev);  in tool_init_mws()
1005 		tc->peers[pidx].inmws = malloc(tc->peers[pidx].inmw_cnt *  in tool_init_mws()
1006 		    sizeof(*tc->peers[pidx].inmws), M_NTB_TOOL,  in tool_init_mws()
1009 		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {  in tool_init_mws()
1010 			mw = &tc->peers[pidx].inmws[widx];  in tool_init_mws()
1012 			mw->tc = tc;  in tool_init_mws()
1013 			mw->widx = widx;  in tool_init_mws()
1014 			mw->pidx = pidx;  in tool_init_mws()
1015 			mw->mw_buf_offset = DEFAULT_MW_OFF;  in tool_init_mws()
1016 			mw->mw_buf_size = DEFAULT_MW_SIZE;  in tool_init_mws()
1018 			rc = ntb_mw_get_range(tc->dev, widx, &mw->phys_addr,  in tool_init_mws()
1019 			    &mw->mm_base, &mw->phys_size, &mw->xlat_align,  in tool_init_mws()
1020 			    &mw->xlat_align_size, &mw->addr_limit);  in tool_init_mws()
1044 	if (req->newptr == NULL) {  in sysctl_db_handle()
1045 		db_bits = ntb_db_read(tc->dev);  in sysctl_db_handle()
1062 	tc->db_valid_mask = ntb_db_valid_mask(tc->dev);  in sysctl_db_valid_mask_handle()
1063 	if (!tc->db_valid_mask) {  in sysctl_db_valid_mask_handle()
1064 		device_printf(tc->dev, "Error getting db_valid_mask from "  in sysctl_db_valid_mask_handle()
1068 		return read_out(req, tc->db_valid_mask);  in sysctl_db_valid_mask_handle()
1079 	if (req->newptr == NULL) {  in sysctl_db_mask_handle()
1080 		if (tc->db_mask_val == 0)  in sysctl_db_mask_handle()
1081 		     ntb_db_valid_mask(tc->dev);  in sysctl_db_mask_handle()
1082 		return tool_fn_read(tc, req, NULL, tc->db_mask_val);  in sysctl_db_mask_handle()
1087 		return tool_fn_write(tc, oidp, req, buf, &tc->db_mask_val, true,  in sysctl_db_mask_handle()
1100 	if (req->newptr == NULL)  in sysctl_peer_db_handle()
1101 		return tool_fn_read(tc, req, NULL, tc->peer_db_val);  in sysctl_peer_db_handle()
1105 		return tool_fn_write(tc, oidp, req, buf, &tc->peer_db_val,  in sysctl_peer_db_handle()
1118 	if (req->newptr == NULL){  in sysctl_peer_db_mask_handle()
1119 		if (tc->peer_db_mask_val == 0)  in sysctl_peer_db_mask_handle()
1120 			ntb_db_valid_mask(tc->dev);  in sysctl_peer_db_mask_handle()
1121 		return tool_fn_read(tc, req, NULL, tc->peer_db_mask_val);  in sysctl_peer_db_mask_handle()
1126 		return tool_fn_write(tc, oidp, req, buf, &tc->peer_db_mask_val,  in sysctl_peer_db_mask_handle()
1140 	if (req->newptr == NULL)  in sysctl_db_event_handle()
1141 		return read_out(req, tc->db_event_val);  in sysctl_db_event_handle()
1148 	tc->db_event_val = bits;  in sysctl_db_event_handle()
1149 	callout_reset(&tc->db_event_timer, 1, tool_db_event_handler, tc);  in sysctl_db_event_handle()
1166 	if (req->newptr == NULL) {  in sysctl_spad_handle()
1167 		rc = ntb_spad_read(tc->dev, sidx, &bits);  in sysctl_spad_handle()
1177 		return ntb_spad_write(tc->dev, sidx, bits);  in sysctl_spad_handle()
1192 	if (req->newptr == NULL) {  in sysctl_peer_spad_handle()
1193 		rc = ntb_peer_spad_read(tc->dev, sidx, &bits);  in sysctl_peer_spad_handle()
1203 		return ntb_peer_spad_write(tc->dev, sidx, bits);  in sysctl_peer_spad_handle()
1215 	tc->inspad_cnt = ntb_spad_count(tc->dev);  in tool_init_spads()
1216 	tc->inspads = malloc(tc->inspad_cnt * sizeof(*tc->inspads), M_NTB_TOOL,  in tool_init_spads()
1219 	for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {  in tool_init_spads()
1220 		tc->inspads[sidx].sidx = sidx;  in tool_init_spads()
1221 		tc->inspads[sidx].pidx = -1;  in tool_init_spads()
1222 		tc->inspads[sidx].tc = tc;  in tool_init_spads()
1226 	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {  in tool_init_spads()
1227 		tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->dev);  in tool_init_spads()
1228 		tc->peers[pidx].outspads =  malloc(tc->peers[pidx].outspad_cnt *  in tool_init_spads()
1229 		    sizeof(*tc->peers[pidx].outspads), M_NTB_TOOL, M_WAITOK |  in tool_init_spads()
1232 		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {  in tool_init_spads()
1233 			tc->peers[pidx].outspads[sidx].sidx = sidx;  in tool_init_spads()
1234 			tc->peers[pidx].outspads[sidx].pidx = pidx;  in tool_init_spads()
1235 			tc->peers[pidx].outspads[sidx].tc = tc;  in tool_init_spads()
1246 	free(tc->inspads, M_NTB_TOOL);  in tool_clear_spads()
1249 	for (pidx = 0; pidx < tc->peer_cnt; pidx++)  in tool_clear_spads()
1250 		free(tc->peers[pidx].outspads, M_NTB_TOOL);  in tool_clear_spads()
1261 	callout_init(&tc->link_event_timer, 1);  in tool_check_ntb()
1264 	callout_init(&tc->db_event_timer, 1);  in tool_check_ntb()
1267 	tc->link_status = 'U';  in tool_check_ntb()
1268 	tc->db_mask_val = 0;  in tool_check_ntb()
1269 	tc->peer_db_val = 0;  in tool_check_ntb()
1270 	tc->peer_db_mask_val = 0;  in tool_check_ntb()
1271 	tc->db_event_val = 0;  in tool_check_ntb()
1272 	tc->link_bits = 0;  in tool_check_ntb()
1281 	callout_drain(&tc->link_event_timer);  in tool_clear_data()
1282 	callout_drain(&tc->db_event_timer);  in tool_clear_data()
1289 	return ntb_set_ctx(tc->dev, tc, &tool_ops);  in tool_init_ntb()
1296 	ntb_clear_ctx(tc->dev);  in tool_clear_ntb()
1297 	ntb_link_disable(tc->dev);  in tool_clear_ntb()
1313 	clist = device_get_sysctl_ctx(tc->dev);  in tool_setup_sysctl()
1314 	parent = device_get_sysctl_tree(tc->dev);  in tool_setup_sysctl()
1349 	if (tc->inspad_cnt != 0) {  in tool_setup_sysctl()
1350 		for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {  in tool_setup_sysctl()
1360 	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {  in tool_setup_sysctl()
1379 		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {  in tool_setup_sysctl()
1385 			    &tc->peers[pidx].inmws[widx], 0,  in tool_setup_sysctl()
1393 			    &tc->peers[pidx].inmws[widx], 0,  in tool_setup_sysctl()
1401 			    &tc->peers[pidx].inmws[widx], 0,  in tool_setup_sysctl()
1405 		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {  in tool_setup_sysctl()
1429 	tc->dev = dev;  in ntb_tool_attach()