1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (C) 2011-2016 Universita` di Pisa
45  *		we can access ring->cur, ring->head, ring->tail, etc.
47  *	ring->slot[i] gives us the i-th slot (we can access
51  *		the buffer numbered x
112 	nifp, (nifp)->ring_ofs[index] )
115 	nifp, (nifp)->ring_ofs[index + (nifp)->ni_tx_rings + 		\
116 		(nifp)->ni_host_tx_rings] )
119 	((char *)(ring) + (ring)->buf_ofs + ((size_t)(index)*(ring)->nr_buf_size))
122 	( ((char *)(buf) - ((char *)(ring) + (ring)->buf_ofs) ) / \
123 		(ring)->nr_buf_size )
127 	((slot)->ptr & (ring)->offset_mask)
131 	do { (slot)->ptr = ((slot)->ptr & ~(ring)->offset_mask) | \
132 		((offset) & (ring)->offset_mask); } while (0)
138 	(NETMAP_BUF(ring, (slot)->buf_idx) + NETMAP_ROFFSET(ring, slot))
144 	return ( unlikely(i + 1 == r->num_slots) ? 0 : i + 1);  in nm_ring_next()
150  * When everything is complete ring->head = ring->tail + 1 (modulo ring size)
155 	return nm_ring_next(r, r->tail) != r->head;  in nm_tx_pending()
159  * ring->head as explained in the comment above nm_ring_empty(). */
163         int ret = ring->tail - ring->head;  in nm_ring_space()
165                 ret += ring->num_slots;  in nm_ring_space()
200  * with other odd sizes. We assume there is enough room
213 	for (; likely(l > 0); l-=64) {  in nm_pkt_copy()
306  * when the descriptor is open correctly, d->self == d
310 #define IS_NETMAP_DESC(d)	((d) && P2NMD(d)->self == P2NMD(d))
311 #define NETMAP_FD(d)		(P2NMD(d)->fd)
319  *--- the pcap-like API ---
327  *		-NN		bind individual NIC ring pair
419 	for (curr = win_netmap_fd_list_head; curr; curr = curr->next) {  in win_insert_fd_record()
420 		if (fd == curr->win_netmap_fd) {  in win_insert_fd_record()
425 	curr->next = win_netmap_fd_list_head;  in win_insert_fd_record()
426 	curr->win_netmap_fd = fd;  in win_insert_fd_record()
427 	curr->win_netmap_handle = IntToPtr(_get_osfhandle(fd));  in win_insert_fd_record()
436 	for (; curr ; prev = curr, curr = curr->next) {  in win_remove_fd_record()
437 		if (fd != curr->win_netmap_fd)  in win_remove_fd_record()
441 			win_netmap_fd_list_head = curr->next;  in win_remove_fd_record()
443 			prev->next = curr->next;  in win_remove_fd_record()
456 	for (curr = win_netmap_fd_list_head; curr; curr = curr->next) {  in win_get_netmap_handle()
457 		if (fd == curr->win_netmap_fd) {  in win_get_netmap_handle()
458 			return curr->win_netmap_handle;  in win_get_netmap_handle()
470  * same as ioctl, returns 0 on success and -1 on error
500 		return -1;  in win_nm_ioctl_internal()
504 		return -1;  in win_nm_ioctl_internal()
513 	return ioctlReturnStatus ? 0 : -1;  in win_nm_ioctl_internal()
517  * this function is what must be called from user-space programs
518  * same as ioctl, returns 0 on success and -1 on error
563 	if (nfds != 1 || fds == NULL || (h = win_get_netmap_handle(fds->fd)) == NULL) {;  in win_nm_poll()
569 		prd.events = fds->events;  in win_nm_poll()
573 			return -1;  in win_nm_poll()
588 			return -1;  in win_nm_open()
603 	if (fd != -1) {  in win_nm_close()
665 	for (; *port && !index("-*^{}/@", *port); port++)  in nm_parse()
673 	namelen = port - ifname;  in nm_parse()
674 	if (namelen >= sizeof(d->req.nr_name)) {  in nm_parse()
678 	memcpy(d->req.nr_name, ifname, namelen);  in nm_parse()
679 	d->req.nr_name[namelen] = '\0';  in nm_parse()
695 			case '-': /* one NIC ring pair */  in nm_parse()
803 	d->req.nr_flags |= nr_flags;  in nm_parse()
804 	d->req.nr_ringid |= nr_ringid;  in nm_parse()
805 	d->req.nr_arg2 = nr_arg2;  in nm_parse()
807 	d->self = d;  in nm_parse()
815 	return -1;  in nm_parse()
821  * You can pass a pointer to a pre-filled nm_desc to add special
850 	d->self = d;	/* set this early so nm_close() works */  in nm_open()
851 	d->fd = open(NETMAP_DEVICE_NAME, O_RDWR);  in nm_open()
852 	if (d->fd < 0) {  in nm_open()
858 		d->req = *req;  in nm_open()
865 	d->req.nr_version = NETMAP_API;  in nm_open()
866 	d->req.nr_ringid &= NETMAP_RING_MASK;  in nm_open()
871 			D("overriding ARG1 %d", parent->req.nr_arg1);  in nm_open()
872 		d->req.nr_arg1 = new_flags & NM_OPEN_ARG1 ?  in nm_open()
873 			parent->req.nr_arg1 : 4;  in nm_open()
875 			D("overriding ARG2 %d", parent->req.nr_arg2);  in nm_open()
876 			d->req.nr_arg2 =  parent->req.nr_arg2;  in nm_open()
879 			D("overriding ARG3 %d", parent->req.nr_arg3);  in nm_open()
880 		d->req.nr_arg3 = new_flags & NM_OPEN_ARG3 ?  in nm_open()
881 			parent->req.nr_arg3 : 0;  in nm_open()
884 			d->req.nr_tx_slots = parent->req.nr_tx_slots;  in nm_open()
885 			d->req.nr_rx_slots = parent->req.nr_rx_slots;  in nm_open()
886 			d->req.nr_tx_rings = parent->req.nr_tx_rings;  in nm_open()
887 			d->req.nr_rx_rings = parent->req.nr_rx_rings;  in nm_open()
891 				parent->req.nr_name, parent->req.nr_ringid,  in nm_open()
892 				parent->req.nr_flags);  in nm_open()
893 			memcpy(d->req.nr_name, parent->req.nr_name,  in nm_open()
894 				sizeof(d->req.nr_name));  in nm_open()
895 			d->req.nr_ringid = parent->req.nr_ringid;  in nm_open()
896 			d->req.nr_flags = parent->req.nr_flags;  in nm_open()
900 	d->req.nr_ringid |= new_flags & (NETMAP_NO_TX_POLL | NETMAP_DO_RX_POLL);  in nm_open()
902 	if (ioctl(d->fd, NIOCREGIF, &d->req)) {  in nm_open()
907 	nr_reg = d->req.nr_flags & NR_REG_MASK;  in nm_open()
910 		d->first_tx_ring = d->last_tx_ring = d->req.nr_tx_rings;  in nm_open()
911 		d->first_rx_ring = d->last_rx_ring = d->req.nr_rx_rings;  in nm_open()
913 		d->first_tx_ring = 0;  in nm_open()
914 		d->first_rx_ring = 0;  in nm_open()
915 		d->last_tx_ring = d->req.nr_tx_rings - 1;  in nm_open()
916 		d->last_rx_ring = d->req.nr_rx_rings - 1;  in nm_open()
918 		d->first_tx_ring = 0;  in nm_open()
919 		d->first_rx_ring = 0;  in nm_open()
920 		d->last_tx_ring = d->req.nr_tx_rings;  in nm_open()
921 		d->last_rx_ring = d->req.nr_rx_rings;  in nm_open()
924 		d->first_tx_ring = d->last_tx_ring =  in nm_open()
925 		d->first_rx_ring = d->last_rx_ring = d->req.nr_ringid & NETMAP_RING_MASK;  in nm_open()
927 		d->first_tx_ring = d->last_tx_ring = 0;  in nm_open()
928 		d->first_rx_ring = d->last_rx_ring = 0;  in nm_open()
943 		d->first_tx_ring, d->last_tx_ring, d->req.nr_tx_rings,  in nm_open()
944                 d->first_rx_ring, d->last_rx_ring, d->req.nr_rx_rings);  in nm_open()
945 	for (i = 0; i <= d->req.nr_tx_rings; i++) {  in nm_open()
946 		struct netmap_ring *r = NETMAP_TXRING(d->nifp, i);  in nm_open()
947 		D("TX%d %p h %d c %d t %d", i, r, r->head, r->cur, r->tail);  in nm_open()
949 	for (i = 0; i <= d->req.nr_rx_rings; i++) {  in nm_open()
950 		struct netmap_ring *r = NETMAP_RXRING(d->nifp, i);  in nm_open()
951 		D("RX%d %p h %d c %d t %d", i, r, r->head, r->cur, r->tail);  in nm_open()
956 	d->cur_tx_ring = d->first_tx_ring;  in nm_open()
957 	d->cur_rx_ring = d->first_rx_ring;  in nm_open()
980 	if (d == NULL || d->self != d)  in nm_close()
982 	if (d->done_mmap && d->mem)  in nm_close()
983 		munmap(d->mem, d->memsize);  in nm_close()
984 	if (d->fd != -1) {  in nm_close()
985 		close(d->fd);  in nm_close()
997 	if (d->done_mmap)  in nm_mmap()
1000 	if (IS_NETMAP_DESC(parent) && parent->mem &&  in nm_mmap()
1001 	    parent->req.nr_arg2 == d->req.nr_arg2) {  in nm_mmap()
1004 		d->memsize = parent->memsize;  in nm_mmap()
1005 		d->mem = parent->mem;  in nm_mmap()
1008 		d->memsize = d->req.nr_memsize;  in nm_mmap()
1009 		d->mem = mmap(0, d->memsize, PROT_WRITE | PROT_READ, MAP_SHARED,  in nm_mmap()
1010 				d->fd, 0);  in nm_mmap()
1011 		if (d->mem == MAP_FAILED) {  in nm_mmap()
1014 		d->done_mmap = 1;  in nm_mmap()
1017 		struct netmap_if *nifp = NETMAP_IF(d->mem, d->req.nr_offset);  in nm_mmap()
1018 		struct netmap_ring *r = NETMAP_RXRING(nifp, d->first_rx_ring);  in nm_mmap()
1021 			r = NETMAP_TXRING(nifp, d->first_tx_ring);  in nm_mmap()
1024 		*(struct netmap_if **)(uintptr_t)&(d->nifp) = nifp;  in nm_mmap()
1025 		*(struct netmap_ring **)(uintptr_t)&d->some_ring = r;  in nm_mmap()
1026 		*(void **)(uintptr_t)&d->buf_start = NETMAP_BUF(r, 0);  in nm_mmap()
1027 		*(void **)(uintptr_t)&d->buf_end =  in nm_mmap()
1028 			(char *)d->mem + d->memsize;  in nm_mmap()
1043 	u_int c, n = d->last_tx_ring - d->first_tx_ring + 1,  in nm_inject()
1044 		ri = d->cur_tx_ring;  in nm_inject()
1052 		if (ri > d->last_tx_ring)  in nm_inject()
1053 			ri = d->first_tx_ring;  in nm_inject()
1054 		ring = NETMAP_TXRING(d->nifp, ri);  in nm_inject()
1056 		j = ring->cur;  in nm_inject()
1057 		while (rem > ring->nr_buf_size && j != ring->tail) {  in nm_inject()
1058 			rem -= ring->nr_buf_size;  in nm_inject()
1061 		if (j == ring->tail && rem > 0)  in nm_inject()
1063 		i = ring->cur;  in nm_inject()
1065 			idx = ring->slot[i].buf_idx;  in nm_inject()
1066 			ring->slot[i].len = ring->nr_buf_size;  in nm_inject()
1067 			ring->slot[i].flags = NS_MOREFRAG;  in nm_inject()
1068 			nm_pkt_copy(buf, NETMAP_BUF(ring, idx), ring->nr_buf_size);  in nm_inject()
1070 			buf = (const char *)buf + ring->nr_buf_size;  in nm_inject()
1072 		idx = ring->slot[i].buf_idx;  in nm_inject()
1073 		ring->slot[i].len = rem;  in nm_inject()
1074 		ring->slot[i].flags = 0;  in nm_inject()
1076 		ring->head = ring->cur = nm_ring_next(ring, i);  in nm_inject()
1077 		d->cur_tx_ring = ri;  in nm_inject()
1090 	int n = d->last_rx_ring - d->first_rx_ring + 1;  in nm_dispatch()
1091 	int c, got = 0, ri = d->cur_rx_ring;  in nm_dispatch()
1092 	d->hdr.buf = NULL;  in nm_dispatch()
1093 	d->hdr.flags = NM_MORE_PKTS;  in nm_dispatch()
1094 	d->hdr.d = d;  in nm_dispatch()
1097 		cnt = -1;  in nm_dispatch()
1098 	/* cnt == -1 means infinite, but rings have a finite amount  in nm_dispatch()
1100 	 * so we can omit checking for -1  in nm_dispatch()
1106 		if (ri > d->last_rx_ring)  in nm_dispatch()
1107 			ri = d->first_rx_ring;  in nm_dispatch()
1108 		ring = NETMAP_RXRING(d->nifp, ri);  in nm_dispatch()
1113 			if (d->hdr.buf) { /* from previous round */  in nm_dispatch()
1114 				cb(arg, &d->hdr, d->hdr.buf);  in nm_dispatch()
1116 			i = ring->cur;  in nm_dispatch()
1117 			slot = &ring->slot[i];  in nm_dispatch()
1118 			idx = slot->buf_idx;  in nm_dispatch()
1119 			/* d->cur_rx_ring doesn't change inside this loop, but  in nm_dispatch()
1120 			 * set it here, so it reflects d->hdr.buf's ring */  in nm_dispatch()
1121 			d->cur_rx_ring = ri;  in nm_dispatch()
1122 			d->hdr.slot = slot;  in nm_dispatch()
1123 			oldbuf = d->hdr.buf = (u_char *)NETMAP_BUF(ring, idx);  in nm_dispatch()
1125 			d->hdr.len = d->hdr.caplen = slot->len;  in nm_dispatch()
1126 			while (slot->flags & NS_MOREFRAG) {  in nm_dispatch()
1128 				u_int oldlen = slot->len;  in nm_dispatch()
1130 				slot = &ring->slot[i];  in nm_dispatch()
1131 				d->hdr.len += slot->len;  in nm_dispatch()
1132 				nbuf = (u_char *)NETMAP_BUF(ring, slot->buf_idx);  in nm_dispatch()
1133 				if (oldbuf != NULL && (uint32_t)(nbuf - oldbuf) == ring->nr_buf_size &&  in nm_dispatch()
1134 						oldlen == ring->nr_buf_size) {  in nm_dispatch()
1135 					d->hdr.caplen += slot->len;  in nm_dispatch()
1141 			d->hdr.ts = ring->ts;  in nm_dispatch()
1142 			ring->head = ring->cur = nm_ring_next(ring, i);  in nm_dispatch()
1145 	if (d->hdr.buf) { /* from previous round */  in nm_dispatch()
1146 		d->hdr.flags = 0;  in nm_dispatch()
1147 		cb(arg, &d->hdr, d->hdr.buf);  in nm_dispatch()
1155 	int ri = d->cur_rx_ring;  in nm_nextpkt()
1159 		struct netmap_ring *ring = NETMAP_RXRING(d->nifp, ri);  in nm_nextpkt()
1161 			u_int i = ring->cur;  in nm_nextpkt()
1162 			u_int idx = ring->slot[i].buf_idx;  in nm_nextpkt()
1166 			hdr->ts = ring->ts;  in nm_nextpkt()
1167 			hdr->len = hdr->caplen = ring->slot[i].len;  in nm_nextpkt()
1168 			ring->cur = nm_ring_next(ring, i);  in nm_nextpkt()
1173 			ring->head = ring->cur;  in nm_nextpkt()
1174 			d->cur_rx_ring = ri;  in nm_nextpkt()
1178 		if (ri > d->last_rx_ring)  in nm_nextpkt()
1179 			ri = d->first_rx_ring;  in nm_nextpkt()
1180 	} while (ri != d->cur_rx_ring);  in nm_nextpkt()