xref: /titanic_44/usr/src/uts/common/io/rge/rge_rxtx.c (revision 56b2bdd1f04d465cfe4a95b88ae5cba5884154e4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include "rge.h"
27 
28 #define	U32TOPTR(x)	((void *)(uintptr_t)(uint32_t)(x))
29 #define	PTRTOU32(x)	((uint32_t)(uintptr_t)(void *)(x))
30 
31 /*
32  * ========== RX side routines ==========
33  */
34 
35 #define	RGE_DBG		RGE_DBG_RECV	/* debug flag for this code	*/
36 
37 static uint32_t rge_atomic_reserve(uint32_t *count_p, uint32_t n);
38 #pragma	inline(rge_atomic_reserve)
39 
40 static uint32_t
41 rge_atomic_reserve(uint32_t *count_p, uint32_t n)
42 {
43 	uint32_t oldval;
44 	uint32_t newval;
45 
46 	/* ATOMICALLY */
47 	do {
48 		oldval = *count_p;
49 		newval = oldval - n;
50 		if (oldval <= n)
51 			return (0);		/* no resources left	*/
52 	} while (cas32(count_p, oldval, newval) != oldval);
53 
54 	return (newval);
55 }
56 
57 /*
58  * Atomically increment a counter
59  */
60 static void rge_atomic_renounce(uint32_t *count_p, uint32_t n);
61 #pragma	inline(rge_atomic_renounce)
62 
63 static void
64 rge_atomic_renounce(uint32_t *count_p, uint32_t n)
65 {
66 	uint32_t oldval;
67 	uint32_t newval;
68 
69 	/* ATOMICALLY */
70 	do {
71 		oldval = *count_p;
72 		newval = oldval + n;
73 	} while (cas32(count_p, oldval, newval) != oldval);
74 }
75 
76 /*
77  * Callback code invoked from STREAMs when the recv data buffer is free
78  * for recycling.
79  */
80 void
81 rge_rx_recycle(caddr_t arg)
82 {
83 	rge_t *rgep;
84 	dma_buf_t *rx_buf;
85 	sw_rbd_t *free_srbdp;
86 	uint32_t slot_recy;
87 
88 	rx_buf = (dma_buf_t *)arg;
89 	rgep = (rge_t *)rx_buf->private;
90 
91 	/*
92 	 * In rge_unattach() and rge_attach(), this callback function will
93 	 * also be called to free mp in rge_fini_rings() and rge_init_rings().
94 	 * In such situation, we shouldn't do below desballoc(), otherwise,
95 	 * there'll be memory leak.
96 	 */
97 	if (rgep->rge_mac_state == RGE_MAC_UNATTACH ||
98 	    rgep->rge_mac_state == RGE_MAC_ATTACH)
99 		return;
100 
101 	/*
102 	 * Recycle the data buffer again
103 	 * and fill them in free ring
104 	 */
105 	rx_buf->mp = desballoc(DMA_VPTR(rx_buf->pbuf),
106 	    rgep->rxbuf_size, 0, &rx_buf->rx_recycle);
107 	if (rx_buf->mp == NULL) {
108 		rge_problem(rgep, "rge_rx_recycle: desballoc() failed");
109 		return;
110 	}
111 	mutex_enter(rgep->rc_lock);
112 	slot_recy = rgep->rc_next;
113 	free_srbdp = &rgep->free_srbds[slot_recy];
114 
115 	ASSERT(free_srbdp->rx_buf == NULL);
116 	free_srbdp->rx_buf = rx_buf;
117 	rgep->rc_next = NEXT(slot_recy, RGE_BUF_SLOTS);
118 	rge_atomic_renounce(&rgep->rx_free, 1);
119 	if (rgep->rx_bcopy && rgep->rx_free == RGE_BUF_SLOTS)
120 		rgep->rx_bcopy = B_FALSE;
121 	ASSERT(rgep->rx_free <= RGE_BUF_SLOTS);
122 
123 	mutex_exit(rgep->rc_lock);
124 }
125 
126 static int rge_rx_refill(rge_t *rgep, uint32_t slot);
127 #pragma	inline(rge_rx_refill)
128 
129 static int
130 rge_rx_refill(rge_t *rgep, uint32_t slot)
131 {
132 	dma_buf_t *free_buf;
133 	rge_bd_t *hw_rbd_p;
134 	sw_rbd_t *srbdp;
135 	uint32_t free_slot;
136 
137 	srbdp = &rgep->sw_rbds[slot];
138 	hw_rbd_p = &rgep->rx_ring[slot];
139 	free_slot = rgep->rf_next;
140 	free_buf = rgep->free_srbds[free_slot].rx_buf;
141 	if (free_buf != NULL) {
142 		srbdp->rx_buf = free_buf;
143 		rgep->free_srbds[free_slot].rx_buf = NULL;
144 		hw_rbd_p->host_buf_addr = RGE_BSWAP_32(rgep->head_room +
145 		    + free_buf->pbuf.cookie.dmac_laddress);
146 		hw_rbd_p->host_buf_addr_hi =
147 		    RGE_BSWAP_32(free_buf->pbuf.cookie.dmac_laddress >> 32);
148 		rgep->rf_next = NEXT(free_slot, RGE_BUF_SLOTS);
149 		return (1);
150 	} else {
151 		/*
152 		 * This situation shouldn't happen
153 		 */
154 		rge_problem(rgep, "rge_rx_refill: free buffer %d is NULL",
155 		    free_slot);
156 		rgep->rx_bcopy = B_TRUE;
157 		return (0);
158 	}
159 }
160 
161 static mblk_t *rge_receive_packet(rge_t *rgep, uint32_t slot);
162 #pragma	inline(rge_receive_packet)
163 
164 static mblk_t *
165 rge_receive_packet(rge_t *rgep, uint32_t slot)
166 {
167 	rge_bd_t *hw_rbd_p;
168 	sw_rbd_t *srbdp;
169 	uchar_t *dp;
170 	mblk_t *mp;
171 	uint8_t *rx_ptr;
172 	uint32_t rx_status;
173 	uint_t packet_len;
174 	uint_t minsize;
175 	uint_t maxsize;
176 	uint32_t proto;
177 	uint32_t pflags;
178 	struct ether_vlan_header *ehp;
179 	uint16_t vtag = 0;
180 
181 	hw_rbd_p = &rgep->rx_ring[slot];
182 	srbdp = &rgep->sw_rbds[slot];
183 
184 	/*
185 	 * Read receive status
186 	 */
187 	rx_status = RGE_BSWAP_32(hw_rbd_p->flags_len) & RBD_FLAGS_MASK;
188 
189 	/*
190 	 * Handle error packet
191 	 */
192 	if (!(rx_status & BD_FLAG_PKT_END)) {
193 		RGE_DEBUG(("rge_receive_packet: not a complete packat"));
194 		return (NULL);
195 	}
196 	if (rx_status & RBD_FLAG_ERROR) {
197 		if (rx_status & RBD_FLAG_CRC_ERR)
198 			rgep->stats.crc_err++;
199 		if (rx_status & RBD_FLAG_RUNT)
200 			rgep->stats.in_short++;
201 		/*
202 		 * Set chip_error flag to reset chip:
203 		 * (suggested in Realtek programming guide.)
204 		 */
205 		RGE_DEBUG(("rge_receive_packet: error packet, status = %x",
206 		    rx_status));
207 		mutex_enter(rgep->genlock);
208 		rgep->rge_chip_state = RGE_CHIP_ERROR;
209 		mutex_exit(rgep->genlock);
210 		return (NULL);
211 	}
212 
213 	/*
214 	 * Handle size error packet
215 	 */
216 	packet_len = RGE_BSWAP_32(hw_rbd_p->flags_len) & RBD_LEN_MASK;
217 	packet_len -= ETHERFCSL;
218 	minsize = ETHERMIN;
219 	pflags = RGE_BSWAP_32(hw_rbd_p->vlan_tag);
220 	if (pflags & RBD_VLAN_PKT)
221 		minsize -= VLAN_TAGSZ;
222 	maxsize = rgep->ethmax_size;
223 	if (packet_len < minsize || packet_len > maxsize) {
224 		RGE_DEBUG(("rge_receive_packet: len err = %d", packet_len));
225 		return (NULL);
226 	}
227 
228 	DMA_SYNC(srbdp->rx_buf->pbuf, DDI_DMA_SYNC_FORKERNEL);
229 	if (rgep->rx_bcopy || packet_len <= RGE_RECV_COPY_SIZE ||
230 	    !rge_atomic_reserve(&rgep->rx_free, 1)) {
231 		/*
232 		 * Allocate buffer to receive this good packet
233 		 */
234 		mp = allocb(packet_len + RGE_HEADROOM, 0);
235 		if (mp == NULL) {
236 			RGE_DEBUG(("rge_receive_packet: allocate buffer fail"));
237 			rgep->stats.no_rcvbuf++;
238 			return (NULL);
239 		}
240 
241 		/*
242 		 * Copy the data found into the new cluster
243 		 */
244 		rx_ptr = DMA_VPTR(srbdp->rx_buf->pbuf);
245 		mp->b_rptr = dp = mp->b_rptr + RGE_HEADROOM;
246 		bcopy(rx_ptr + rgep->head_room, dp, packet_len);
247 		mp->b_wptr = dp + packet_len;
248 	} else {
249 		mp = srbdp->rx_buf->mp;
250 		mp->b_rptr += rgep->head_room;
251 		mp->b_wptr = mp->b_rptr + packet_len;
252 		mp->b_next = mp->b_cont = NULL;
253 		/*
254 		 * Refill the current receive bd buffer
255 		 *   if fails, will just keep the mp.
256 		 */
257 		if (!rge_rx_refill(rgep, slot))
258 			return (NULL);
259 	}
260 	rgep->stats.rbytes += packet_len;
261 	rgep->stats.rpackets ++;
262 
263 	/*
264 	 * VLAN packet ?
265 	 */
266 	if (pflags & RBD_VLAN_PKT)
267 		vtag = pflags & RBD_VLAN_TAG;
268 	if (vtag) {
269 		vtag = TCI_CHIP2OS(vtag);
270 		/*
271 		 * As h/w strips the VLAN tag from incoming packet, we need
272 		 * insert VLAN tag into this packet before send up here.
273 		 */
274 		(void) memmove(mp->b_rptr - VLAN_TAGSZ, mp->b_rptr,
275 		    2 * ETHERADDRL);
276 		mp->b_rptr -= VLAN_TAGSZ;
277 		ehp = (struct ether_vlan_header *)mp->b_rptr;
278 		ehp->ether_tpid = htons(ETHERTYPE_VLAN);
279 		ehp->ether_tci = htons(vtag);
280 		rgep->stats.rbytes += VLAN_TAGSZ;
281 	}
282 
283 	/*
284 	 * Check h/w checksum offload status
285 	 */
286 	pflags = 0;
287 	proto = rx_status & RBD_FLAG_PROTOCOL;
288 	if ((proto == RBD_FLAG_TCP && !(rx_status & RBD_TCP_CKSUM_ERR)) ||
289 	    (proto == RBD_FLAG_UDP && !(rx_status & RBD_UDP_CKSUM_ERR)))
290 		pflags |= HCK_FULLCKSUM_OK;
291 	if (proto != RBD_FLAG_NONE_IP && !(rx_status & RBD_IP_CKSUM_ERR))
292 		pflags |= HCK_IPV4_HDRCKSUM_OK;
293 	if (pflags != 0)  {
294 		mac_hcksum_set(mp, 0, 0, 0, 0, pflags);
295 	}
296 
297 	return (mp);
298 }
299 
300 /*
301  * Accept the packets received in rx ring.
302  *
303  * Returns a chain of mblks containing the received data, to be
304  * passed up to mac_rx().
305  * The routine returns only when a complete scan has been performed
306  * without finding any packets to receive.
307  * This function must SET the OWN bit of BD to indicate the packets
308  * it has accepted from the ring.
309  */
310 static mblk_t *rge_receive_ring(rge_t *rgep);
311 #pragma	inline(rge_receive_ring)
312 
313 static mblk_t *
314 rge_receive_ring(rge_t *rgep)
315 {
316 	rge_bd_t *hw_rbd_p;
317 	mblk_t *head;
318 	mblk_t **tail;
319 	mblk_t *mp;
320 	uint32_t slot;
321 
322 	ASSERT(mutex_owned(rgep->rx_lock));
323 
324 	/*
325 	 * Sync (all) the receive ring descriptors
326 	 * before accepting the packets they describe
327 	 */
328 	DMA_SYNC(rgep->rx_desc, DDI_DMA_SYNC_FORKERNEL);
329 	slot = rgep->rx_next;
330 	hw_rbd_p = &rgep->rx_ring[slot];
331 	head = NULL;
332 	tail = &head;
333 
334 	while (!(hw_rbd_p->flags_len & RGE_BSWAP_32(BD_FLAG_HW_OWN))) {
335 		if ((mp = rge_receive_packet(rgep, slot)) != NULL) {
336 			*tail = mp;
337 			tail = &mp->b_next;
338 		}
339 
340 		/*
341 		 * Clear RBD flags
342 		 */
343 		hw_rbd_p->flags_len =
344 		    RGE_BSWAP_32(rgep->rxbuf_size - rgep->head_room);
345 		HW_RBD_INIT(hw_rbd_p, slot);
346 		slot = NEXT(slot, RGE_RECV_SLOTS);
347 		hw_rbd_p = &rgep->rx_ring[slot];
348 	}
349 
350 	rgep->rx_next = slot;
351 	return (head);
352 }
353 
354 /*
355  * Receive all ready packets.
356  */
357 void rge_receive(rge_t *rgep);
358 #pragma	no_inline(rge_receive)
359 
360 void
361 rge_receive(rge_t *rgep)
362 {
363 	mblk_t *mp;
364 
365 	mutex_enter(rgep->rx_lock);
366 	mp = rge_receive_ring(rgep);
367 	mutex_exit(rgep->rx_lock);
368 
369 	if (mp != NULL)
370 		mac_rx(rgep->mh, NULL, mp);
371 }
372 
373 
374 #undef	RGE_DBG
375 #define	RGE_DBG		RGE_DBG_SEND	/* debug flag for this code	*/
376 
377 
378 /*
379  * ========== Send-side recycle routines ==========
380  */
381 static uint32_t rge_send_claim(rge_t *rgep);
382 #pragma	inline(rge_send_claim)
383 
384 static uint32_t
385 rge_send_claim(rge_t *rgep)
386 {
387 	uint32_t slot;
388 	uint32_t next;
389 
390 	mutex_enter(rgep->tx_lock);
391 	slot = rgep->tx_next;
392 	next = NEXT(slot, RGE_SEND_SLOTS);
393 	rgep->tx_next = next;
394 	rgep->tx_flow++;
395 	mutex_exit(rgep->tx_lock);
396 
397 	/*
398 	 * We check that our invariants still hold:
399 	 * +	the slot and next indexes are in range
400 	 * +	the slot must not be the last one (i.e. the *next*
401 	 *	index must not match the next-recycle index), 'cos
402 	 *	there must always be at least one free slot in a ring
403 	 */
404 	ASSERT(slot < RGE_SEND_SLOTS);
405 	ASSERT(next < RGE_SEND_SLOTS);
406 	ASSERT(next != rgep->tc_next);
407 
408 	return (slot);
409 }
410 
411 /*
412  * We don't want to call this function every time after a successful
413  * h/w transmit done in ISR.  Instead, we call this function in the
414  * rge_send() when there're few or no free tx BDs remained.
415  */
416 void rge_send_recycle(rge_t *rgep);
417 #pragma	inline(rge_send_recycle)
418 
419 void
420 rge_send_recycle(rge_t *rgep)
421 {
422 	rge_bd_t *hw_sbd_p;
423 	uint32_t tc_tail;
424 	uint32_t tc_head;
425 	uint32_t n;
426 
427 	mutex_enter(rgep->tc_lock);
428 	tc_head = rgep->tc_next;
429 	tc_tail = rgep->tc_tail;
430 	if (tc_head == tc_tail)
431 		goto resched;
432 
433 	do {
434 		tc_tail = LAST(tc_tail, RGE_SEND_SLOTS);
435 		hw_sbd_p = &rgep->tx_ring[tc_tail];
436 		if (tc_tail == tc_head) {
437 			if (hw_sbd_p->flags_len &
438 			    RGE_BSWAP_32(BD_FLAG_HW_OWN)) {
439 				/*
440 				 * Recyled nothing: bump the watchdog counter,
441 				 * thus guaranteeing that it's nonzero
442 				 * (watchdog activated).
443 				 */
444 				if (rgep->watchdog == 0)
445 					rgep->watchdog = 1;
446 				mutex_exit(rgep->tc_lock);
447 				return;
448 			}
449 			break;
450 		}
451 	} while (hw_sbd_p->flags_len & RGE_BSWAP_32(BD_FLAG_HW_OWN));
452 
453 	/*
454 	 * Recyled something :-)
455 	 */
456 	rgep->tc_next = NEXT(tc_tail, RGE_SEND_SLOTS);
457 	n = rgep->tc_next - tc_head;
458 	if (rgep->tc_next < tc_head)
459 		n += RGE_SEND_SLOTS;
460 	rge_atomic_renounce(&rgep->tx_free, n);
461 	rgep->watchdog = 0;
462 	ASSERT(rgep->tx_free <= RGE_SEND_SLOTS);
463 
464 resched:
465 	mutex_exit(rgep->tc_lock);
466 	if (rgep->resched_needed &&
467 	    rgep->rge_mac_state == RGE_MAC_STARTED) {
468 		rgep->resched_needed = B_FALSE;
469 		mac_tx_update(rgep->mh);
470 	}
471 }
472 
473 /*
474  * Send a message by copying it into a preallocated (and premapped) buffer
475  */
476 static void rge_send_copy(rge_t *rgep, mblk_t *mp, uint16_t tci);
477 #pragma	inline(rge_send_copy)
478 
479 static void
480 rge_send_copy(rge_t *rgep, mblk_t *mp, uint16_t tci)
481 {
482 	rge_bd_t *hw_sbd_p;
483 	sw_sbd_t *ssbdp;
484 	mblk_t *bp;
485 	char *txb;
486 	uint32_t slot;
487 	size_t totlen;
488 	size_t mblen;
489 	uint32_t pflags;
490 	struct ether_header *ethhdr;
491 	struct ip *ip_hdr;
492 
493 	/*
494 	 * IMPORTANT:
495 	 *	Up to the point where it claims a place, a send_msg()
496 	 *	routine can indicate failure by returning B_FALSE.  Once it's
497 	 *	claimed a place, it mustn't fail.
498 	 *
499 	 * In this version, there's no setup to be done here, and there's
500 	 * nothing that can fail, so we can go straight to claiming our
501 	 * already-reserved place on the train.
502 	 *
503 	 * This is the point of no return!
504 	 */
505 	slot = rge_send_claim(rgep);
506 	ssbdp = &rgep->sw_sbds[slot];
507 
508 	/*
509 	 * Copy the data into a pre-mapped buffer, which avoids the
510 	 * overhead (and complication) of mapping/unmapping STREAMS
511 	 * buffers and keeping hold of them until the DMA has completed.
512 	 *
513 	 * Because all buffers are the same size, and larger than the
514 	 * longest single valid message, we don't have to bother about
515 	 * splitting the message across multiple buffers either.
516 	 */
517 	txb = DMA_VPTR(ssbdp->pbuf);
518 	totlen = 0;
519 	bp = mp;
520 	if (tci != 0) {
521 		/*
522 		 * Do not copy the vlan tag
523 		 */
524 		bcopy(bp->b_rptr, txb, 2 * ETHERADDRL);
525 		txb += 2 * ETHERADDRL;
526 		totlen += 2 * ETHERADDRL;
527 		mblen = MBLKL(bp);
528 		ASSERT(mblen >= 2 * ETHERADDRL + VLAN_TAGSZ);
529 		mblen -= 2 * ETHERADDRL + VLAN_TAGSZ;
530 		if ((totlen += mblen) <= rgep->ethmax_size) {
531 			bcopy(bp->b_rptr + 2 * ETHERADDRL + VLAN_TAGSZ,
532 			    txb, mblen);
533 			txb += mblen;
534 		}
535 		bp = bp->b_cont;
536 		rgep->stats.obytes += VLAN_TAGSZ;
537 	}
538 	for (; bp != NULL; bp = bp->b_cont) {
539 		mblen = MBLKL(bp);
540 		if ((totlen += mblen) <= rgep->ethmax_size) {
541 			bcopy(bp->b_rptr, txb, mblen);
542 			txb += mblen;
543 		}
544 	}
545 	rgep->stats.obytes += totlen;
546 	rgep->stats.tx_pre_ismax = rgep->stats.tx_cur_ismax;
547 	if (totlen == rgep->ethmax_size)
548 		rgep->stats.tx_cur_ismax = B_TRUE;
549 	else
550 		rgep->stats.tx_cur_ismax = B_FALSE;
551 
552 	/*
553 	 * We'e reached the end of the chain; and we should have
554 	 * collected no more than ETHERMAX bytes into our buffer.
555 	 */
556 	ASSERT(bp == NULL);
557 	ASSERT(totlen <= rgep->ethmax_size);
558 	DMA_SYNC(ssbdp->pbuf, DDI_DMA_SYNC_FORDEV);
559 
560 	/*
561 	 * Update the hardware send buffer descriptor flags
562 	 */
563 	hw_sbd_p = &rgep->tx_ring[slot];
564 	ASSERT(hw_sbd_p == ssbdp->desc.mem_va);
565 	hw_sbd_p->flags_len = RGE_BSWAP_32(totlen & SBD_LEN_MASK);
566 	if (tci != 0) {
567 		tci = TCI_OS2CHIP(tci);
568 		hw_sbd_p->vlan_tag = RGE_BSWAP_32(tci);
569 		hw_sbd_p->vlan_tag |= RGE_BSWAP_32(SBD_VLAN_PKT);
570 	} else {
571 		hw_sbd_p->vlan_tag = 0;
572 	}
573 
574 	/*
575 	 * h/w checksum offload flags
576 	 */
577 	mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags);
578 	if (pflags & HCK_FULLCKSUM) {
579 		ASSERT(totlen >= sizeof (struct ether_header) +
580 		    sizeof (struct ip));
581 		ethhdr = (struct ether_header *)(DMA_VPTR(ssbdp->pbuf));
582 		/*
583 		 * Is the packet an IP(v4) packet?
584 		 */
585 		if (ntohs(ethhdr->ether_type) == ETHERTYPE_IP) {
586 			ip_hdr = (struct ip *)
587 			    ((uint8_t *)DMA_VPTR(ssbdp->pbuf) +
588 			    sizeof (struct ether_header));
589 			if (ip_hdr->ip_p == IPPROTO_TCP)
590 				hw_sbd_p->flags_len |=
591 				    RGE_BSWAP_32(SBD_FLAG_TCP_CKSUM);
592 			else if (ip_hdr->ip_p == IPPROTO_UDP)
593 				hw_sbd_p->flags_len |=
594 				    RGE_BSWAP_32(SBD_FLAG_UDP_CKSUM);
595 		}
596 	}
597 	if (pflags & HCK_IPV4_HDRCKSUM)
598 		hw_sbd_p->flags_len |= RGE_BSWAP_32(SBD_FLAG_IP_CKSUM);
599 
600 	HW_SBD_SET(hw_sbd_p, slot);
601 
602 	/*
603 	 * We're done.
604 	 * The message can be freed right away, as we've already
605 	 * copied the contents ...
606 	 */
607 	freemsg(mp);
608 }
609 
610 static boolean_t
611 rge_send(rge_t *rgep, mblk_t *mp)
612 {
613 	struct ether_vlan_header *ehp;
614 	uint16_t tci;
615 
616 	ASSERT(mp->b_next == NULL);
617 
618 	/*
619 	 * Try to reserve a place in the transmit ring.
620 	 */
621 	if (!rge_atomic_reserve(&rgep->tx_free, 1)) {
622 		RGE_DEBUG(("rge_send: no free slots"));
623 		rgep->stats.defer++;
624 		rgep->resched_needed = B_TRUE;
625 		return (B_FALSE);
626 	}
627 
628 	/*
629 	 * Determine if the packet is VLAN tagged.
630 	 */
631 	ASSERT(MBLKL(mp) >= sizeof (struct ether_header));
632 	tci = 0;
633 	ehp = (struct ether_vlan_header *)mp->b_rptr;
634 	if (ehp->ether_tpid == htons(ETHERTYPE_VLAN))
635 		tci = ntohs(ehp->ether_tci);
636 
637 	/*
638 	 * We've reserved a place :-)
639 	 * These ASSERTions check that our invariants still hold:
640 	 *	there must still be at least one free place
641 	 *	there must be at least one place NOT free (ours!)
642 	 */
643 	ASSERT(rgep->tx_free < RGE_SEND_SLOTS);
644 	rge_send_copy(rgep, mp, tci);
645 
646 	/*
647 	 * Trigger chip h/w transmit ...
648 	 */
649 	mutex_enter(rgep->tx_lock);
650 	if (--rgep->tx_flow == 0) {
651 		DMA_SYNC(rgep->tx_desc, DDI_DMA_SYNC_FORDEV);
652 		rgep->tc_tail = rgep->tx_next;
653 	}
654 	rgep->stats.opackets++;
655 	mutex_exit(rgep->tx_lock);
656 
657 	return (B_TRUE);
658 }
659 
660 uint_t
661 rge_reschedule(caddr_t arg1, caddr_t arg2)
662 {
663 	rge_t *rgep;
664 
665 	rgep = (rge_t *)arg1;
666 	_NOTE(ARGUNUSED(arg2))
667 
668 	rge_send_recycle(rgep);
669 
670 	if (rgep->chipid.is_pcie && rgep->tx_free != RGE_SEND_SLOTS) {
671 		/*
672 		 * It's observed that in current Realtek PCI-E chips, tx
673 		 * request of the second fragment for upper layer packets
674 		 * will be ignored if the hardware transmission is in
675 		 * progress and will not be processed when the tx engine
676 		 * is idle. So one solution is to re-issue the requests
677 		 * if there are untransmitted packets after tx interrupts
678 		 * occur.
679 		 */
680 		rge_tx_trigger(rgep);
681 	}
682 
683 	return (DDI_INTR_CLAIMED);
684 }
685 
686 /*
687  * rge_m_tx() - send a chain of packets
688  */
689 mblk_t *
690 rge_m_tx(void *arg, mblk_t *mp)
691 {
692 	rge_t *rgep = arg;		/* private device info	*/
693 	mblk_t *next;
694 	mblk_t *mp_org = mp;
695 
696 	ASSERT(mp != NULL);
697 
698 	rw_enter(rgep->errlock, RW_READER);
699 	if ((rgep->rge_mac_state != RGE_MAC_STARTED) ||
700 	    (rgep->rge_chip_state != RGE_CHIP_RUNNING) ||
701 	    (rgep->param_link_up != LINK_STATE_UP)) {
702 		rw_exit(rgep->errlock);
703 		RGE_DEBUG(("rge_m_tx: tx doesn't work"));
704 		freemsgchain(mp);
705 		return (NULL);
706 	}
707 
708 	while (mp != NULL) {
709 		next = mp->b_next;
710 		mp->b_next = NULL;
711 
712 		if (!rge_send(rgep, mp)) {
713 			mp->b_next = next;
714 			break;
715 		}
716 
717 		mp = next;
718 	}
719 	if (mp != mp_org) {
720 		rge_tx_trigger(rgep);
721 	}
722 	rw_exit(rgep->errlock);
723 
724 	return (mp);
725 }
726