xref: /titanic_50/usr/src/uts/common/io/bge/bge_send.c (revision 789d94c2889bedf502063bc22addcabfa798a438)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include "sys/bge_impl2.h"
30 
31 
32 /*
33  * The transmit-side code uses an allocation process which is similar
34  * to some theme park roller-coaster rides, where riders sit in cars
35  * that can go individually, but work better in a train.
36  *
37  * 1)	RESERVE a place - this doesn't refer to any specific car or
38  *	seat, just that you will get a ride.  The attempt to RESERVE a
39  *	place can fail if all spaces in all cars are already committed.
40  *
41  * 2)	Prepare yourself; this may take an arbitrary (but not unbounded)
42  *	time, and you can back out at this stage, in which case you must
43  *	give up (RENOUNCE) your place.
44  *
45  * 3)	CLAIM your space - a specific car (the next sequentially
46  *	numbered one) is allocated at this stage, and is guaranteed
47  *	to be part of the next train to depart.  Once you've done
48  *	this, you can't back out, nor wait for any external event
49  *	or resource.
50  *
51  * 4)	Occupy your car - when all CLAIMED cars are OCCUPIED, they
52  *	all depart together as a single train!
53  *
54  * 5)	At the end of the ride, you climb out of the car and RENOUNCE
55  *	your right to it, so that it can be recycled for another rider.
56  *
57  * For each rider, these have to occur in this order, but the riders
58  * don't have to stay in the same order at each stage.  In particular,
59  * they may overtake each other between RESERVING a place and CLAIMING
60  * it, or between CLAIMING and OCCUPYING a space.
61  *
62  * Once a car is CLAIMED, the train currently being assembled can't go
63  * without that car (this guarantees that the cars in a single train
64  * make up a consecutively-numbered set).  Therefore, when any train
65  * leaves, we know there can't be any riders in transit between CLAIMING
66  * and OCCUPYING their cars.  There can be some who have RESERVED but
67  * not yet CLAIMED their places.  That's OK, though, because they'll go
68  * into the next train.
69  */
70 
71 #define	BGE_DBG		BGE_DBG_SEND	/* debug flag for this code	*/
72 
73 
74 /*
75  * ========== Send-side recycle routines ==========
76  */
77 
78 /*
79  * Recycle all the completed buffers in the specified send ring up to
80  * (but not including) the consumer index in the status block.
81  *
82  * This function must advance (srp->tc_next) AND adjust (srp->tx_free)
83  * to account for the packets it has recycled.
84  *
85  * This is a trivial version that just does that and nothing more, but
86  * it suffices while there's only one method for sending messages (by
87  * copying) and that method doesn't need any special per-buffer action
88  * for recycling.
89  */
90 static void bge_recycle_ring(bge_t *bgep, send_ring_t *srp);
91 #pragma	inline(bge_recycle_ring)
92 
93 static void
94 bge_recycle_ring(bge_t *bgep, send_ring_t *srp)
95 {
96 	uint64_t slot;
97 	uint64_t n;
98 
99 	_NOTE(ARGUNUSED(bgep))
100 
101 	ASSERT(mutex_owned(srp->tc_lock));
102 
103 	slot = *srp->cons_index_p;			/* volatile	*/
104 	n = slot - srp->tc_next;
105 	if (slot < srp->tc_next)
106 		n += srp->desc.nslots;
107 
108 	/*
109 	 * We're about to release one or more places :-)
110 	 * These ASSERTions check that our invariants still hold:
111 	 *	there must always be at least one free place
112 	 *	at this point, there must be at least one place NOT free
113 	 *	we're not about to free more places than were claimed!
114 	 */
115 	ASSERT(srp->tx_free > 0);
116 	ASSERT(srp->tx_free < srp->desc.nslots);
117 	ASSERT(srp->tx_free + n <= srp->desc.nslots);
118 
119 	srp->tc_next = slot;
120 	bge_atomic_renounce(&srp->tx_free, n);
121 
122 	/*
123 	 * Reset the watchdog count: to 0 if all buffers are
124 	 * now free, or to 1 if some are still outstanding.
125 	 * Note: non-synchonised access here means we may get
126 	 * the "wrong" answer, but only in a harmless fashion
127 	 * (i.e. we deactivate the watchdog because all buffers
128 	 * are apparently free, even though another thread may
129 	 * have claimed one before we leave here; in this case
130 	 * the watchdog will restart on the next send() call).
131 	 */
132 	bgep->watchdog = srp->tx_free == srp->desc.nslots ? 0 : 1;
133 }
134 
135 /*
136  * Recycle all returned slots in all rings.
137  *
138  * To give priority to low-numbered rings, whenever we have recycled any
139  * slots in any ring except 0, we restart scanning again from ring 0.
140  * Thus, for example, if rings 0, 3, and 10 are carrying traffic, the
141  * pattern of recycles might go 0, 3, 10, 3, 0, 10, 0:
142  *
143  *	0	found some - recycle them
144  *	1..2					none found
145  *	3	found some - recycle them	and restart scan
146  *	0..9					none found
147  *	10	found some - recycle them	and restart scan
148  *	0..2					none found
149  *	3	found some more - recycle them	and restart scan
150  *	0	found some more - recycle them
151  *	0..9					none found
152  *	10	found some more - recycle them	and restart scan
153  *	0	found some more - recycle them
154  *	1..15					none found
155  *
156  * The routine returns only when a complete scan has been performed
157  * without finding any slots to recycle.
158  *
159  * Note: the expression (BGE_SEND_RINGS_USED > 1) yields a compile-time
160  * constant and allows the compiler to optimise away the outer do-loop
161  * if only one send ring is being used.
162  */
163 void bge_recycle(bge_t *bgep, bge_status_t *bsp);
164 #pragma	no_inline(bge_recycle)
165 
166 void
167 bge_recycle(bge_t *bgep, bge_status_t *bsp)
168 {
169 	send_ring_t *srp;
170 	uint64_t ring;
171 	uint64_t tx_rings = bgep->chipid.tx_rings;
172 
173 restart:
174 	ring = 0;
175 	srp = &bgep->send[ring];
176 	do {
177 		/*
178 		 * For each ring, (srp->cons_index_p) points to the
179 		 * proper index within the status block (which has
180 		 * already been sync'd by the caller).
181 		 */
182 		ASSERT(srp->cons_index_p == SEND_INDEX_P(bsp, ring));
183 
184 		if (*srp->cons_index_p == srp->tc_next)
185 			continue;		/* no slots to recycle	*/
186 
187 		mutex_enter(srp->tc_lock);
188 		bge_recycle_ring(bgep, srp);
189 		mutex_exit(srp->tc_lock);
190 
191 		if (bgep->resched_needed && !bgep->resched_running) {
192 			bgep->resched_running = B_TRUE;
193 			ddi_trigger_softintr(bgep->resched_id);
194 		}
195 		/*
196 		 * Restart from ring 0, if we're not on ring 0 already.
197 		 * As H/W selects send BDs totally based on priority and
198 		 * available BDs on the higher priority ring are always
199 		 * selected first, driver should keep consistence with H/W
200 		 * and gives lower-numbered ring with higher priority.
201 		 */
202 		if (tx_rings > 1 && ring > 0)
203 			goto restart;
204 
205 		/*
206 		 * Loop over all rings (if there *are* multiple rings)
207 		 */
208 	} while (++srp, ++ring < tx_rings);
209 }
210 
211 
212 /*
213  * ========== Send-side transmit routines ==========
214  */
215 
216 /*
217  * CLAIM an already-reserved place on the next train
218  *
219  * This is the point of no return!
220  */
221 static uint64_t bge_send_claim(bge_t *bgep, send_ring_t *srp);
222 #pragma	inline(bge_send_claim)
223 
224 static uint64_t
225 bge_send_claim(bge_t *bgep, send_ring_t *srp)
226 {
227 	uint64_t slot;
228 
229 	mutex_enter(srp->tx_lock);
230 	atomic_add_64(&srp->tx_flow, 1);
231 	slot = bge_atomic_claim(&srp->tx_next, srp->desc.nslots);
232 	mutex_exit(srp->tx_lock);
233 
234 	/*
235 	 * Bump the watchdog counter, thus guaranteeing that it's
236 	 * nonzero (watchdog activated).  Note that non-synchonised
237 	 * access here means we may race with the reclaim() code
238 	 * above, but the outcome will be harmless.  At worst, the
239 	 * counter may not get reset on a partial reclaim; but the
240 	 * large trigger threshold makes false positives unlikely
241 	 */
242 	bgep->watchdog += 1;
243 
244 	return (slot);
245 }
246 
247 /*
248  * Send a message by copying it into a preallocated (and premapped) buffer
249  */
250 static enum send_status bge_send_copy(bge_t *bgep, mblk_t *mp,
251 	send_ring_t *srp, uint16_t tci);
252 #pragma	inline(bge_send_copy)
253 
254 static enum send_status
255 bge_send_copy(bge_t *bgep, mblk_t *mp, send_ring_t *srp, uint16_t tci)
256 {
257 	bge_sbd_t *hw_sbd_p;
258 	sw_sbd_t *ssbdp;
259 	mblk_t *bp;
260 	char *txb;
261 	uint64_t slot;
262 	size_t totlen;
263 	size_t mblen;
264 	uint32_t pflags;
265 
266 	BGE_TRACE(("bge_send_copy($%p, $%p, $%p, 0x%x)",
267 		(void *)bgep, (void *)mp, (void *)srp));
268 
269 	/*
270 	 * IMPORTANT:
271 	 *	Up to the point where it claims a place, a send_msg()
272 	 *	routine can indicate failure by returning SEND_FAIL.
273 	 *	Once it's claimed a place, it mustn't fail.
274 	 *
275 	 * In this version, there's no setup to be done here, and there's
276 	 * nothing that can fail, so we can go straight to claiming our
277 	 * already-reserved place on the train.
278 	 *
279 	 * This is the point of no return!
280 	 */
281 	slot = bge_send_claim(bgep, srp);
282 	ssbdp = &srp->sw_sbds[slot];
283 
284 	/*
285 	 * Copy the data into a pre-mapped buffer, which avoids the
286 	 * overhead (and complication) of mapping/unmapping STREAMS
287 	 * buffers and keeping hold of them until the DMA has completed.
288 	 *
289 	 * Because all buffers are the same size, and larger than the
290 	 * longest single valid message, we don't have to bother about
291 	 * splitting the message across multiple buffers either.
292 	 */
293 	txb = DMA_VPTR(ssbdp->pbuf);
294 	for (totlen = 0, bp = mp; bp != NULL; bp = bp->b_cont) {
295 		mblen = bp->b_wptr - bp->b_rptr;
296 		if ((totlen += mblen) <= bgep->chipid.ethmax_size) {
297 			bcopy(bp->b_rptr, txb, mblen);
298 			txb += mblen;
299 		}
300 	}
301 
302 	/*
303 	 * We'e reached the end of the chain; and we should have
304 	 * collected no more than ETHERMAX bytes into our buffer.
305 	 */
306 	ASSERT(bp == NULL);
307 	ASSERT(totlen <= bgep->chipid.ethmax_size);
308 	DMA_SYNC(ssbdp->pbuf, DDI_DMA_SYNC_FORDEV);
309 
310 	/*
311 	 * Update the hardware send buffer descriptor; then we're done.
312 	 * The return status indicates that the message can be freed
313 	 * right away, as we've already copied the contents ...
314 	 */
315 	hw_sbd_p = DMA_VPTR(ssbdp->desc);
316 	hw_sbd_p->host_buf_addr = ssbdp->pbuf.cookie.dmac_laddress;
317 	hw_sbd_p->len = totlen;
318 	hw_sbd_p->flags = SBD_FLAG_PACKET_END;
319 	if (tci != 0) {
320 		hw_sbd_p->vlan_tci = tci;
321 		hw_sbd_p->flags |= SBD_FLAG_VLAN_TAG;
322 	}
323 
324 	hcksum_retrieve(mp, NULL, NULL, NULL, NULL, NULL, NULL, &pflags);
325 	if (pflags & HCK_IPV4_HDRCKSUM)
326 		hw_sbd_p->flags |= SBD_FLAG_IP_CKSUM;
327 	if (pflags & HCK_FULLCKSUM)
328 		hw_sbd_p->flags |= SBD_FLAG_TCP_UDP_CKSUM;
329 
330 	return (SEND_FREE);
331 }
332 
333 static boolean_t
334 bge_send(bge_t *bgep, mblk_t *mp)
335 {
336 	send_ring_t *srp;
337 	enum send_status status;
338 	struct ether_vlan_header *ehp;
339 	boolean_t need_strip = B_FALSE;
340 	bge_status_t *bsp;
341 	uint16_t tci;
342 	uint_t ring = 0;
343 
344 	ASSERT(mp->b_next == NULL);
345 
346 	/*
347 	 * Determine if the packet is VLAN tagged.
348 	 */
349 	ASSERT(MBLKL(mp) >= sizeof (struct ether_header));
350 	ehp = (struct ether_vlan_header *)mp->b_rptr;
351 
352 	if (ehp->ether_tpid == htons(VLAN_TPID)) {
353 		if (MBLKL(mp) < sizeof (struct ether_vlan_header)) {
354 			uint32_t pflags;
355 
356 			/*
357 			 * Need to preserve checksum flags across pullup.
358 			 */
359 			hcksum_retrieve(mp, NULL, NULL, NULL, NULL, NULL,
360 			    NULL, &pflags);
361 
362 			if (!pullupmsg(mp,
363 			    sizeof (struct ether_vlan_header))) {
364 				BGE_DEBUG(("bge_send: pullup failure"));
365 				bgep->resched_needed = B_TRUE;
366 				return (B_FALSE);
367 			}
368 
369 			(void) hcksum_assoc(mp, NULL, NULL, NULL, NULL, NULL,
370 			    NULL, pflags, KM_NOSLEEP);
371 		}
372 
373 		ehp = (struct ether_vlan_header *)mp->b_rptr;
374 		need_strip = B_TRUE;
375 	}
376 
377 	/*
378 	 * Try to reserve a place in the chosen ring. Shouldn't try next
379 	 * higher-numbered (lower-priority) ring, if there aren't any
380 	 * available. Otherwise, packets with same priority may get
381 	 * transmission starvation.
382 	 */
383 	srp = &bgep->send[ring];
384 	if (!bge_atomic_reserve(&srp->tx_free, 1)) {
385 		BGE_DEBUG(("bge_send: no free slots"));
386 		bgep->resched_needed = B_TRUE;
387 		return (B_FALSE);
388 	}
389 
390 	/*
391 	 * Now that we know that there is space to transmit the packet
392 	 * strip any VLAN tag that is present.
393 	 */
394 	if (need_strip) {
395 		tci = ntohs(ehp->ether_tci);
396 
397 		(void) memmove(mp->b_rptr + VLAN_TAGSZ, mp->b_rptr,
398 		    2 * ETHERADDRL);
399 		mp->b_rptr += VLAN_TAGSZ;
400 	} else {
401 		tci = 0;
402 	}
403 
404 	if (srp->tx_free <= 16) {
405 		bsp = DMA_VPTR(bgep->status_block);
406 		bge_recycle(bgep, bsp);
407 	}
408 	/*
409 	 * We've reserved a place :-)
410 	 * These ASSERTions check that our invariants still hold:
411 	 *	there must still be at least one free place
412 	 *	there must be at least one place NOT free (ours!)
413 	 */
414 	ASSERT(srp->tx_free > 0);
415 	ASSERT(srp->tx_free < srp->desc.nslots);
416 
417 	if ((status = bge_send_copy(bgep, mp, srp, tci)) == SEND_FAIL) {
418 		/*
419 		 * The send routine failed :(  So we have to renounce
420 		 * our reservation before returning the error.
421 		 */
422 		bge_atomic_renounce(&srp->tx_free, 1);
423 		bgep->resched_needed = B_TRUE;
424 		return (B_FALSE);
425 	}
426 
427 	/*
428 	 * The send routine succeeded; it will have updated the
429 	 * h/w ring descriptor, and the <tx_next> and <tx_flow>
430 	 * counters.
431 	 *
432 	 * Because there can be multiple concurrent threads in
433 	 * transit through this code, we only want to prod the
434 	 * hardware once the last one is departing ...
435 	 */
436 	mutex_enter(srp->tx_lock);
437 	if (--srp->tx_flow == 0) {
438 		DMA_SYNC(srp->desc, DDI_DMA_SYNC_FORDEV);
439 		bge_mbx_put(bgep, srp->chip_mbx_reg, srp->tx_next);
440 	}
441 	mutex_exit(srp->tx_lock);
442 
443 	if (status == SEND_FREE)
444 		freemsg(mp);
445 	return (B_TRUE);
446 }
447 
448 uint_t
449 bge_reschedule(caddr_t arg)
450 {
451 	bge_t *bgep;
452 
453 	bgep = (bge_t *)arg;
454 
455 	BGE_TRACE(("bge_reschedule($%p)", (void *)bgep));
456 
457 	if (bgep->bge_mac_state == BGE_MAC_STARTED && bgep->resched_needed) {
458 		mac_tx_update(bgep->macp);
459 		bgep->resched_needed = B_FALSE;
460 		bgep->resched_running = B_FALSE;
461 	}
462 
463 	return (DDI_INTR_CLAIMED);
464 }
465 
466 /*
467  * bge_m_tx() - send a chain of packets
468  */
469 mblk_t *
470 bge_m_tx(void *arg, mblk_t *mp)
471 {
472 	bge_t *bgep = arg;		/* private device info	*/
473 	mblk_t *next;
474 
475 	BGE_TRACE(("bge_m_tx($%p, $%p)", arg, (void *)mp));
476 
477 	ASSERT(mp != NULL);
478 	ASSERT(bgep->bge_mac_state == BGE_MAC_STARTED);
479 
480 	if (bgep->bge_chip_state != BGE_CHIP_RUNNING) {
481 		BGE_DEBUG(("bge_m_tx: chip not running"));
482 		return (mp);
483 	}
484 
485 	rw_enter(bgep->errlock, RW_READER);
486 	while (mp != NULL) {
487 		next = mp->b_next;
488 		mp->b_next = NULL;
489 
490 		if (!bge_send(bgep, mp)) {
491 			mp->b_next = next;
492 			break;
493 		}
494 
495 		mp = next;
496 	}
497 	rw_exit(bgep->errlock);
498 
499 	return (mp);
500 }
501