1 /*
2 * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
3 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
4 * Copyright (c) 2002-2010 Mellanox Technologies LTD. All rights reserved.
5 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 *
35 */
36
37 /*
38 * Abstract:
39 * Implementation of osm_vl15_t.
40 * This object represents the VL15 Interface object.
41 * This object is part of the opensm family of objects.
42 */
43
44 #if HAVE_CONFIG_H
45 # include <config.h>
46 #endif /* HAVE_CONFIG_H */
47
48 #include <string.h>
49 #include <iba/ib_types.h>
50 #include <complib/cl_thread.h>
51 #include <opensm/osm_file_ids.h>
52 #define FILE_ID OSM_FILE_VL15INTF_C
53 #include <vendor/osm_vendor_api.h>
54 #include <opensm/osm_vl15intf.h>
55 #include <opensm/osm_madw.h>
56 #include <opensm/osm_log.h>
57 #include <opensm/osm_helper.h>
58
vl15_send_mad(osm_vl15_t * p_vl,osm_madw_t * p_madw)59 static void vl15_send_mad(osm_vl15_t * p_vl, osm_madw_t * p_madw)
60 {
61 ib_api_status_t status;
62 boolean_t resp_expected = p_madw->resp_expected;
63 ib_smp_t * p_smp;
64 ib_net16_t attr_id;
65 uint8_t method;
66
67 p_smp = osm_madw_get_smp_ptr(p_madw);
68 method = p_smp->method;
69 attr_id = p_smp->attr_id;
70
71 /*
72 Non-response-expected mads are not throttled on the wire
73 since we can have no confirmation that they arrived
74 at their destination.
75 */
76 if (resp_expected)
77 /*
78 Note that other threads may not see the response MAD
79 arrive before send() even returns.
80 In that case, the wire count would temporarily go negative.
81 To avoid this confusion, preincrement the counts on the
82 assumption that send() will succeed.
83 */
84 cl_atomic_inc(&p_vl->p_stats->qp0_mads_outstanding_on_wire);
85 else
86 cl_atomic_inc(&p_vl->p_stats->qp0_unicasts_sent);
87
88 cl_atomic_inc(&p_vl->p_stats->qp0_mads_sent);
89
90 status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
91 p_madw, p_madw->resp_expected);
92
93 if (status == IB_SUCCESS) {
94 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
95 "%u QP0 MADs on wire, %u outstanding, "
96 "%u unicasts sent, %u total sent\n",
97 p_vl->p_stats->qp0_mads_outstanding_on_wire,
98 p_vl->p_stats->qp0_mads_outstanding,
99 p_vl->p_stats->qp0_unicasts_sent,
100 p_vl->p_stats->qp0_mads_sent);
101 return;
102 }
103
104 OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E03: "
105 "MAD send failed (%s)\n", ib_get_err_str(status));
106
107 /*
108 The MAD was never successfully sent, so
109 fix up the pre-incremented count values.
110 */
111
112 /* Decrement qp0_mads_sent that were incremented in the code above.
113 qp0_mads_outstanding will be decremented by send error callback
114 (called by osm_vendor_send() */
115 cl_atomic_dec(&p_vl->p_stats->qp0_mads_sent);
116 if (!resp_expected) {
117 cl_atomic_dec(&p_vl->p_stats->qp0_unicasts_sent);
118 return;
119 }
120
121 /* need to cause heavy-sweep if resp_expected MAD sending failed */
122 OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E04: "
123 "%s method failed for attribute 0x%X (%s)\n",
124 method == IB_MAD_METHOD_SET ? "SET" : "GET",
125 cl_ntoh16(attr_id), ib_get_sm_attr_str(attr_id));
126
127 p_vl->p_subn->subnet_initialization_error = TRUE;
128
129 }
130
vl15_poller(IN void * p_ptr)131 static void vl15_poller(IN void *p_ptr)
132 {
133 ib_api_status_t status;
134 osm_madw_t *p_madw;
135 osm_vl15_t *p_vl = p_ptr;
136 cl_qlist_t *p_fifo;
137 int32_t max_smps = p_vl->max_wire_smps;
138 int32_t max_smps2 = p_vl->max_wire_smps2;
139
140 OSM_LOG_ENTER(p_vl->p_log);
141
142 if (p_vl->thread_state == OSM_THREAD_STATE_NONE)
143 p_vl->thread_state = OSM_THREAD_STATE_RUN;
144
145 while (p_vl->thread_state == OSM_THREAD_STATE_RUN) {
146 /*
147 Start servicing the FIFOs by pulling off MAD wrappers
148 and passing them to the transport interface.
149 There are lots of corner cases here so tread carefully.
150
151 The unicast FIFO has priority, since somebody is waiting
152 for a timely response.
153 */
154 cl_spinlock_acquire(&p_vl->lock);
155
156 if (cl_qlist_count(&p_vl->ufifo) != 0)
157 p_fifo = &p_vl->ufifo;
158 else
159 p_fifo = &p_vl->rfifo;
160
161 p_madw = (osm_madw_t *) cl_qlist_remove_head(p_fifo);
162
163 cl_spinlock_release(&p_vl->lock);
164
165 if (p_madw != (osm_madw_t *) cl_qlist_end(p_fifo)) {
166 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
167 "Servicing p_madw = %p\n", p_madw);
168 if (OSM_LOG_IS_ACTIVE_V2(p_vl->p_log, OSM_LOG_FRAMES))
169 osm_dump_dr_smp_v2(p_vl->p_log,
170 osm_madw_get_smp_ptr(p_madw),
171 FILE_ID, OSM_LOG_FRAMES);
172
173 vl15_send_mad(p_vl, p_madw);
174 } else
175 /*
176 The VL15 FIFO is empty, so we have nothing left to do.
177 */
178 status = cl_event_wait_on(&p_vl->signal,
179 EVENT_NO_TIMEOUT, TRUE);
180
181 while (p_vl->p_stats->qp0_mads_outstanding_on_wire >= max_smps &&
182 p_vl->thread_state == OSM_THREAD_STATE_RUN) {
183 status = cl_event_wait_on(&p_vl->signal,
184 p_vl->max_smps_timeout,
185 TRUE);
186 if (status == CL_TIMEOUT) {
187 if (max_smps < max_smps2)
188 max_smps++;
189 break;
190 } else if (status != CL_SUCCESS) {
191 OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E02: "
192 "Event wait failed (%s)\n",
193 CL_STATUS_MSG(status));
194 break;
195 }
196 max_smps = p_vl->max_wire_smps;
197 }
198 }
199
200 /*
201 since we abort immediately when the state != OSM_THREAD_STATE_RUN
202 we might have some mads on the queues. After the thread exits
203 the vl15 destroy routine should put these mads back...
204 */
205
206 OSM_LOG_EXIT(p_vl->p_log);
207 }
208
osm_vl15_construct(IN osm_vl15_t * p_vl)209 void osm_vl15_construct(IN osm_vl15_t * p_vl)
210 {
211 memset(p_vl, 0, sizeof(*p_vl));
212 p_vl->state = OSM_VL15_STATE_INIT;
213 p_vl->thread_state = OSM_THREAD_STATE_NONE;
214 cl_event_construct(&p_vl->signal);
215 cl_spinlock_construct(&p_vl->lock);
216 cl_qlist_init(&p_vl->rfifo);
217 cl_qlist_init(&p_vl->ufifo);
218 cl_thread_construct(&p_vl->poller);
219 }
220
osm_vl15_destroy(IN osm_vl15_t * p_vl,IN struct osm_mad_pool * p_pool)221 void osm_vl15_destroy(IN osm_vl15_t * p_vl, IN struct osm_mad_pool *p_pool)
222 {
223 osm_madw_t *p_madw;
224
225 OSM_LOG_ENTER(p_vl->p_log);
226
227 /*
228 Signal our threads that we're leaving.
229 */
230 p_vl->thread_state = OSM_THREAD_STATE_EXIT;
231
232 /*
233 Don't trigger unless event has been initialized.
234 Destroy the thread before we tear down the other objects.
235 */
236 if (p_vl->state != OSM_VL15_STATE_INIT)
237 cl_event_signal(&p_vl->signal);
238
239 cl_thread_destroy(&p_vl->poller);
240
241 /*
242 Return the outstanding messages to the pool
243 */
244
245 cl_spinlock_acquire(&p_vl->lock);
246
247 while (!cl_is_qlist_empty(&p_vl->rfifo)) {
248 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
249 osm_mad_pool_put(p_pool, p_madw);
250 }
251 while (!cl_is_qlist_empty(&p_vl->ufifo)) {
252 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
253 osm_mad_pool_put(p_pool, p_madw);
254 }
255
256 cl_spinlock_release(&p_vl->lock);
257
258 cl_event_destroy(&p_vl->signal);
259 p_vl->state = OSM_VL15_STATE_INIT;
260 cl_spinlock_destroy(&p_vl->lock);
261
262 OSM_LOG_EXIT(p_vl->p_log);
263 }
264
osm_vl15_init(IN osm_vl15_t * p_vl,IN osm_vendor_t * p_vend,IN osm_log_t * p_log,IN osm_stats_t * p_stats,IN osm_subn_t * p_subn,IN int32_t max_wire_smps,IN int32_t max_wire_smps2,IN uint32_t max_smps_timeout)265 ib_api_status_t osm_vl15_init(IN osm_vl15_t * p_vl, IN osm_vendor_t * p_vend,
266 IN osm_log_t * p_log, IN osm_stats_t * p_stats,
267 IN osm_subn_t * p_subn,
268 IN int32_t max_wire_smps,
269 IN int32_t max_wire_smps2,
270 IN uint32_t max_smps_timeout)
271 {
272 ib_api_status_t status = IB_SUCCESS;
273
274 OSM_LOG_ENTER(p_log);
275
276 p_vl->p_vend = p_vend;
277 p_vl->p_log = p_log;
278 p_vl->p_stats = p_stats;
279 p_vl->p_subn = p_subn;
280 p_vl->max_wire_smps = max_wire_smps;
281 p_vl->max_wire_smps2 = max_wire_smps2;
282 p_vl->max_smps_timeout = max_wire_smps < max_wire_smps2 ?
283 max_smps_timeout : EVENT_NO_TIMEOUT;
284
285 status = cl_event_init(&p_vl->signal, FALSE);
286 if (status != IB_SUCCESS)
287 goto Exit;
288
289 p_vl->state = OSM_VL15_STATE_READY;
290
291 status = cl_spinlock_init(&p_vl->lock);
292 if (status != IB_SUCCESS)
293 goto Exit;
294
295 /*
296 Initialize the thread after all other dependent objects
297 have been initialized.
298 */
299 status = cl_thread_init(&p_vl->poller, vl15_poller, p_vl,
300 "opensm poller");
301 Exit:
302 OSM_LOG_EXIT(p_log);
303 return status;
304 }
305
osm_vl15_poll(IN osm_vl15_t * p_vl)306 void osm_vl15_poll(IN osm_vl15_t * p_vl)
307 {
308 OSM_LOG_ENTER(p_vl->p_log);
309
310 CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
311
312 /*
313 If we have room for more VL15 MADs on the wire,
314 then signal the poller thread.
315
316 This is not an airtight check, since the poller thread
317 could be just about to send another MAD as we signal
318 the event here. To cover this rare case, the poller
319 thread checks for a spurious wake-up.
320 */
321 if (p_vl->p_stats->qp0_mads_outstanding_on_wire <
322 (int32_t) p_vl->max_wire_smps) {
323 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
324 "Signalling poller thread\n");
325 cl_event_signal(&p_vl->signal);
326 }
327
328 OSM_LOG_EXIT(p_vl->p_log);
329 }
330
osm_vl15_post(IN osm_vl15_t * p_vl,IN osm_madw_t * p_madw)331 void osm_vl15_post(IN osm_vl15_t * p_vl, IN osm_madw_t * p_madw)
332 {
333 OSM_LOG_ENTER(p_vl->p_log);
334
335 CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
336
337 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG, "Posting p_madw = %p\n", p_madw);
338
339 /*
340 Determine in which fifo to place the pending madw.
341 */
342 cl_spinlock_acquire(&p_vl->lock);
343 if (p_madw->resp_expected == TRUE) {
344 cl_qlist_insert_tail(&p_vl->rfifo, &p_madw->list_item);
345 osm_stats_inc_qp0_outstanding(p_vl->p_stats);
346 } else
347 cl_qlist_insert_tail(&p_vl->ufifo, &p_madw->list_item);
348 cl_spinlock_release(&p_vl->lock);
349
350 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
351 "%u QP0 MADs on wire, %u QP0 MADs outstanding\n",
352 p_vl->p_stats->qp0_mads_outstanding_on_wire,
353 p_vl->p_stats->qp0_mads_outstanding);
354
355 osm_vl15_poll(p_vl);
356
357 OSM_LOG_EXIT(p_vl->p_log);
358 }
359
osm_vl15_shutdown(IN osm_vl15_t * p_vl,IN osm_mad_pool_t * p_mad_pool)360 void osm_vl15_shutdown(IN osm_vl15_t * p_vl, IN osm_mad_pool_t * p_mad_pool)
361 {
362 osm_madw_t *p_madw;
363
364 OSM_LOG_ENTER(p_vl->p_log);
365
366 /* we only should get here after the VL15 interface was initialized */
367 CL_ASSERT(p_vl->state == OSM_VL15_STATE_READY);
368
369 /* grab a lock on the object */
370 cl_spinlock_acquire(&p_vl->lock);
371
372 /* go over all outstanding MADs and retire their transactions */
373
374 /* first we handle the list of response MADs */
375 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
376 while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->ufifo)) {
377 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
378 "Releasing Response p_madw = %p\n", p_madw);
379
380 osm_mad_pool_put(p_mad_pool, p_madw);
381
382 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->ufifo);
383 }
384
385 /* Request MADs we send out */
386 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
387 while (p_madw != (osm_madw_t *) cl_qlist_end(&p_vl->rfifo)) {
388 OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
389 "Releasing Request p_madw = %p\n", p_madw);
390
391 osm_mad_pool_put(p_mad_pool, p_madw);
392 osm_stats_dec_qp0_outstanding(p_vl->p_stats);
393
394 p_madw = (osm_madw_t *) cl_qlist_remove_head(&p_vl->rfifo);
395 }
396
397 /* free the lock */
398 cl_spinlock_release(&p_vl->lock);
399
400 OSM_LOG_EXIT(p_vl->p_log);
401 }
402