1 /*
2 * Copyright (c) 2002-2013 Mellanox Technologies LTD. All rights reserved.
3 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 *
35 */
36
37 /*
38 * Abstract:
39 * Implementation of osm_sm_state_mgr_t.
40 * This file implements the SM State Manager object.
41 */
42
43 #if HAVE_CONFIG_H
44 # include <config.h>
45 #endif /* HAVE_CONFIG_H */
46
47 #include <string.h>
48 #include <time.h>
49 #include <iba/ib_types.h>
50 #include <complib/cl_passivelock.h>
51 #include <complib/cl_debug.h>
52 #include <opensm/osm_file_ids.h>
53 #define FILE_ID OSM_FILE_SM_STATE_MGR_C
54 #include <opensm/osm_sm.h>
55 #include <opensm/osm_madw.h>
56 #include <opensm/osm_switch.h>
57 #include <opensm/osm_log.h>
58 #include <opensm/osm_subnet.h>
59 #include <opensm/osm_helper.h>
60 #include <opensm/osm_msgdef.h>
61 #include <opensm/osm_node.h>
62 #include <opensm/osm_port.h>
63 #include <vendor/osm_vendor_api.h>
64 #include <opensm/osm_helper.h>
65 #include <opensm/osm_opensm.h>
66
osm_report_sm_state(osm_sm_t * sm)67 void osm_report_sm_state(osm_sm_t * sm)
68 {
69 char buf[64];
70 const char *state_str = osm_get_sm_mgr_state_str(sm->p_subn->sm_state);
71
72 osm_log_v2(sm->p_log, OSM_LOG_SYS, FILE_ID, "Entering %s state\n", state_str);
73 snprintf(buf, sizeof(buf), "ENTERING SM %s STATE", state_str);
74 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, buf);
75 }
76
sm_state_mgr_send_master_sm_info_req(osm_sm_t * sm,uint8_t sm_state)77 static boolean_t sm_state_mgr_send_master_sm_info_req(osm_sm_t * sm, uint8_t sm_state)
78 {
79 osm_madw_context_t context;
80 const osm_port_t *p_port;
81 ib_api_status_t status;
82 osm_dr_path_t dr_path;
83 ib_net64_t guid;
84 boolean_t sent_req = FALSE;
85
86 OSM_LOG_ENTER(sm->p_log);
87
88 memset(&context, 0, sizeof(context));
89 if (sm_state == IB_SMINFO_STATE_STANDBY) {
90 /*
91 * We are in STANDBY state - this means we need to poll the
92 * master SM (according to master_guid).
93 * Send a query of SubnGet(SMInfo) to the subn
94 * master_sm_base_lid object.
95 */
96 guid = sm->master_sm_guid;
97 } else {
98 /*
99 * We are not in STANDBY - this means we are in MASTER state -
100 * so we need to poll the SM that is saved in polling_sm_guid
101 * under sm.
102 * Send a query of SubnGet(SMInfo) to that SM.
103 */
104 guid = sm->polling_sm_guid;
105 }
106
107 /* Verify that SM is not polling itself */
108 if (guid == sm->p_subn->sm_port_guid) {
109 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
110 "OpenSM doesn't poll itself\n");
111 goto Exit;
112 }
113
114 p_port = osm_get_port_by_guid(sm->p_subn, guid);
115
116 if (p_port == NULL) {
117 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3203: "
118 "No port object for GUID 0x%016" PRIx64 "\n",
119 cl_ntoh64(guid));
120 goto Exit;
121 }
122
123 context.smi_context.port_guid = guid;
124 context.smi_context.set_method = FALSE;
125 memcpy(&dr_path, osm_physp_get_dr_path_ptr(p_port->p_physp), sizeof(osm_dr_path_t));
126
127 status = osm_req_get(sm, &dr_path,
128 IB_MAD_ATTR_SM_INFO, 0, FALSE,
129 ib_port_info_get_m_key(&p_port->p_physp->port_info),
130 CL_DISP_MSGID_NONE, &context);
131
132 if (status != IB_SUCCESS)
133 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3204: "
134 "Failure requesting SMInfo (%s)\n",
135 ib_get_err_str(status));
136 else
137 sent_req = TRUE;
138
139 Exit:
140 OSM_LOG_EXIT(sm->p_log);
141
142 return (sent_req);
143 }
144
sm_state_mgr_start_polling(osm_sm_t * sm)145 static void sm_state_mgr_start_polling(osm_sm_t * sm)
146 {
147 uint32_t timeout;
148 cl_status_t cl_status;
149
150 OSM_LOG_ENTER(sm->p_log);
151
152 /*
153 * Init the retry_number back to zero - need to restart counting
154 */
155 sm->retry_number = 0;
156
157 /*
158 * Send a SubnGet(SMInfo) query to the current (or new) master found.
159 */
160 CL_PLOCK_ACQUIRE(sm->p_lock);
161 timeout = sm->p_subn->opt.sminfo_polling_timeout;
162 sm_state_mgr_send_master_sm_info_req(sm, sm->p_subn->sm_state);
163 CL_PLOCK_RELEASE(sm->p_lock);
164
165 /*
166 * Start a timer that will wake up every sminfo_polling_timeout milliseconds.
167 * The callback of the timer will send a SubnGet(SMInfo) to the Master SM
168 * and restart the timer
169 */
170 cl_status = cl_timer_start(&sm->polling_timer, timeout);
171 if (cl_status != CL_SUCCESS)
172 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3210: "
173 "Failed to start polling timer\n");
174
175 OSM_LOG_EXIT(sm->p_log);
176 }
177
osm_sm_state_mgr_polling_callback(IN void * context)178 void osm_sm_state_mgr_polling_callback(IN void *context)
179 {
180 osm_sm_t *sm = context;
181 uint32_t timeout;
182 cl_status_t cl_status;
183 uint8_t sm_state;
184
185 OSM_LOG_ENTER(sm->p_log);
186
187 cl_spinlock_acquire(&sm->state_lock);
188 sm_state = sm->p_subn->sm_state;
189 cl_spinlock_release(&sm->state_lock);
190
191 CL_PLOCK_ACQUIRE(sm->p_lock);
192 timeout = sm->p_subn->opt.sminfo_polling_timeout;
193
194 /*
195 * We can be here in one of two cases:
196 * 1. We are a STANDBY sm polling on the master SM.
197 * 2. We are a MASTER sm, waiting for a handover from a remote master sm.
198 * If we are not in one of these cases - don't need to restart the poller.
199 */
200 if (!((sm_state == IB_SMINFO_STATE_MASTER &&
201 sm->polling_sm_guid != 0) ||
202 sm_state == IB_SMINFO_STATE_STANDBY)) {
203 CL_PLOCK_RELEASE(sm->p_lock);
204 goto Exit;
205 }
206
207 /*
208 * If we are a STANDBY sm and the osm_exit_flag is set, then let's
209 * signal the subnet_up. This is relevant for the case of running only
210 * once. In that case - the program is stuck until this signal is
211 * received. In other cases - it is not relevant whether or not the
212 * signal is on - since we are currently in exit flow
213 */
214 if (sm_state == IB_SMINFO_STATE_STANDBY && osm_exit_flag) {
215 CL_PLOCK_RELEASE(sm->p_lock);
216 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
217 "Signalling subnet_up_event\n");
218 cl_event_signal(&sm->subnet_up_event);
219 goto Exit;
220 }
221
222 /*
223 * If retry number reached the max_retry_number in the subnet opt - call
224 * osm_sm_state_mgr_process with signal OSM_SM_SIGNAL_POLLING_TIMEOUT
225 */
226 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "SM State %d (%s), Retry number:%d\n",
227 sm->p_subn->sm_state, osm_get_sm_mgr_state_str(sm->p_subn->sm_state),
228 sm->retry_number);
229
230 if (sm->retry_number > sm->p_subn->opt.polling_retry_number) {
231 CL_PLOCK_RELEASE(sm->p_lock);
232 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
233 "Reached polling_retry_number value in retry_number. "
234 "Go to DISCOVERY state\n");
235 osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_POLLING_TIMEOUT);
236 goto Exit;
237 }
238
239 /* Send a SubnGet(SMInfo) request to the remote sm (depends on our state) */
240 if (sm_state_mgr_send_master_sm_info_req(sm, sm_state)) {
241 /* Request sent, increment the retry number */
242 sm->retry_number++;
243 }
244
245 CL_PLOCK_RELEASE(sm->p_lock);
246
247 /* restart the timer */
248 cl_status = cl_timer_start(&sm->polling_timer, timeout);
249 if (cl_status != CL_SUCCESS)
250 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3211: "
251 "Failed to restart polling timer\n");
252
253 Exit:
254 OSM_LOG_EXIT(sm->p_log);
255 }
256
sm_state_mgr_signal_error(osm_sm_t * sm,IN osm_sm_signal_t signal)257 static void sm_state_mgr_signal_error(osm_sm_t * sm, IN osm_sm_signal_t signal)
258 {
259 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3207: "
260 "Invalid signal %s in state %s\n",
261 osm_get_sm_mgr_signal_str(signal),
262 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
263 }
264
osm_sm_state_mgr_signal_master_is_alive(osm_sm_t * sm)265 void osm_sm_state_mgr_signal_master_is_alive(osm_sm_t * sm)
266 {
267 OSM_LOG_ENTER(sm->p_log);
268 sm->retry_number = 0;
269 OSM_LOG_EXIT(sm->p_log);
270 }
271
osm_sm_state_mgr_process(osm_sm_t * sm,IN osm_sm_signal_t signal)272 ib_api_status_t osm_sm_state_mgr_process(osm_sm_t * sm,
273 IN osm_sm_signal_t signal)
274 {
275 ib_api_status_t status = IB_SUCCESS;
276
277 CL_ASSERT(sm);
278
279 OSM_LOG_ENTER(sm->p_log);
280
281 /*
282 * The state lock prevents many race conditions from screwing
283 * up the state transition process.
284 */
285 cl_spinlock_acquire(&sm->state_lock);
286
287 OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
288 "Received signal %s in state %s\n",
289 osm_get_sm_mgr_signal_str(signal),
290 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
291
292 switch (sm->p_subn->sm_state) {
293 case IB_SMINFO_STATE_DISCOVERING:
294 switch (signal) {
295 case OSM_SM_SIGNAL_DISCOVERY_COMPLETED:
296 /*
297 * Update the state of the SM to MASTER
298 */
299 /* Turn on the first_time_master_sweep flag */
300 sm->p_subn->sm_state = IB_SMINFO_STATE_MASTER;
301 osm_report_sm_state(sm);
302 /*
303 * Make sure to set the subnet master_sm_base_lid
304 * to the sm_base_lid value
305 */
306 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
307 sm->p_subn->first_time_master_sweep = TRUE;
308 sm->p_subn->master_sm_base_lid =
309 sm->p_subn->sm_base_lid;
310 CL_PLOCK_RELEASE(sm->p_lock);
311 break;
312 case OSM_SM_SIGNAL_MASTER_OR_HIGHER_SM_DETECTED:
313 /*
314 * Finished all discovery actions - move to STANDBY
315 * start the polling
316 */
317 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY;
318 osm_report_sm_state(sm);
319 /*
320 * Since another SM is doing the LFT config - we should not
321 * ignore the results of it
322 */
323 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
324 sm->p_subn->ignore_existing_lfts = FALSE;
325 CL_PLOCK_RELEASE(sm->p_lock);
326 sm_state_mgr_start_polling(sm);
327 break;
328 case OSM_SM_SIGNAL_HANDOVER:
329 /*
330 * Signal for a new sweep. We need to discover the other SM.
331 * If we already discovered this SM, and got the
332 * HANDOVER - this means the remote SM is of lower priority.
333 * In this case we will stop polling it (since it is a lower
334 * priority SM in STANDBY state).
335 */
336 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
337 break;
338 default:
339 sm_state_mgr_signal_error(sm, signal);
340 status = IB_INVALID_PARAMETER;
341 break;
342 }
343 break;
344
345 case IB_SMINFO_STATE_STANDBY:
346 switch (signal) {
347 case OSM_SM_SIGNAL_POLLING_TIMEOUT:
348 case OSM_SM_SIGNAL_DISCOVER:
349 /*
350 * case 1: Polling timeout occured - this means that the Master SM
351 * is no longer alive.
352 * case 2: Got a signal to move to DISCOVERING
353 * Move to DISCOVERING state and start sweeping
354 */
355 sm->p_subn->sm_state = IB_SMINFO_STATE_DISCOVERING;
356 osm_report_sm_state(sm);
357 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
358 sm->p_subn->coming_out_of_standby = TRUE;
359 CL_PLOCK_RELEASE(sm->p_lock);
360 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
361 break;
362 case OSM_SM_SIGNAL_DISABLE:
363 /*
364 * Update the state to NOT_ACTIVE
365 */
366 sm->p_subn->sm_state = IB_SMINFO_STATE_NOTACTIVE;
367 osm_report_sm_state(sm);
368 break;
369 case OSM_SM_SIGNAL_HANDOVER:
370 /*
371 * Update the state to MASTER, and start sweeping
372 * OPTIONAL: send ACKNOWLEDGE
373 */
374 /* Turn on the force_first_time_master_sweep flag */
375 /* We want full reconfiguration to occur on the first */
376 /* master sweep of this SM */
377 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
378 /*
379 * Make sure to set the subnet master_sm_base_lid
380 * to the sm_base_lid value
381 */
382 sm->p_subn->master_sm_base_lid =
383 sm->p_subn->sm_base_lid;
384
385 sm->p_subn->force_first_time_master_sweep = TRUE;
386 CL_PLOCK_RELEASE(sm->p_lock);
387
388 sm->p_subn->sm_state = IB_SMINFO_STATE_MASTER;
389 osm_report_sm_state(sm);
390 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
391 break;
392 case OSM_SM_SIGNAL_ACKNOWLEDGE:
393 /*
394 * Do nothing - already moved to STANDBY
395 */
396 break;
397 default:
398 sm_state_mgr_signal_error(sm, signal);
399 status = IB_INVALID_PARAMETER;
400 break;
401 }
402 break;
403
404 case IB_SMINFO_STATE_NOTACTIVE:
405 switch (signal) {
406 case OSM_SM_SIGNAL_STANDBY:
407 /*
408 * Update the state to STANDBY
409 * start the polling
410 */
411 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY;
412 osm_report_sm_state(sm);
413 sm_state_mgr_start_polling(sm);
414 break;
415 default:
416 sm_state_mgr_signal_error(sm, signal);
417 status = IB_INVALID_PARAMETER;
418 break;
419 }
420 break;
421
422 case IB_SMINFO_STATE_MASTER:
423 switch (signal) {
424 case OSM_SM_SIGNAL_POLLING_TIMEOUT:
425 /*
426 * We received a polling timeout - this means that we
427 * waited for a remote master sm to send us a handover,
428 * but didn't get it, and didn't get a response from
429 * that remote sm.
430 * We want to force a heavy sweep - hopefully this
431 * occurred because the remote sm died, and we'll find
432 * this out and configure the subnet after a heavy sweep.
433 * We also want to clear the polling_sm_guid - since
434 * we are done polling on that remote sm - we are
435 * sweeping again.
436 */
437 case OSM_SM_SIGNAL_HANDOVER:
438 /*
439 * If we received a handover in a master state - then we
440 * want to force a heavy sweep. This means that either
441 * we are in a sweep currently - in this case - no
442 * change, or we are in idle state - since we
443 * recognized a master SM before - so we want to make a
444 * heavy sweep and reconfigure the new subnet.
445 * We also want to clear the polling_sm_guid - since
446 * we are done polling on that remote sm - we got a
447 * handover from it.
448 */
449 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
450 "Forcing heavy sweep. Received signal %s\n",
451 osm_get_sm_mgr_signal_str(signal));
452 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
453 sm->polling_sm_guid = 0;
454 sm->p_subn->force_first_time_master_sweep = TRUE;
455 CL_PLOCK_RELEASE(sm->p_lock);
456 osm_sm_signal(sm, OSM_SIGNAL_SWEEP);
457 break;
458 case OSM_SM_SIGNAL_HANDOVER_SENT:
459 /*
460 * Just sent a HANDOVER signal - move to STANDBY
461 * start the polling
462 */
463 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY;
464 osm_report_sm_state(sm);
465 sm_state_mgr_start_polling(sm);
466 break;
467 case OSM_SM_SIGNAL_WAIT_FOR_HANDOVER:
468 /*
469 * We found a remote master SM, and we are waiting for
470 * it to handover the mastership to us. Need to start
471 * polling that SM, to make sure it is alive, if it
472 * isn't - then we should move back to discovering,
473 * since something must have happened to it.
474 */
475 sm_state_mgr_start_polling(sm);
476 break;
477 case OSM_SM_SIGNAL_DISCOVER:
478 sm->p_subn->sm_state = IB_SMINFO_STATE_DISCOVERING;
479 osm_report_sm_state(sm);
480 break;
481 default:
482 sm_state_mgr_signal_error(sm, signal);
483 status = IB_INVALID_PARAMETER;
484 break;
485 }
486 break;
487
488 default:
489 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3208: "
490 "Invalid state %s\n",
491 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
492
493 }
494
495 cl_spinlock_release(&sm->state_lock);
496
497 OSM_LOG_EXIT(sm->p_log);
498 return status;
499 }
500
osm_sm_state_mgr_check_legality(osm_sm_t * sm,IN osm_sm_signal_t signal)501 ib_api_status_t osm_sm_state_mgr_check_legality(osm_sm_t * sm,
502 IN osm_sm_signal_t signal)
503 {
504 ib_api_status_t status = IB_SUCCESS;
505
506 CL_ASSERT(sm);
507
508 OSM_LOG_ENTER(sm->p_log);
509
510 /*
511 * The state lock prevents many race conditions from screwing
512 * up the state transition process.
513 */
514 cl_spinlock_acquire(&sm->state_lock);
515
516 OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Received signal %s in state %s\n",
517 osm_get_sm_mgr_signal_str(signal),
518 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
519
520 switch (sm->p_subn->sm_state) {
521 case IB_SMINFO_STATE_DISCOVERING:
522 switch (signal) {
523 case OSM_SM_SIGNAL_DISCOVERY_COMPLETED:
524 case OSM_SM_SIGNAL_MASTER_OR_HIGHER_SM_DETECTED:
525 case OSM_SM_SIGNAL_HANDOVER:
526 status = IB_SUCCESS;
527 break;
528 default:
529 sm_state_mgr_signal_error(sm, signal);
530 status = IB_INVALID_PARAMETER;
531 break;
532 }
533 break;
534
535 case IB_SMINFO_STATE_STANDBY:
536 switch (signal) {
537 case OSM_SM_SIGNAL_POLLING_TIMEOUT:
538 case OSM_SM_SIGNAL_DISCOVER:
539 case OSM_SM_SIGNAL_DISABLE:
540 case OSM_SM_SIGNAL_HANDOVER:
541 case OSM_SM_SIGNAL_ACKNOWLEDGE:
542 status = IB_SUCCESS;
543 break;
544 default:
545 sm_state_mgr_signal_error(sm, signal);
546 status = IB_INVALID_PARAMETER;
547 break;
548 }
549 break;
550
551 case IB_SMINFO_STATE_NOTACTIVE:
552 switch (signal) {
553 case OSM_SM_SIGNAL_STANDBY:
554 status = IB_SUCCESS;
555 break;
556 default:
557 sm_state_mgr_signal_error(sm, signal);
558 status = IB_INVALID_PARAMETER;
559 break;
560 }
561 break;
562
563 case IB_SMINFO_STATE_MASTER:
564 switch (signal) {
565 case OSM_SM_SIGNAL_HANDOVER:
566 case OSM_SM_SIGNAL_HANDOVER_SENT:
567 status = IB_SUCCESS;
568 break;
569 default:
570 sm_state_mgr_signal_error(sm, signal);
571 status = IB_INVALID_PARAMETER;
572 break;
573 }
574 break;
575
576 default:
577 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3209: "
578 "Invalid state %s\n",
579 osm_get_sm_mgr_state_str(sm->p_subn->sm_state));
580 status = IB_INVALID_PARAMETER;
581
582 }
583
584 cl_spinlock_release(&sm->state_lock);
585
586 OSM_LOG_EXIT(sm->p_log);
587 return status;
588 }
589