1 /*
2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2011 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 *
35 */
36
37 /*
38 * Abstract:
39 * Implementation of the P_Key Manager (Partition Manager).
40 * This is part of the OpenSM.
41 */
42
43 #if HAVE_CONFIG_H
44 # include <config.h>
45 #endif /* HAVE_CONFIG_H */
46
47 #include <string.h>
48 #include <iba/ib_types.h>
49 #include <complib/cl_qmap.h>
50 #include <complib/cl_debug.h>
51 #include <opensm/osm_file_ids.h>
52 #define FILE_ID OSM_FILE_PKEY_MGR_C
53 #include <opensm/osm_node.h>
54 #include <opensm/osm_switch.h>
55 #include <opensm/osm_partition.h>
56 #include <opensm/osm_opensm.h>
57
58 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
59 uint16_t pkey_index);
60
61 /*
62 The max number of pkeys/pkey blocks for a physical port is located
63 in a different place for switch external ports (SwitchInfo) and the
64 rest of the ports (NodeInfo).
65 */
66 static uint16_t
pkey_mgr_get_physp_max_pkeys(IN const osm_physp_t * p_physp)67 pkey_mgr_get_physp_max_pkeys(IN const osm_physp_t * p_physp)
68 {
69 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
70 uint16_t num_pkeys = 0;
71
72 if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0))
73 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap);
74 else
75 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap);
76 return num_pkeys;
77 }
78
79 static uint16_t
pkey_mgr_get_physp_max_blocks(IN const osm_physp_t * p_physp)80 pkey_mgr_get_physp_max_blocks(IN const osm_physp_t * p_physp)
81 {
82 return ((pkey_mgr_get_physp_max_pkeys(p_physp) + 31) / 32);
83 }
84
85 /*
86 * Insert new pending pkey entry to the specific port pkey table
87 * pending pkeys. New entries are inserted at the back.
88 */
89 static void
pkey_mgr_process_physical_port(IN osm_log_t * p_log,IN osm_sm_t * sm,IN const ib_net16_t pkey,IN osm_physp_t * p_physp)90 pkey_mgr_process_physical_port(IN osm_log_t * p_log,
91 IN osm_sm_t * sm,
92 IN const ib_net16_t pkey,
93 IN osm_physp_t * p_physp)
94 {
95 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
96 osm_pkey_tbl_t *p_pkey_tbl;
97 ib_net16_t *p_orig_pkey;
98 osm_pending_pkey_t *p_pending;
99
100 p_pkey_tbl = &p_physp->pkeys;
101 p_pending = (osm_pending_pkey_t *) calloc(1, sizeof(osm_pending_pkey_t));
102 if (!p_pending) {
103 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: "
104 "Failed to allocate new pending pkey entry for node "
105 "0x%016" PRIx64 " port %u\n",
106 cl_ntoh64(osm_node_get_node_guid(p_node)),
107 osm_physp_get_port_num(p_physp));
108 return;
109 }
110 p_pending->pkey = pkey;
111 if (sm->p_subn->opt.allow_both_pkeys)
112 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, pkey);
113 else
114 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys,
115 ib_pkey_get_base(pkey));
116
117 if (!p_orig_pkey) {
118 p_pending->is_new = TRUE;
119 } else {
120 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) ==
121 ib_pkey_get_base(pkey));
122 p_pending->is_new = FALSE;
123 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey,
124 &p_pending->block,
125 &p_pending->index) !=
126 IB_SUCCESS) {
127 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: "
128 "Failed to obtain P_Key 0x%04x block and index "
129 "for node 0x%016" PRIx64 " port %u\n",
130 cl_ntoh16(ib_pkey_get_base(pkey)),
131 cl_ntoh64(osm_node_get_node_guid(p_node)),
132 osm_physp_get_port_num(p_physp));
133 free(p_pending);
134 return;
135 }
136 if (p_physp->pkeys.indx0_pkey) {
137 /*
138 * Remove the pkey that should be at index 0 from
139 * accum pkey if current position is not index 0
140 */
141 if (((sm->p_subn->opt.allow_both_pkeys &&
142 pkey == p_physp->pkeys.indx0_pkey) ||
143 (!sm->p_subn->opt.allow_both_pkeys &&
144 ib_pkey_get_base(pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) &&
145 (p_pending->block != 0 || p_pending->index != 0)) {
146 p_pending->is_new = TRUE;
147 clear_accum_pkey_index(p_pkey_tbl,
148 p_pending->block *
149 IB_NUM_PKEY_ELEMENTS_IN_BLOCK +
150 p_pending->index);
151 }
152
153 if (p_pending->block == 0 && p_pending->index == 0) {
154 /* Move the pkey away from index 0 */
155 if ((sm->p_subn->opt.allow_both_pkeys &&
156 pkey != p_physp->pkeys.indx0_pkey) ||
157 (!sm->p_subn->opt.allow_both_pkeys &&
158 ib_pkey_get_base(pkey) != ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) {
159 p_pending->is_new = TRUE;
160 clear_accum_pkey_index(p_pkey_tbl, 0);
161 }
162 }
163 } else {
164 /* If index 0 is occupied by non-default, it should reoccupied by pkey 0x7FFF */
165 if (p_pending->block == 0 && p_pending->index == 0) {
166 if (ib_pkey_get_base(pkey) != IB_DEFAULT_PARTIAL_PKEY) {
167 p_pending->is_new = TRUE;
168 clear_accum_pkey_index(p_pkey_tbl, 0);
169 }
170 /* Need to move default pkey to index 0 */
171 } else if ((sm->p_subn->opt.allow_both_pkeys &&
172 pkey == IB_DEFAULT_PKEY) ||
173 (!sm->p_subn->opt.allow_both_pkeys &&
174 ib_pkey_get_base(pkey) == IB_DEFAULT_PARTIAL_PKEY)) {
175 p_pending->is_new = TRUE;
176 clear_accum_pkey_index(p_pkey_tbl,
177 p_pending->block *
178 IB_NUM_PKEY_ELEMENTS_IN_BLOCK +
179 p_pending->index);
180 }
181 }
182
183 }
184 if (p_pending->is_new == TRUE)
185 cl_qlist_insert_tail(&p_pkey_tbl->pending,
186 (cl_list_item_t *) p_pending);
187 else
188 cl_qlist_insert_head(&p_pkey_tbl->pending,
189 (cl_list_item_t *) p_pending);
190
191 OSM_LOG(p_log, OSM_LOG_DEBUG,
192 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n",
193 cl_ntoh16(pkey), p_pending->is_new ? "inserted" : "updated",
194 cl_ntoh64(osm_node_get_node_guid(p_node)),
195 osm_physp_get_port_num(p_physp));
196 }
197
198 static void
pkey_mgr_process_partition_table(osm_log_t * p_log,osm_sm_t * sm,const osm_prtn_t * p_prtn,const boolean_t full)199 pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm,
200 const osm_prtn_t * p_prtn,
201 const boolean_t full)
202 {
203 const cl_map_t *p_tbl =
204 full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl;
205 cl_map_iterator_t i, i_next;
206 ib_net16_t pkey = p_prtn->pkey;
207 osm_physp_t *p_physp;
208
209 if (full)
210 pkey |= cl_hton16(0x8000);
211
212 i_next = cl_map_head(p_tbl);
213 while (i_next != cl_map_end(p_tbl)) {
214 i = i_next;
215 i_next = cl_map_next(i);
216 p_physp = cl_map_obj(i);
217 if (p_physp)
218 pkey_mgr_process_physical_port(p_log, sm, pkey,
219 p_physp);
220 }
221 }
222
223 static ib_api_status_t
pkey_mgr_update_pkey_entry(IN osm_sm_t * sm,IN const osm_physp_t * p_physp,IN const ib_pkey_table_t * block,IN const uint16_t block_index)224 pkey_mgr_update_pkey_entry(IN osm_sm_t * sm,
225 IN const osm_physp_t * p_physp,
226 IN const ib_pkey_table_t * block,
227 IN const uint16_t block_index)
228 {
229 osm_madw_context_t context;
230 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp);
231 osm_physp_t *physp0;
232 uint32_t attr_mod;
233 ib_net64_t m_key;
234
235 context.pkey_context.node_guid = osm_node_get_node_guid(p_node);
236 context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp);
237 context.pkey_context.set_method = TRUE;
238 attr_mod = block_index;
239 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH &&
240 osm_physp_get_port_num(p_physp) != 0) {
241 attr_mod |= osm_physp_get_port_num(p_physp) << 16;
242 physp0 = osm_node_get_physp_ptr(p_node, 0);
243 m_key = ib_port_info_get_m_key(&physp0->port_info);
244 } else
245 m_key = ib_port_info_get_m_key(&p_physp->port_info);
246 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
247 (uint8_t *) block, sizeof(*block),
248 IB_MAD_ATTR_P_KEY_TABLE,
249 cl_hton32(attr_mod), FALSE, m_key,
250 CL_DISP_MSGID_NONE, &context);
251 }
252
253 static ib_api_status_t
pkey_mgr_enforce_partition(IN osm_log_t * p_log,osm_sm_t * sm,IN osm_physp_t * p_physp,IN osm_partition_enforce_type_enum enforce_type)254 pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm,
255 IN osm_physp_t * p_physp,
256 IN osm_partition_enforce_type_enum enforce_type)
257 {
258 osm_madw_context_t context;
259 uint8_t payload[IB_SMP_DATA_SIZE];
260 ib_port_info_t *p_pi;
261 ib_net64_t m_key;
262 osm_physp_t *physp0;
263 ib_api_status_t status;
264 uint8_t enforce_bits;
265
266 p_pi = &p_physp->port_info;
267
268 if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_BOTH)
269 enforce_bits = 0xc;
270 else if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_IN)
271 enforce_bits = 0x8;
272 else
273 enforce_bits = 0x4;
274
275 if ((p_pi->vl_enforce & 0xc) == enforce_bits *
276 (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF)) {
277 OSM_LOG(p_log, OSM_LOG_DEBUG,
278 "No need to update PortInfo for "
279 "node 0x%016" PRIx64 " port %u (%s)\n",
280 cl_ntoh64(osm_node_get_node_guid
281 (osm_physp_get_node_ptr(p_physp))),
282 osm_physp_get_port_num(p_physp),
283 p_physp->p_node->print_desc);
284 return IB_SUCCESS;
285 }
286
287 memcpy(payload, p_pi, sizeof(ib_port_info_t));
288
289 p_pi = (ib_port_info_t *) payload;
290 p_pi->vl_enforce &= ~0xc;
291 if (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF)
292 p_pi->vl_enforce |= enforce_bits;
293
294 p_pi->state_info2 = 0;
295 ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE);
296
297 physp0 = osm_node_get_physp_ptr(p_physp->p_node, 0);
298 m_key = ib_port_info_get_m_key(&physp0->port_info);
299
300 context.pi_context.node_guid =
301 osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp));
302 context.pi_context.port_guid = osm_physp_get_port_guid(p_physp);
303 context.pi_context.set_method = TRUE;
304 context.pi_context.light_sweep = FALSE;
305 context.pi_context.active_transition = FALSE;
306 context.pi_context.client_rereg = FALSE;
307
308 status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp),
309 payload, sizeof(payload),
310 IB_MAD_ATTR_PORT_INFO,
311 cl_hton32(osm_physp_get_port_num(p_physp)),
312 FALSE, m_key,
313 CL_DISP_MSGID_NONE, &context);
314 if (status != IB_SUCCESS)
315 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: "
316 "Failed to set PortInfo for "
317 "node 0x%016" PRIx64 " port %u (%s)\n",
318 cl_ntoh64(osm_node_get_node_guid
319 (osm_physp_get_node_ptr(p_physp))),
320 osm_physp_get_port_num(p_physp),
321 p_physp->p_node->print_desc);
322 else
323 OSM_LOG(p_log, OSM_LOG_DEBUG,
324 "Set PortInfo for node 0x%016" PRIx64 " port %u (%s)\n",
325 cl_ntoh64(osm_node_get_node_guid
326 (osm_physp_get_node_ptr(p_physp))),
327 osm_physp_get_port_num(p_physp),
328 p_physp->p_node->print_desc);
329 return status;
330 }
331
clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,uint16_t pkey_index)332 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
333 uint16_t pkey_index)
334 {
335 uint16_t pkey_idx_bias, pkey_idx;
336 void *ptr;
337 uintptr_t pkey_idx_ptr;
338 cl_map_iterator_t map_iter, map_iter_temp;
339
340 map_iter = cl_map_head(&p_pkey_tbl->accum_pkeys);
341
342 pkey_idx_bias = pkey_index + 1; // adjust for pkey index bias in accum_pkeys
343
344 while (map_iter != cl_map_end(&p_pkey_tbl->accum_pkeys)) {
345 map_iter_temp = cl_map_next(map_iter);
346 ptr = (uint16_t *) cl_map_obj(map_iter);
347 CL_ASSERT(ptr);
348 pkey_idx_ptr = (uintptr_t) ptr;
349 pkey_idx = pkey_idx_ptr;
350 if (pkey_idx == pkey_idx_bias) {
351 cl_map_remove_item(&p_pkey_tbl->accum_pkeys, map_iter);
352 if (p_pkey_tbl->last_pkey_idx == pkey_idx)
353 osm_pkey_find_last_accum_pkey_index(p_pkey_tbl);
354 break;
355 }
356 map_iter = map_iter_temp;
357 }
358 }
359
last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,uint16_t * p_block_idx,uint8_t * p_pkey_idx)360 static int last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl,
361 uint16_t * p_block_idx,
362 uint8_t * p_pkey_idx)
363 {
364 if (p_pkey_tbl->last_pkey_idx) {
365 *p_block_idx = (p_pkey_tbl->last_pkey_idx - 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
366 *p_pkey_idx = (p_pkey_tbl->last_pkey_idx - 1) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
367 return 1;
368 }
369
370 return 0;
371 }
372
pkey_mgr_update_port(osm_log_t * p_log,osm_sm_t * sm,const osm_port_t * const p_port)373 static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm,
374 const osm_port_t * const p_port)
375 {
376 osm_physp_t *p_physp;
377 osm_node_t *p_node;
378 ib_pkey_table_t *block, *new_block;
379 osm_pkey_tbl_t *p_pkey_tbl;
380 uint16_t block_index;
381 uint8_t pkey_index;
382 uint16_t last_free_block_index = 0;
383 uint8_t last_free_pkey_index = 0;
384 uint16_t num_of_blocks;
385 uint16_t max_num_of_blocks;
386 ib_api_status_t status;
387 osm_pending_pkey_t *p_pending;
388 boolean_t found;
389 ib_pkey_table_t empty_block;
390 int ret = 0, full = 0;
391 void *ptr;
392 uintptr_t pkey_idx_ptr;
393 uint16_t pkey_idx;
394
395 p_physp = p_port->p_physp;
396 if (!p_physp)
397 return FALSE;
398
399 memset(&empty_block, 0, sizeof(ib_pkey_table_t));
400
401 p_node = osm_physp_get_node_ptr(p_physp);
402 p_pkey_tbl = &p_physp->pkeys;
403 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl);
404 max_num_of_blocks = pkey_mgr_get_physp_max_blocks(p_physp);
405 if (p_pkey_tbl->max_blocks > max_num_of_blocks) {
406 OSM_LOG(p_log, OSM_LOG_INFO,
407 "Max number of blocks reduced from %u to %u "
408 "for node 0x%016" PRIx64 " port %u (%s)\n",
409 p_pkey_tbl->max_blocks, max_num_of_blocks,
410 cl_ntoh64(osm_node_get_node_guid(p_node)),
411 osm_physp_get_port_num(p_physp),
412 p_physp->p_node->print_desc);
413 }
414 p_pkey_tbl->max_blocks = max_num_of_blocks;
415
416 osm_pkey_tbl_init_new_blocks(p_pkey_tbl);
417 p_pkey_tbl->used_blocks = 0;
418
419 /*
420 process every pending pkey in order -
421 first must be "updated" last are "new"
422 */
423 p_pending =
424 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
425 while (p_pending !=
426 (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) {
427
428 found = FALSE;
429 ptr = NULL;
430
431 if (p_pending->is_new == FALSE) {
432 block_index = p_pending->block;
433 pkey_index = p_pending->index;
434 found = TRUE;
435 } else {
436 ptr = cl_map_get(&p_pkey_tbl->accum_pkeys,p_pending->pkey);
437 if (ptr != NULL) {
438 pkey_idx_ptr = (uintptr_t) ptr;
439 pkey_idx = pkey_idx_ptr;
440 pkey_idx--; /* adjust pkey index for bias */
441 block_index = pkey_idx / IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
442 pkey_index = pkey_idx % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
443
444 if (((sm->p_subn->opt.allow_both_pkeys &&
445 p_pending->pkey == p_physp->pkeys.indx0_pkey) ||
446 (!sm->p_subn->opt.allow_both_pkeys &&
447 ib_pkey_get_base(p_pending->pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) ||
448 ((p_pending->pkey != p_physp->pkeys.indx0_pkey &&
449 pkey_idx == 0))) {
450 clear_accum_pkey_index(p_pkey_tbl, pkey_idx);
451 cl_qlist_insert_tail(&p_pkey_tbl->pending,
452 (cl_list_item_t *)p_pending);
453 p_pending =
454 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending);
455 continue;
456 } else
457 found = TRUE;
458 }
459
460 if (!found) {
461 if (!p_pkey_tbl->indx0_pkey &&
462 ((sm->p_subn->opt.allow_both_pkeys &&
463 p_pending->pkey == IB_DEFAULT_PKEY) ||
464 (!sm->p_subn->opt.allow_both_pkeys &&
465 ib_pkey_get_base(p_pending->pkey) == IB_DEFAULT_PARTIAL_PKEY))) {
466 block_index = 0;
467 pkey_index = 0;
468 } else if ((sm->p_subn->opt.allow_both_pkeys &&
469 p_pending->pkey == p_pkey_tbl->indx0_pkey) ||
470 (!sm->p_subn->opt.allow_both_pkeys &&
471 ib_pkey_get_base(p_pending->pkey) ==
472 ib_pkey_get_base(p_pkey_tbl->indx0_pkey))) {
473 block_index = 0;
474 pkey_index = 0;
475 } else if (last_accum_pkey_index(p_pkey_tbl,
476 &last_free_block_index,
477 &last_free_pkey_index)) {
478 block_index = last_free_block_index;
479 pkey_index = last_free_pkey_index + 1;
480 if (pkey_index >= IB_NUM_PKEY_ELEMENTS_IN_BLOCK) {
481 block_index++;
482 pkey_index -= IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
483 }
484 } else {
485 block_index = 0;
486 pkey_index = 1;
487 }
488
489 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) {
490 if ((sm->p_subn->opt.allow_both_pkeys &&
491 p_pending->pkey != IB_DEFAULT_PKEY) ||
492 (!sm->p_subn->opt.allow_both_pkeys &&
493 ib_pkey_get_base(p_pending->pkey) != IB_DEFAULT_PARTIAL_PKEY)) {
494 last_free_block_index = 0;
495 last_free_pkey_index = 1;
496 found = osm_pkey_find_next_free_entry(p_pkey_tbl, &last_free_block_index, &last_free_pkey_index);
497 } else
498 found = FALSE;
499 if (!found)
500 full = 1;
501 else {
502 block_index = last_free_block_index;
503 pkey_index = last_free_pkey_index;
504 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) {
505 full = 1;
506 found = FALSE;
507 } else {
508 OSM_LOG(p_log, OSM_LOG_INFO,
509 "Reusing PKeyTable block index %u pkey index %u "
510 "for pkey 0x%x on 0x%016" PRIx64 " port %u (%s)\n",
511 block_index,
512 pkey_index,
513 cl_ntoh16(p_pending->pkey),
514 cl_ntoh64(osm_node_get_node_guid(p_node)),
515 osm_physp_get_port_num(p_physp),
516 p_physp->p_node->print_desc);
517
518 clear_accum_pkey_index(p_pkey_tbl, block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index);
519 }
520 }
521 if (full)
522 OSM_LOG(p_log, OSM_LOG_ERROR,
523 "ERR 0512: "
524 "Failed to set PKey 0x%04x because Pkey table is full "
525 "for node 0x%016" PRIx64 " port %u (%s)\n",
526 cl_ntoh16(p_pending->pkey),
527 cl_ntoh64(osm_node_get_node_guid(p_node)),
528 osm_physp_get_port_num(p_physp),
529 p_physp->p_node->print_desc);
530 } else
531 found = TRUE;
532 }
533 }
534
535 if (found) {
536 if (IB_SUCCESS !=
537 osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index,
538 pkey_index,
539 p_pending->pkey)) {
540 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: "
541 "Failed to set PKey 0x%04x in block %u idx %u "
542 "for node 0x%016" PRIx64 " port %u (%s)\n",
543 cl_ntoh16(p_pending->pkey), block_index,
544 pkey_index,
545 cl_ntoh64(osm_node_get_node_guid
546 (p_node)),
547 osm_physp_get_port_num(p_physp),
548 p_physp->p_node->print_desc);
549 }
550 if (ptr == NULL &&
551 CL_SUCCESS !=
552 osm_pkey_tbl_set_accum_pkeys(p_pkey_tbl,
553 p_pending->pkey,
554 block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index)) {
555 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: "
556 "Failed to set accum_pkeys PKey 0x%04x "
557 "in block %u idx %u for node 0x%016"
558 PRIx64 " port %u (%s)\n",
559 cl_ntoh16(p_pending->pkey), block_index,
560 pkey_index,
561 cl_ntoh64(osm_node_get_node_guid(p_node)),
562 osm_physp_get_port_num(p_physp),
563 p_physp->p_node->print_desc);
564 }
565 }
566 free(p_pending);
567 p_pending =
568 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->
569 pending);
570 }
571
572 p_pkey_tbl->indx0_pkey = 0;
573 /* now look for changes and store */
574 for (block_index = 0; block_index < num_of_blocks; block_index++) {
575 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index);
576 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
577 if (!new_block)
578 new_block = &empty_block;
579 if (block && !memcmp(new_block, block, sizeof(*block)))
580 continue;
581
582 status =
583 pkey_mgr_update_pkey_entry(sm, p_physp, new_block,
584 block_index);
585 if (status == IB_SUCCESS)
586 OSM_LOG(p_log, OSM_LOG_DEBUG,
587 "Updated pkey table block %u for node 0x%016"
588 PRIx64 " port %u (%s)\n", block_index,
589 cl_ntoh64(osm_node_get_node_guid(p_node)),
590 osm_physp_get_port_num(p_physp),
591 p_physp->p_node->print_desc);
592 else {
593 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: "
594 "pkey_mgr_update_pkey_entry() failed to update "
595 "pkey table block %u for node 0x%016" PRIx64
596 " port %u (%s)\n", block_index,
597 cl_ntoh64(osm_node_get_node_guid(p_node)),
598 osm_physp_get_port_num(p_physp),
599 p_physp->p_node->print_desc);
600 ret = -1;
601 }
602 }
603
604 return ret;
605 }
606
last_used_pkey_index(const osm_port_t * const p_port,const osm_pkey_tbl_t * p_pkey_tbl,uint16_t * p_last_index)607 static int last_used_pkey_index(const osm_port_t * const p_port,
608 const osm_pkey_tbl_t * p_pkey_tbl,
609 uint16_t * p_last_index)
610 {
611 ib_pkey_table_t *last_block;
612 uint16_t index, last_index = 0;
613
614 CL_ASSERT(p_last_index);
615
616 last_block = osm_pkey_tbl_new_block_get(p_pkey_tbl,
617 p_pkey_tbl->used_blocks - 1);
618 if (!last_block)
619 return 1;
620
621 if (p_pkey_tbl->used_blocks == p_pkey_tbl->max_blocks)
622 last_index = cl_ntoh16(p_port->p_node->node_info.partition_cap) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
623 if (last_index == 0)
624 last_index = IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
625 index = last_index;
626 do {
627 index--;
628 if (!ib_pkey_is_invalid(last_block->pkey_entry[index]))
629 break;
630 } while (index != 0);
631
632 *p_last_index = index;
633 return 0;
634 }
635
update_peer_block(osm_log_t * p_log,osm_sm_t * sm,osm_physp_t * peer,osm_pkey_tbl_t * p_peer_pkey_tbl,ib_pkey_table_t * new_peer_block,uint16_t peer_block_idx,osm_node_t * p_node)636 static int update_peer_block(osm_log_t * p_log, osm_sm_t * sm,
637 osm_physp_t * peer,
638 osm_pkey_tbl_t * p_peer_pkey_tbl,
639 ib_pkey_table_t * new_peer_block,
640 uint16_t peer_block_idx, osm_node_t * p_node)
641 {
642 int ret = 0;
643 ib_pkey_table_t *peer_block;
644
645 peer_block = osm_pkey_tbl_block_get(p_peer_pkey_tbl, peer_block_idx);
646 if (!peer_block ||
647 memcmp(peer_block, new_peer_block, sizeof(*peer_block))) {
648 if (pkey_mgr_update_pkey_entry(sm, peer, new_peer_block,
649 peer_block_idx) != IB_SUCCESS) {
650 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: "
651 "pkey_mgr_update_pkey_entry() failed to update "
652 "pkey table block %u for node 0x%016"
653 PRIx64 " port %u (%s)\n",
654 peer_block_idx,
655 cl_ntoh64(osm_node_get_node_guid(p_node)),
656 osm_physp_get_port_num(peer),
657 p_node->print_desc);
658 ret = -1;
659 }
660 }
661
662 return ret;
663 }
664
new_pkey_exists(osm_pkey_tbl_t * p_pkey_tbl,ib_net16_t pkey)665 static int new_pkey_exists(osm_pkey_tbl_t * p_pkey_tbl, ib_net16_t pkey)
666 {
667 uint16_t num_blocks;
668 uint16_t block_index;
669 ib_pkey_table_t *block;
670 uint16_t pkey_idx;
671
672 num_blocks = (uint16_t) cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks);
673 for (block_index = 0; block_index < num_blocks; block_index++) {
674 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
675 if (!block)
676 continue;
677
678 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
679 pkey_idx++) {
680 if (block->pkey_entry[pkey_idx] == pkey)
681 return 1;
682 }
683 }
684 return 0;
685 }
686
pkey_mgr_update_peer_port(osm_log_t * p_log,osm_sm_t * sm,const osm_subn_t * p_subn,const osm_port_t * const p_port,osm_partition_enforce_type_enum enforce_type)687 static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm,
688 const osm_subn_t * p_subn,
689 const osm_port_t * const p_port,
690 osm_partition_enforce_type_enum enforce_type)
691 {
692 osm_physp_t *p_physp, *peer;
693 osm_node_t *p_node;
694 ib_pkey_table_t *block;
695 const osm_pkey_tbl_t *p_pkey_tbl;
696 osm_pkey_tbl_t *p_peer_pkey_tbl;
697 uint16_t block_index, peer_block_idx;
698 uint16_t peer_max_blocks;
699 uint16_t last_index;
700 ib_pkey_table_t new_peer_block;
701 uint16_t pkey_idx, peer_pkey_idx;
702 ib_net16_t pkey, full_pkey;
703 int ret = 0, loop_exit = 0;
704
705 p_physp = p_port->p_physp;
706 if (!p_physp)
707 return -1;
708 peer = osm_physp_get_remote(p_physp);
709 if (!peer)
710 return -1;
711 p_node = osm_physp_get_node_ptr(peer);
712 if (!p_node->sw || !p_node->sw->switch_info.enforce_cap)
713 return 0;
714
715 if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_OFF) {
716 pkey_mgr_enforce_partition(p_log, sm, peer, OSM_PARTITION_ENFORCE_TYPE_OFF);
717 return ret;
718 }
719
720 p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp);
721 peer_max_blocks = pkey_mgr_get_physp_max_blocks(peer);
722 p_peer_pkey_tbl = &peer->pkeys;
723 peer_block_idx = 0;
724 peer_pkey_idx = 0;
725 for (block_index = 0; block_index < p_pkey_tbl->used_blocks;
726 block_index++) {
727 if (loop_exit)
728 break;
729 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index);
730 if (!block)
731 continue;
732 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
733 pkey_idx++) {
734 pkey = block->pkey_entry[pkey_idx];
735 if (ib_pkey_is_invalid(pkey))
736 continue;
737 if (!ib_pkey_is_full_member(pkey)) {
738 full_pkey = pkey | IB_PKEY_TYPE_MASK;
739 if (new_pkey_exists(&p_physp->pkeys, full_pkey))
740 continue;
741 }
742 new_peer_block.pkey_entry[peer_pkey_idx] = pkey;
743 if (peer_block_idx >= peer_max_blocks) {
744 loop_exit = 1;
745 break;
746 }
747 if (++peer_pkey_idx == IB_NUM_PKEY_ELEMENTS_IN_BLOCK) {
748 if (update_peer_block(p_log, sm, peer,
749 p_peer_pkey_tbl,
750 &new_peer_block,
751 peer_block_idx, p_node))
752 ret = -1;
753 peer_pkey_idx = 0;
754 peer_block_idx++;
755 }
756 }
757 }
758
759 if (peer_block_idx < peer_max_blocks) {
760 if (peer_pkey_idx) {
761 /* Handle partial last block */
762 for (; peer_pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
763 peer_pkey_idx++)
764 new_peer_block.pkey_entry[peer_pkey_idx] = 0;
765 if (update_peer_block(p_log, sm, peer, p_peer_pkey_tbl,
766 &new_peer_block, peer_block_idx,
767 p_node))
768 ret = -1;
769 } else
770 peer_block_idx--;
771
772 p_peer_pkey_tbl->used_blocks = peer_block_idx + 1;
773 if (p_peer_pkey_tbl->used_blocks == peer_max_blocks) {
774 /* Is last used pkey index beyond switch peer port capacity ? */
775 if (!last_used_pkey_index(p_port, p_peer_pkey_tbl,
776 &last_index)) {
777 last_index += peer_block_idx * IB_NUM_PKEY_ELEMENTS_IN_BLOCK;
778 if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) <= last_index) {
779 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0507: "
780 "Not enough pkey entries (%u <= %u) on switch 0x%016"
781 PRIx64 " port %u (%s). Clearing Enforcement bit\n",
782 cl_ntoh16(p_node->sw->switch_info.enforce_cap),
783 last_index,
784 cl_ntoh64(osm_node_get_node_guid(p_node)),
785 osm_physp_get_port_num(peer),
786 p_node->print_desc);
787 enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF;
788 ret = -1;
789 }
790 }
791 }
792 } else {
793 p_peer_pkey_tbl->used_blocks = peer_max_blocks;
794 enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF;
795 }
796
797 if (!ret)
798 OSM_LOG(p_log, OSM_LOG_DEBUG,
799 "Pkey table was successfully updated for node 0x%016"
800 PRIx64 " port %u (%s)\n",
801 cl_ntoh64(osm_node_get_node_guid(p_node)),
802 osm_physp_get_port_num(peer), p_node->print_desc);
803
804 if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce_type))
805 ret = -1;
806
807 return ret;
808 }
809
osm_pkey_mgr_process(IN osm_opensm_t * p_osm)810 int osm_pkey_mgr_process(IN osm_opensm_t * p_osm)
811 {
812 cl_qmap_t *p_tbl;
813 cl_map_item_t *p_next;
814 osm_prtn_t *p_prtn;
815 osm_port_t *p_port;
816 osm_switch_t *p_sw;
817 osm_physp_t *p_physp;
818 osm_node_t *p_remote_node;
819 uint8_t i;
820 int ret = 0;
821
822 CL_ASSERT(p_osm);
823
824 OSM_LOG_ENTER(&p_osm->log);
825
826 CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock);
827
828 if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) {
829 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: "
830 "osm_prtn_make_partitions() failed\n");
831 ret = -1;
832 goto _err;
833 }
834
835 /* populate the pending pkey entries by scanning all partitions */
836 p_tbl = &p_osm->subn.prtn_pkey_tbl;
837 p_next = cl_qmap_head(p_tbl);
838 while (p_next != cl_qmap_end(p_tbl)) {
839 p_prtn = (osm_prtn_t *) p_next;
840 p_next = cl_qmap_next(p_next);
841 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
842 p_prtn, FALSE);
843 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm,
844 p_prtn, TRUE);
845 }
846
847 /* calculate and set new pkey tables */
848 p_tbl = &p_osm->subn.port_guid_tbl;
849 p_next = cl_qmap_head(p_tbl);
850 while (p_next != cl_qmap_end(p_tbl)) {
851 p_port = (osm_port_t *) p_next;
852 p_next = cl_qmap_next(p_next);
853 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port))
854 ret = -1;
855 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH)
856 && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm,
857 &p_osm->subn, p_port,
858 p_osm->subn.opt.part_enforce_enum))
859 ret = -1;
860 }
861
862 /* clear partition enforcement on inter-switch links */
863 p_tbl = &p_osm->subn.sw_guid_tbl;
864 p_next = cl_qmap_head(p_tbl);
865 while (p_next != cl_qmap_end(p_tbl)) {
866 p_sw = (osm_switch_t *) p_next;
867 p_next = cl_qmap_next(p_next);
868 for (i = 1; i < p_sw->num_ports; i++) {
869 p_physp = osm_node_get_physp_ptr(p_sw->p_node, i);
870 if (p_physp && p_physp->p_remote_physp)
871 p_remote_node = p_physp->p_remote_physp->p_node;
872 else
873 continue;
874
875 if (osm_node_get_type(p_remote_node) != IB_NODE_TYPE_SWITCH)
876 continue;
877
878 if(! (p_physp->port_info.vl_enforce & 0xc ))
879 continue;
880
881 /* clear partition enforcement */
882 if (pkey_mgr_enforce_partition(&p_osm->log, &p_osm->sm, p_physp, OSM_PARTITION_ENFORCE_TYPE_OFF))
883 ret = -1;
884 }
885 }
886 _err:
887 CL_PLOCK_RELEASE(&p_osm->lock);
888 OSM_LOG_EXIT(&p_osm->log);
889 return ret;
890 }
891