xref: /freebsd/sys/dev/liquidio/base/lio_device.c (revision 1323ec571215a77ddd21294f0871979d5ad6b992)
1 /*
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Cavium, Inc. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 /*$FreeBSD$*/
34 
35 #include "lio_bsd.h"
36 #include "lio_common.h"
37 #include "lio_droq.h"
38 #include "lio_iq.h"
39 #include "lio_response_manager.h"
40 #include "lio_device.h"
41 #include "lio_main.h"
42 #include "lio_network.h"
43 #include "cn23xx_pf_device.h"
44 #include "lio_image.h"
45 #include "lio_mem_ops.h"
46 
47 static struct lio_config default_cn23xx_conf = {
48 	.card_type			= LIO_23XX,
49 	.card_name			= LIO_23XX_NAME,
50 	/* IQ attributes */
51 	.iq = {
52 		.max_iqs		= LIO_CN23XX_CFG_IO_QUEUES,
53 		.pending_list_size	= (LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS *
54 					   LIO_CN23XX_CFG_IO_QUEUES),
55 		.instr_type		= LIO_64BYTE_INSTR,
56 		.db_min			= LIO_CN23XX_DB_MIN,
57 		.db_timeout		= LIO_CN23XX_DB_TIMEOUT,
58 		.iq_intr_pkt		= LIO_CN23XX_DEF_IQ_INTR_THRESHOLD,
59 	},
60 
61 	/* OQ attributes */
62 	.oq = {
63 		.max_oqs		= LIO_CN23XX_CFG_IO_QUEUES,
64 		.pkts_per_intr		= LIO_CN23XX_OQ_PKTS_PER_INTR,
65 		.refill_threshold	= LIO_CN23XX_OQ_REFIL_THRESHOLD,
66 		.oq_intr_pkt		= LIO_CN23XX_OQ_INTR_PKT,
67 		.oq_intr_time		= LIO_CN23XX_OQ_INTR_TIME,
68 	},
69 
70 	.num_nic_ports			= LIO_CN23XX_DEFAULT_NUM_PORTS,
71 	.num_def_rx_descs		= LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
72 	.num_def_tx_descs		= LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
73 	.def_rx_buf_size		= LIO_CN23XX_OQ_BUF_SIZE,
74 
75 	/* For ethernet interface 0:  Port cfg Attributes */
76 	.nic_if_cfg[0] = {
77 		/* Max Txqs: Half for each of the two ports :max_iq/2 */
78 		.max_txqs		= LIO_MAX_TXQS_PER_INTF,
79 
80 		/* Actual configured value. Range could be: 1...max_txqs */
81 		.num_txqs		= LIO_DEF_TXQS_PER_INTF,
82 
83 		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
84 		.max_rxqs		= LIO_MAX_RXQS_PER_INTF,
85 
86 		/* Actual configured value. Range could be: 1...max_rxqs */
87 		.num_rxqs		= LIO_DEF_RXQS_PER_INTF,
88 
89 		/* Num of desc for rx rings */
90 		.num_rx_descs		= LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
91 
92 		/* Num of desc for tx rings */
93 		.num_tx_descs		= LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
94 
95 		/*
96 		 * Mbuf size, We need not change buf size even for Jumbo frames.
97 		 * Octeon can send jumbo frames in 4 consecutive descriptors,
98 		 */
99 		.rx_buf_size			= LIO_CN23XX_OQ_BUF_SIZE,
100 
101 		.base_queue			= LIO_BASE_QUEUE_NOT_REQUESTED,
102 
103 		.gmx_port_id			= 0,
104 	},
105 
106 	.nic_if_cfg[1] = {
107 		/* Max Txqs: Half for each of the two ports :max_iq/2 */
108 		.max_txqs		= LIO_MAX_TXQS_PER_INTF,
109 
110 		/* Actual configured value. Range could be: 1...max_txqs */
111 		.num_txqs		= LIO_DEF_TXQS_PER_INTF,
112 
113 		/* Max Rxqs: Half for each of the two ports :max_oq/2  */
114 		.max_rxqs		= LIO_MAX_RXQS_PER_INTF,
115 
116 		/* Actual configured value. Range could be: 1...max_rxqs */
117 		.num_rxqs		= LIO_DEF_RXQS_PER_INTF,
118 
119 		/* Num of desc for rx rings */
120 		.num_rx_descs		= LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS,
121 
122 		/* Num of desc for tx rings */
123 		.num_tx_descs		= LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS,
124 
125 		/*
126 		 * Mbuf size, We need not change buf size even for Jumbo frames.
127 		 * Octeon can send jumbo frames in 4 consecutive descriptors,
128 		 */
129 		.rx_buf_size			= LIO_CN23XX_OQ_BUF_SIZE,
130 
131 		.base_queue			= LIO_BASE_QUEUE_NOT_REQUESTED,
132 
133 		.gmx_port_id			= 1,
134 	},
135 
136 	.misc					= {
137 		/* Host driver link query interval */
138 		.oct_link_query_interval	= 100,
139 
140 		/* Octeon link query interval */
141 		.host_link_query_interval	= 500,
142 
143 		.enable_sli_oq_bp		= 0,
144 
145 		/* Control queue group */
146 		.ctrlq_grp			= 1,
147 	}
148 };
149 
150 static struct lio_config_ptr {
151 	uint32_t	conf_type;
152 }	oct_conf_info[LIO_MAX_DEVICES] = {
153 
154 	{
155 		LIO_CFG_TYPE_DEFAULT,
156 	}, {
157 		LIO_CFG_TYPE_DEFAULT,
158 	}, {
159 		LIO_CFG_TYPE_DEFAULT,
160 	}, {
161 		LIO_CFG_TYPE_DEFAULT,
162 	},
163 };
164 
165 static char lio_state_str[LIO_DEV_STATES + 1][32] = {
166 	"BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE",
167 	"IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE",
168 	"DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE",
169 	"INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE",
170 	"HOST-READY", "CORE-READY", "RUNNING", "IN-RESET",
171 	"INVALID"
172 };
173 
174 static char	lio_app_str[LIO_DRV_APP_COUNT + 1][32] = {"BASE", "NIC", "UNKNOWN"};
175 
176 static struct octeon_device	*octeon_device[LIO_MAX_DEVICES];
177 static volatile int		lio_adapter_refcounts[LIO_MAX_DEVICES];
178 
179 static uint32_t	octeon_device_count;
180 /* locks device array (i.e. octeon_device[]) */
181 struct mtx	octeon_devices_lock;
182 
183 static struct lio_core_setup	core_setup[LIO_MAX_DEVICES];
184 
185 static void
186 oct_set_config_info(int oct_id, int conf_type)
187 {
188 
189 	if (conf_type < 0 || conf_type > (LIO_NUM_CFGS - 1))
190 		conf_type = LIO_CFG_TYPE_DEFAULT;
191 	oct_conf_info[oct_id].conf_type = conf_type;
192 }
193 
194 void
195 lio_init_device_list(int conf_type)
196 {
197 	int	i;
198 
199 	bzero(octeon_device, (sizeof(void *) * LIO_MAX_DEVICES));
200 	for (i = 0; i < LIO_MAX_DEVICES; i++)
201 		oct_set_config_info(i, conf_type);
202 	mtx_init(&octeon_devices_lock, "octeon_devices_lock", NULL, MTX_DEF);
203 }
204 
205 static void *
206 __lio_retrieve_config_info(struct octeon_device *oct, uint16_t card_type)
207 {
208 	void		*ret = NULL;
209 	uint32_t	oct_id = oct->octeon_id;
210 
211 	switch (oct_conf_info[oct_id].conf_type) {
212 	case LIO_CFG_TYPE_DEFAULT:
213 		if (oct->chip_id == LIO_CN23XX_PF_VID) {
214 			ret = &default_cn23xx_conf;
215 		}
216 
217 		break;
218 	default:
219 		break;
220 	}
221 	return (ret);
222 }
223 
224 void   *
225 lio_get_config_info(struct octeon_device *oct, uint16_t card_type)
226 {
227 	void	*conf = NULL;
228 
229 	conf = __lio_retrieve_config_info(oct, card_type);
230 	if (conf == NULL)
231 		return (NULL);
232 
233 	return (conf);
234 }
235 
236 char   *
237 lio_get_state_string(volatile int *state_ptr)
238 {
239 	int32_t	istate = (int32_t)atomic_load_acq_int(state_ptr);
240 
241 	if (istate > LIO_DEV_STATES || istate < 0)
242 		return (lio_state_str[LIO_DEV_STATE_INVALID]);
243 
244 	return (lio_state_str[istate]);
245 }
246 
247 static char *
248 lio_get_app_string(uint32_t app_mode)
249 {
250 
251 	if (app_mode <= LIO_DRV_APP_END)
252 		return (lio_app_str[app_mode - LIO_DRV_APP_START]);
253 
254 	return (lio_app_str[LIO_DRV_INVALID_APP - LIO_DRV_APP_START]);
255 }
256 
257 void
258 lio_free_device_mem(struct octeon_device *oct)
259 {
260 	int	i;
261 
262 	for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) {
263 		if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i]))
264 			free(oct->droq[i], M_DEVBUF);
265 	}
266 
267 	for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) {
268 		if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i]))
269 			free(oct->instr_queue[i], M_DEVBUF);
270 	}
271 
272 	i = oct->octeon_id;
273 	free(oct->chip, M_DEVBUF);
274 
275 	octeon_device[i] = NULL;
276 	octeon_device_count--;
277 }
278 
279 static struct octeon_device *
280 lio_allocate_device_mem(device_t device)
281 {
282 	struct octeon_device	*oct;
283 	uint32_t	configsize = 0, pci_id = 0, size;
284 	uint8_t		*buf = NULL;
285 
286 	pci_id = pci_get_device(device);
287 	switch (pci_id) {
288 	case LIO_CN23XX_PF_VID:
289 		configsize = sizeof(struct lio_cn23xx_pf);
290 		break;
291 	default:
292 		device_printf(device, "Error: Unknown PCI Device: 0x%x\n",
293 			      pci_id);
294 		return (NULL);
295 	}
296 
297 	if (configsize & 0x7)
298 		configsize += (8 - (configsize & 0x7));
299 
300 	size = configsize +
301 		(sizeof(struct lio_dispatch) * LIO_DISPATCH_LIST_SIZE);
302 
303 	buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
304 	if (buf == NULL)
305 		return (NULL);
306 
307 	oct = (struct octeon_device *)device_get_softc(device);
308 	oct->chip = (void *)(buf);
309 	oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize);
310 
311 	return (oct);
312 }
313 
314 struct octeon_device *
315 lio_allocate_device(device_t device)
316 {
317 	struct octeon_device	*oct = NULL;
318 	uint32_t	oct_idx = 0;
319 
320 	mtx_lock(&octeon_devices_lock);
321 
322 	for (oct_idx = 0; oct_idx < LIO_MAX_DEVICES; oct_idx++)
323 		if (!octeon_device[oct_idx])
324 			break;
325 
326 	if (oct_idx < LIO_MAX_DEVICES) {
327 		oct = lio_allocate_device_mem(device);
328 		if (oct != NULL) {
329 			octeon_device_count++;
330 			octeon_device[oct_idx] = oct;
331 		}
332 	}
333 
334 	mtx_unlock(&octeon_devices_lock);
335 
336 	if (oct == NULL)
337 		return (NULL);
338 
339 	mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF);
340 	mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF);
341 
342 	oct->octeon_id = oct_idx;
343 	snprintf(oct->device_name, sizeof(oct->device_name), "%s%d",
344 		 LIO_DRV_NAME, oct->octeon_id);
345 
346 	return (oct);
347 }
348 
349 /*
350  *  Register a device's bus location at initialization time.
351  *  @param oct        - pointer to the octeon device structure.
352  *  @param bus        - PCIe bus #
353  *  @param dev        - PCIe device #
354  *  @param func       - PCIe function #
355  *  @param is_pf      - TRUE for PF, FALSE for VF
356  *  @return reference count of device's adapter
357  */
358 int
359 lio_register_device(struct octeon_device *oct, int bus, int dev, int func,
360 		    int is_pf)
361 {
362 	int	idx, refcount;
363 
364 	oct->loc.bus = bus;
365 	oct->loc.dev = dev;
366 	oct->loc.func = func;
367 
368 	oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id];
369 	atomic_store_rel_int(oct->adapter_refcount, 0);
370 
371 	mtx_lock(&octeon_devices_lock);
372 	for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) {
373 		if (octeon_device[idx] == NULL) {
374 			lio_dev_err(oct, "%s: Internal driver error, missing dev\n",
375 				    __func__);
376 			mtx_unlock(&octeon_devices_lock);
377 			atomic_add_int(oct->adapter_refcount, 1);
378 			return (1);	/* here, refcount is guaranteed to be 1 */
379 		}
380 
381 		/* if another device is at same bus/dev, use its refcounter */
382 		if ((octeon_device[idx]->loc.bus == bus) &&
383 		    (octeon_device[idx]->loc.dev == dev)) {
384 			oct->adapter_refcount =
385 				octeon_device[idx]->adapter_refcount;
386 			break;
387 		}
388 	}
389 
390 	mtx_unlock(&octeon_devices_lock);
391 
392 	atomic_add_int(oct->adapter_refcount, 1);
393 	refcount = atomic_load_acq_int(oct->adapter_refcount);
394 
395 	lio_dev_dbg(oct, "%s: %02x:%02x:%d refcount %u\n", __func__,
396 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);
397 
398 	return (refcount);
399 }
400 
401 /*
402  *  Deregister a device at de-initialization time.
403  *  @param oct - pointer to the octeon device structure.
404  *  @return reference count of device's adapter
405  */
406 int
407 lio_deregister_device(struct octeon_device *oct)
408 {
409 	int	refcount;
410 
411 	atomic_subtract_int(oct->adapter_refcount, 1);
412 	refcount = atomic_load_acq_int(oct->adapter_refcount);
413 
414 	lio_dev_dbg(oct, "%s: %04d:%02d:%d refcount %u\n", __func__,
415 		    oct->loc.bus, oct->loc.dev, oct->loc.func, refcount);
416 
417 	return (refcount);
418 }
419 
420 int
421 lio_allocate_ioq_vector(struct octeon_device *oct)
422 {
423 	struct lio_ioq_vector	*ioq_vector;
424 	int	i, cpu_num, num_ioqs = 0, size;
425 
426 	if (LIO_CN23XX_PF(oct))
427 		num_ioqs = oct->sriov_info.num_pf_rings;
428 
429 	size = sizeof(struct lio_ioq_vector) * num_ioqs;
430 
431 	oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
432 	if (oct->ioq_vector == NULL)
433 		return (1);
434 
435 	for (i = 0; i < num_ioqs; i++) {
436 		ioq_vector = &oct->ioq_vector[i];
437 		ioq_vector->oct_dev = oct;
438 		ioq_vector->droq_index = i;
439 		cpu_num = i % mp_ncpus;
440 		CPU_SETOF(cpu_num, &ioq_vector->affinity_mask);
441 
442 		if (oct->chip_id == LIO_CN23XX_PF_VID)
443 			ioq_vector->ioq_num = i + oct->sriov_info.pf_srn;
444 		else
445 			ioq_vector->ioq_num = i;
446 	}
447 	return (0);
448 }
449 
450 void
451 lio_free_ioq_vector(struct octeon_device *oct)
452 {
453 
454 	free(oct->ioq_vector, M_DEVBUF);
455 	oct->ioq_vector = NULL;
456 }
457 
458 /* this function is only for setting up the first queue */
459 int
460 lio_setup_instr_queue0(struct octeon_device *oct)
461 {
462 	union octeon_txpciq	txpciq;
463 	uint32_t	iq_no = 0;
464 	uint32_t	num_descs = 0;
465 
466 	if (LIO_CN23XX_PF(oct))
467 		num_descs =
468 			LIO_GET_NUM_DEF_TX_DESCS_CFG(LIO_CHIP_CONF(oct,
469 								   cn23xx_pf));
470 
471 	oct->num_iqs = 0;
472 
473 	oct->instr_queue[0]->q_index = 0;
474 	oct->instr_queue[0]->app_ctx = (void *)(size_t)0;
475 	oct->instr_queue[0]->ifidx = 0;
476 	txpciq.txpciq64 = 0;
477 	txpciq.s.q_no = iq_no;
478 	txpciq.s.pkind = oct->pfvf_hsword.pkind;
479 	txpciq.s.use_qpg = 0;
480 	txpciq.s.qpg = 0;
481 	if (lio_init_instr_queue(oct, txpciq, num_descs)) {
482 		/* prevent memory leak */
483 		lio_delete_instr_queue(oct, 0);
484 		return (1);
485 	}
486 
487 	oct->num_iqs++;
488 	return (0);
489 }
490 
491 int
492 lio_setup_output_queue0(struct octeon_device *oct)
493 {
494 	uint32_t	desc_size = 0, num_descs = 0, oq_no = 0;
495 
496 	if (LIO_CN23XX_PF(oct)) {
497 		num_descs =
498 			LIO_GET_NUM_DEF_RX_DESCS_CFG(LIO_CHIP_CONF(oct,
499 								   cn23xx_pf));
500 		desc_size =
501 			LIO_GET_DEF_RX_BUF_SIZE_CFG(LIO_CHIP_CONF(oct,
502 								  cn23xx_pf));
503 	}
504 
505 	oct->num_oqs = 0;
506 
507 	if (lio_init_droq(oct, oq_no, num_descs, desc_size, NULL)) {
508 		return (1);
509 	}
510 
511 	oct->num_oqs++;
512 
513 	return (0);
514 }
515 
516 int
517 lio_init_dispatch_list(struct octeon_device *oct)
518 {
519 	uint32_t	i;
520 
521 	oct->dispatch.count = 0;
522 
523 	for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) {
524 		oct->dispatch.dlist[i].opcode = 0;
525 		STAILQ_INIT(&oct->dispatch.dlist[i].head);
526 	}
527 
528 	mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF);
529 
530 	return (0);
531 }
532 
533 void
534 lio_delete_dispatch_list(struct octeon_device *oct)
535 {
536 	struct lio_stailq_head	freelist;
537 	struct lio_stailq_node	*temp, *tmp2;
538 	uint32_t		i;
539 
540 	STAILQ_INIT(&freelist);
541 
542 	mtx_lock(&oct->dispatch.lock);
543 
544 	for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) {
545 		struct lio_stailq_head *dispatch;
546 
547 		dispatch = &oct->dispatch.dlist[i].head;
548 		while (!STAILQ_EMPTY(dispatch)) {
549 			temp = STAILQ_FIRST(dispatch);
550 			STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head,
551 					   entries);
552 			STAILQ_INSERT_TAIL(&freelist, temp, entries);
553 		}
554 
555 		oct->dispatch.dlist[i].opcode = 0;
556 	}
557 
558 	oct->dispatch.count = 0;
559 
560 	mtx_unlock(&oct->dispatch.lock);
561 
562 	STAILQ_FOREACH_SAFE(temp, &freelist, entries, tmp2) {
563 		STAILQ_REMOVE_HEAD(&freelist, entries);
564 		free(temp, M_DEVBUF);
565 	}
566 }
567 
568 lio_dispatch_fn_t
569 lio_get_dispatch(struct octeon_device *octeon_dev, uint16_t opcode,
570 		 uint16_t subcode)
571 {
572 	struct lio_stailq_node	*dispatch;
573 	lio_dispatch_fn_t	fn = NULL;
574 	uint32_t		idx;
575 	uint16_t	combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
576 
577 	idx = combined_opcode & LIO_OPCODE_MASK;
578 
579 	mtx_lock(&octeon_dev->dispatch.lock);
580 
581 	if (octeon_dev->dispatch.count == 0) {
582 		mtx_unlock(&octeon_dev->dispatch.lock);
583 		return (NULL);
584 	}
585 
586 	if (!(octeon_dev->dispatch.dlist[idx].opcode)) {
587 		mtx_unlock(&octeon_dev->dispatch.lock);
588 		return (NULL);
589 	}
590 
591 	if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
592 		fn = octeon_dev->dispatch.dlist[idx].dispatch_fn;
593 	} else {
594 		STAILQ_FOREACH(dispatch, &octeon_dev->dispatch.dlist[idx].head,
595 			       entries) {
596 			if (((struct lio_dispatch *)dispatch)->opcode ==
597 			    combined_opcode) {
598 				fn = ((struct lio_dispatch *)
599 				      dispatch)->dispatch_fn;
600 				break;
601 			}
602 		}
603 	}
604 
605 	mtx_unlock(&octeon_dev->dispatch.lock);
606 	return (fn);
607 }
608 
609 /*
610  * lio_register_dispatch_fn
611  * Parameters:
612  *   octeon_id - id of the octeon device.
613  *   opcode    - opcode for which driver should call the registered function
614  *   subcode   - subcode for which driver should call the registered function
615  *   fn        - The function to call when a packet with "opcode" arrives in
616  *               octeon output queues.
617  *   fn_arg    - The argument to be passed when calling function "fn".
618  * Description:
619  *   Registers a function and its argument to be called when a packet
620  *   arrives in Octeon output queues with "opcode".
621  * Returns:
622  *   Success: 0
623  *   Failure: 1
624  * Locks:
625  *   No locks are held.
626  */
627 int
628 lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
629 			 uint16_t subcode, lio_dispatch_fn_t fn, void *fn_arg)
630 {
631 	lio_dispatch_fn_t	pfn;
632 	uint32_t	idx;
633 	uint16_t	combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
634 
635 	idx = combined_opcode & LIO_OPCODE_MASK;
636 
637 	mtx_lock(&oct->dispatch.lock);
638 	/* Add dispatch function to first level of lookup table */
639 	if (oct->dispatch.dlist[idx].opcode == 0) {
640 		oct->dispatch.dlist[idx].opcode = combined_opcode;
641 		oct->dispatch.dlist[idx].dispatch_fn = fn;
642 		oct->dispatch.dlist[idx].arg = fn_arg;
643 		oct->dispatch.count++;
644 		mtx_unlock(&oct->dispatch.lock);
645 		return (0);
646 	}
647 
648 	mtx_unlock(&oct->dispatch.lock);
649 
650 	/*
651 	 * Check if there was a function already registered for this
652 	 * opcode/subcode.
653 	 */
654 	pfn = lio_get_dispatch(oct, opcode, subcode);
655 	if (!pfn) {
656 		struct lio_dispatch *dispatch;
657 
658 		lio_dev_dbg(oct,
659 			    "Adding opcode to dispatch list linked list\n");
660 		dispatch = (struct lio_dispatch *)
661 			malloc(sizeof(struct lio_dispatch),
662 			       M_DEVBUF, M_NOWAIT | M_ZERO);
663 		if (dispatch == NULL) {
664 			lio_dev_err(oct,
665 				    "No memory to add dispatch function\n");
666 			return (1);
667 		}
668 
669 		dispatch->opcode = combined_opcode;
670 		dispatch->dispatch_fn = fn;
671 		dispatch->arg = fn_arg;
672 
673 		/*
674 		 * Add dispatch function to linked list of fn ptrs
675 		 * at the hashed index.
676 		 */
677 		mtx_lock(&oct->dispatch.lock);
678 		STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head,
679 				   &dispatch->node, entries);
680 		oct->dispatch.count++;
681 		mtx_unlock(&oct->dispatch.lock);
682 
683 	} else {
684 		lio_dev_err(oct, "Found previously registered dispatch fn for opcode/subcode: %x/%x\n",
685 			    opcode, subcode);
686 		return (1);
687 	}
688 
689 	return (0);
690 }
691 
692 /*
693  * lio_unregister_dispatch_fn
694  * Parameters:
695  *   oct       - octeon device
696  *   opcode    - driver should unregister the function for this opcode
697  *   subcode   - driver should unregister the function for this subcode
698  * Description:
699  *   Unregister the function set for this opcode+subcode.
700  * Returns:
701  *   Success: 0
702  *   Failure: 1
703  * Locks:
704  *   No locks are held.
705  */
706 int
707 lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode,
708 			   uint16_t subcode)
709 {
710 	struct lio_stailq_head	*dispatch_head;
711 	struct lio_stailq_node	*dispatch, *dfree = NULL, *tmp2;
712 	int		retval = 0;
713 	uint32_t	idx;
714 	uint16_t	combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
715 
716 	idx = combined_opcode & LIO_OPCODE_MASK;
717 
718 	mtx_lock(&oct->dispatch.lock);
719 
720 	if (oct->dispatch.count == 0) {
721 		mtx_unlock(&oct->dispatch.lock);
722 		lio_dev_err(oct, "No dispatch functions registered for this device\n");
723 		return (1);
724 	}
725 	if (oct->dispatch.dlist[idx].opcode == combined_opcode) {
726 		dispatch_head = &oct->dispatch.dlist[idx].head;
727 		if (!STAILQ_EMPTY(dispatch_head)) {
728 			dispatch = STAILQ_FIRST(dispatch_head);
729 			oct->dispatch.dlist[idx].opcode =
730 			    ((struct lio_dispatch *)dispatch)->opcode;
731 			oct->dispatch.dlist[idx].dispatch_fn =
732 			    ((struct lio_dispatch *)dispatch)->dispatch_fn;
733 			oct->dispatch.dlist[idx].arg =
734 			    ((struct lio_dispatch *)dispatch)->arg;
735 			STAILQ_REMOVE_HEAD(dispatch_head, entries);
736 			dfree = dispatch;
737 		} else {
738 			oct->dispatch.dlist[idx].opcode = 0;
739 			oct->dispatch.dlist[idx].dispatch_fn = NULL;
740 			oct->dispatch.dlist[idx].arg = NULL;
741 		}
742 	} else {
743 		retval = 1;
744 		STAILQ_FOREACH_SAFE(dispatch,
745 				    &oct->dispatch.dlist[idx].head,
746 				    entries, tmp2) {
747 			if (((struct lio_dispatch *)dispatch)->opcode ==
748 			    combined_opcode) {
749 				STAILQ_REMOVE(&oct->dispatch.dlist[idx].head,
750 					      dispatch,
751 					      lio_stailq_node, entries);
752 				dfree = dispatch;
753 				retval = 0;
754 			}
755 		}
756 	}
757 
758 	if (!retval)
759 		oct->dispatch.count--;
760 
761 	mtx_unlock(&oct->dispatch.lock);
762 	free(dfree, M_DEVBUF);
763 
764 	return (retval);
765 }
766 
767 int
768 lio_core_drv_init(struct lio_recv_info *recv_info, void *buf)
769 {
770 	struct octeon_device	*oct = (struct octeon_device *)buf;
771 	struct lio_recv_pkt	*recv_pkt = recv_info->recv_pkt;
772 	struct lio_core_setup	*cs = NULL;
773 	uint32_t	i;
774 	uint32_t	num_nic_ports = 0;
775 	char		app_name[16];
776 
777 	if (LIO_CN23XX_PF(oct))
778 		num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG(
779 					       LIO_CHIP_CONF(oct, cn23xx_pf));
780 
781 	if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) {
782 		lio_dev_err(oct, "Received CORE OK when device state is 0x%x\n",
783 			    atomic_load_acq_int(&oct->status));
784 		goto core_drv_init_err;
785 	}
786 
787 	strncpy(app_name,
788 		lio_get_app_string((uint32_t)
789 				   recv_pkt->rh.r_core_drv_init.app_mode),
790 		sizeof(app_name) - 1);
791 	oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
792 	if (recv_pkt->rh.r_core_drv_init.app_mode == LIO_DRV_NIC_APP) {
793 		oct->fw_info.max_nic_ports =
794 		    (uint32_t)recv_pkt->rh.r_core_drv_init.max_nic_ports;
795 		oct->fw_info.num_gmx_ports =
796 		    (uint32_t)recv_pkt->rh.r_core_drv_init.num_gmx_ports;
797 	}
798 
799 	if (oct->fw_info.max_nic_ports < num_nic_ports) {
800 		lio_dev_err(oct, "Config has more ports than firmware allows (%d > %d).\n",
801 			    num_nic_ports, oct->fw_info.max_nic_ports);
802 		goto core_drv_init_err;
803 	}
804 
805 	oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags;
806 	oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
807 	oct->pfvf_hsword.app_mode =
808 	    (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode;
809 
810 	oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind;
811 
812 	for (i = 0; i < oct->num_iqs; i++)
813 		oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind;
814 
815 	atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK);
816 
817 	cs = &core_setup[oct->octeon_id];
818 
819 	if (recv_pkt->buffer_size[0] != (sizeof(*cs) + LIO_DROQ_INFO_SIZE)) {
820 		lio_dev_dbg(oct, "Core setup bytes expected %llu found %d\n",
821 			    LIO_CAST64(sizeof(*cs) + LIO_DROQ_INFO_SIZE),
822 			    recv_pkt->buffer_size[0]);
823 	}
824 
825 	memcpy(cs, recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE,
826 	       sizeof(*cs));
827 	strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME);
828 	strncpy(oct->boardinfo.serial_number, cs->board_serial_number,
829 		LIO_SERIAL_NUM_LEN);
830 
831 	lio_swap_8B_data((uint64_t *)cs, (sizeof(*cs) >> 3));
832 
833 	oct->boardinfo.major = cs->board_rev_major;
834 	oct->boardinfo.minor = cs->board_rev_minor;
835 
836 	lio_dev_info(oct, "Running %s (%llu Hz)\n", app_name,
837 		     LIO_CAST64(cs->corefreq));
838 
839 core_drv_init_err:
840 	for (i = 0; i < recv_pkt->buffer_count; i++)
841 		lio_recv_buffer_free(recv_pkt->buffer_ptr[i]);
842 
843 	lio_free_recv_info(recv_info);
844 	return (0);
845 }
846 
847 int
848 lio_get_tx_qsize(struct octeon_device *oct, uint32_t q_no)
849 {
850 
851 	if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_INSTR_QUEUES(oct)) &&
852 	    (oct->io_qmask.iq & BIT_ULL(q_no)))
853 		return (oct->instr_queue[q_no]->max_count);
854 
855 
856 	return (-1);
857 }
858 
859 int
860 lio_get_rx_qsize(struct octeon_device *oct, uint32_t q_no)
861 {
862 
863 	if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_OUTPUT_QUEUES(oct)) &&
864 	    (oct->io_qmask.oq & BIT_ULL(q_no)))
865 		return (oct->droq[q_no]->max_count);
866 
867 	return (-1);
868 }
869 
870 /* Returns the host firmware handshake OCTEON specific configuration */
871 struct lio_config *
872 lio_get_conf(struct octeon_device *oct)
873 {
874 	struct lio_config	*default_oct_conf = NULL;
875 
876 	/*
877 	 * check the OCTEON Device model & return the corresponding octeon
878 	 * configuration
879 	 */
880 	if (LIO_CN23XX_PF(oct)) {
881 		default_oct_conf = (struct lio_config *)(
882 					       LIO_CHIP_CONF(oct, cn23xx_pf));
883 	}
884 
885 	return (default_oct_conf);
886 }
887 
888 /*
889  *  Get the octeon device pointer.
890  *  @param octeon_id  - The id for which the octeon device pointer is required.
891  *  @return Success: Octeon device pointer.
892  *  @return Failure: NULL.
893  */
894 struct octeon_device *
895 lio_get_device(uint32_t octeon_id)
896 {
897 
898 	if (octeon_id >= LIO_MAX_DEVICES)
899 		return (NULL);
900 	else
901 		return (octeon_device[octeon_id]);
902 }
903 
904 uint64_t
905 lio_pci_readq(struct octeon_device *oct, uint64_t addr)
906 {
907 	uint64_t		val64;
908 	volatile uint32_t	addrhi;
909 
910 	mtx_lock(&oct->pci_win_lock);
911 
912 	/*
913 	 * The windowed read happens when the LSB of the addr is written.
914 	 * So write MSB first
915 	 */
916 	addrhi = (addr >> 32);
917 	if (oct->chip_id == LIO_CN23XX_PF_VID)
918 		addrhi |= 0x00060000;
919 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi);
920 
921 	/* Read back to preserve ordering of writes */
922 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi);
923 
924 	lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo,
925 			addr & 0xffffffff);
926 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo);
927 
928 	val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data);
929 
930 	mtx_unlock(&oct->pci_win_lock);
931 
932 	return (val64);
933 }
934 
935 void
936 lio_pci_writeq(struct octeon_device *oct, uint64_t val, uint64_t addr)
937 {
938 
939 	mtx_lock(&oct->pci_win_lock);
940 
941 	lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr);
942 
943 	/* The write happens when the LSB is written. So write MSB first. */
944 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32);
945 	/* Read the MSB to ensure ordering of writes. */
946 	(void)lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi);
947 
948 	lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo,
949 			val & 0xffffffff);
950 
951 	mtx_unlock(&oct->pci_win_lock);
952 }
953 
954 int
955 lio_mem_access_ok(struct octeon_device *oct)
956 {
957 	uint64_t	access_okay = 0;
958 	uint64_t	lmc0_reset_ctl;
959 
960 	/* Check to make sure a DDR interface is enabled */
961 	if (LIO_CN23XX_PF(oct)) {
962 		lmc0_reset_ctl = lio_pci_readq(oct, LIO_CN23XX_LMC0_RESET_CTL);
963 		access_okay =
964 		    (lmc0_reset_ctl & LIO_CN23XX_LMC0_RESET_CTL_DDR3RST_MASK);
965 	}
966 
967 	return (access_okay ? 0 : 1);
968 }
969 
970 int
971 lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout)
972 {
973 	int		ret = 1;
974 	uint32_t	ms;
975 
976 	if (timeout == NULL)
977 		return (ret);
978 
979 	for (ms = 0; ret && ((*timeout == 0) || (ms <= *timeout)); ms += 100) {
980 		ret = lio_mem_access_ok(oct);
981 
982 		/* wait 100 ms */
983 		if (ret)
984 			lio_sleep_timeout(100);
985 	}
986 
987 	return (ret);
988 }
989 
990 /*
991  *  Get the octeon id assigned to the octeon device passed as argument.
992  *  This function is exported to other modules.
993  *  @param dev - octeon device pointer passed as a void *.
994  *  @return octeon device id
995  */
996 int
997 lio_get_device_id(void *dev)
998 {
999 	struct octeon_device	*octeon_dev = (struct octeon_device *)dev;
1000 	uint32_t		i;
1001 
1002 	for (i = 0; i < LIO_MAX_DEVICES; i++)
1003 		if (octeon_device[i] == octeon_dev)
1004 			return (octeon_dev->octeon_id);
1005 
1006 	return (-1);
1007 }
1008 
1009 void
1010 lio_enable_irq(struct lio_droq *droq, struct lio_instr_queue *iq)
1011 {
1012 	struct octeon_device *oct = NULL;
1013 	uint64_t	instr_cnt;
1014 	uint32_t	pkts_pend;
1015 
1016 	/* the whole thing needs to be atomic, ideally */
1017 	if (droq != NULL) {
1018 		oct = droq->oct_dev;
1019 		pkts_pend = atomic_load_acq_int(&droq->pkts_pending);
1020 		mtx_lock(&droq->lock);
1021 		lio_write_csr32(oct, droq->pkts_sent_reg,
1022 				droq->pkt_count - pkts_pend);
1023 		droq->pkt_count = pkts_pend;
1024 		/* this write needs to be flushed before we release the lock */
1025 		__compiler_membar();
1026 		mtx_unlock(&droq->lock);
1027 	}
1028 
1029 	if (iq != NULL) {
1030 		oct = iq->oct_dev;
1031 		mtx_lock(&iq->lock);
1032 		lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done);
1033 		iq->pkt_in_done = 0;
1034 		/* this write needs to be flushed before we release the lock */
1035 		__compiler_membar();
1036 		mtx_unlock(&iq->lock);
1037 	}
1038 
1039 	/*
1040 	 * Implementation note:
1041 	 *
1042 	 * SLI_PKT(x)_CNTS[RESEND] is written separately so that if an interrupt
1043 	 * DOES occur as a result of RESEND, the DROQ lock will NOT be held.
1044 	 *
1045 	 * Write resend. Writing RESEND in SLI_PKTX_CNTS should be enough
1046 	 * to trigger tx interrupts as well, if they are pending.
1047 	 */
1048 	if ((oct != NULL) && (LIO_CN23XX_PF(oct))) {
1049 		if (droq != NULL)
1050 			lio_write_csr64(oct, droq->pkts_sent_reg,
1051 					LIO_CN23XX_INTR_RESEND);
1052 		/* we race with firmrware here. */
1053 		/* read and write the IN_DONE_CNTS */
1054 		else if (iq != NULL) {
1055 			instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg);
1056 			lio_write_csr64(oct, iq->inst_cnt_reg,
1057 					((instr_cnt & 0xFFFFFFFF00000000ULL) |
1058 					 LIO_CN23XX_INTR_RESEND));
1059 		}
1060 	}
1061 }
1062