56 void gve_parse_device_option(struct gve_priv *priv,  in gve_parse_device_option()  argument
77 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
85 			device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,  in gve_parse_device_option()
94 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
102 			device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,  in gve_parse_device_option()
111 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
119 			device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,  in gve_parse_device_option()
129 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
137 			device_printf(priv->dev, GVE_DEVICE_OPTION_TOO_BIG_FMT,  in gve_parse_device_option()
144 			priv->min_rx_desc_cnt = max(  in gve_parse_device_option()
147 			priv->min_tx_desc_cnt = max(  in gve_parse_device_option()
156 			device_printf(priv->dev, GVE_DEVICE_OPTION_ERROR_FMT,  in gve_parse_device_option()
164 			device_printf(priv->dev,  in gve_parse_device_option()
175 		device_printf(priv->dev, "Unrecognized device option 0x%hx not enabled.\n",  in gve_parse_device_option()
182 gve_process_device_options(struct gve_priv *priv,  in gve_process_device_options()  argument
200 			device_printf(priv->dev,  in gve_process_device_options()
205 		gve_parse_device_option(priv, descriptor, dev_opt,  in gve_process_device_options()
217 static int gve_adminq_execute_cmd(struct gve_priv *priv,
221 gve_adminq_destroy_tx_queue(struct gve_priv *priv, uint32_t id)  in gve_adminq_destroy_tx_queue()  argument
228 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_destroy_tx_queue()
232 gve_adminq_destroy_rx_queue(struct gve_priv *priv, uint32_t id)  in gve_adminq_destroy_rx_queue()  argument
239 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_destroy_rx_queue()
243 gve_adminq_destroy_rx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_destroy_rx_queues()  argument
249 		err = gve_adminq_destroy_rx_queue(priv, i);  in gve_adminq_destroy_rx_queues()
251 			device_printf(priv->dev, "Failed to destroy rxq %d, err: %d\n",  in gve_adminq_destroy_rx_queues()
259 	device_printf(priv->dev, "Destroyed %d rx queues\n", num_queues);  in gve_adminq_destroy_rx_queues()
264 gve_adminq_destroy_tx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_destroy_tx_queues()  argument
270 		err = gve_adminq_destroy_tx_queue(priv, i);  in gve_adminq_destroy_tx_queues()
272 			device_printf(priv->dev, "Failed to destroy txq %d, err: %d\n",  in gve_adminq_destroy_tx_queues()
280 	device_printf(priv->dev, "Destroyed %d tx queues\n", num_queues);  in gve_adminq_destroy_tx_queues()
285 gve_adminq_create_rx_queue(struct gve_priv *priv, uint32_t queue_index)  in gve_adminq_create_rx_queue()  argument
288 	struct gve_rx_ring *rx = &priv->rx[queue_index];  in gve_adminq_create_rx_queue()
298 		.rx_ring_size = htobe16(priv->rx_desc_cnt),  in gve_adminq_create_rx_queue()
302 	if (gve_is_gqi(priv)) {  in gve_adminq_create_rx_queue()
319 		    htobe16(priv->rx_desc_cnt);  in gve_adminq_create_rx_queue()
321 		    !!((if_getcapenable(priv->ifp) & IFCAP_LRO) &&  in gve_adminq_create_rx_queue()
325 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_create_rx_queue()
329 gve_adminq_create_rx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_create_rx_queues()  argument
335 		err = gve_adminq_create_rx_queue(priv, i);  in gve_adminq_create_rx_queues()
337 			device_printf(priv->dev, "Failed to create rxq %d, err: %d\n",  in gve_adminq_create_rx_queues()
344 		device_printf(priv->dev, "Created %d rx queues\n", num_queues);  in gve_adminq_create_rx_queues()
348 	gve_adminq_destroy_rx_queues(priv, i);  in gve_adminq_create_rx_queues()
353 gve_adminq_create_tx_queue(struct gve_priv *priv, uint32_t queue_index)  in gve_adminq_create_tx_queue()  argument
356 	struct gve_tx_ring *tx = &priv->tx[queue_index];  in gve_adminq_create_tx_queue()
367 		.tx_ring_size = htobe16(priv->tx_desc_cnt),  in gve_adminq_create_tx_queue()
370 	if (gve_is_gqi(priv)) {  in gve_adminq_create_tx_queue()
379 		    htobe16(priv->tx_desc_cnt);  in gve_adminq_create_tx_queue()
381 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_create_tx_queue()
385 gve_adminq_create_tx_queues(struct gve_priv *priv, uint32_t num_queues)  in gve_adminq_create_tx_queues()  argument
391 		err = gve_adminq_create_tx_queue(priv, i);  in gve_adminq_create_tx_queues()
393 			device_printf(priv->dev, "Failed to create txq %d, err: %d\n",  in gve_adminq_create_tx_queues()
400 		device_printf(priv->dev, "Created %d tx queues\n", num_queues);  in gve_adminq_create_tx_queues()
404 	gve_adminq_destroy_tx_queues(priv, i);  in gve_adminq_create_tx_queues()
409 gve_adminq_set_mtu(struct gve_priv *priv, uint32_t mtu) {  in gve_adminq_set_mtu()  argument
418 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_set_mtu()
422 gve_enable_supported_features(struct gve_priv *priv,  in gve_enable_supported_features()  argument
430 			device_printf(priv->dev, "MODIFY RING device option enabled.\n");  in gve_enable_supported_features()
431 		priv->modify_ringsize_enabled = true;  in gve_enable_supported_features()
432 		priv->max_rx_desc_cnt = be16toh(dev_op_modify_ring->max_ring_size.rx);  in gve_enable_supported_features()
433 		priv->max_tx_desc_cnt = be16toh(dev_op_modify_ring->max_ring_size.tx);  in gve_enable_supported_features()
439 			device_printf(priv->dev, "JUMBO FRAMES device option enabled: %u.\n",  in gve_enable_supported_features()
441 		priv->max_mtu = be16toh(dev_op_jumbo_frames->max_mtu);  in gve_enable_supported_features()
446 gve_adminq_describe_device(struct gve_priv *priv)  in gve_adminq_describe_device()  argument
460 	rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE, &desc_mem);  in gve_adminq_describe_device()
462 		device_printf(priv->dev, "Failed to alloc DMA mem for DescribeDevice.\n");  in gve_adminq_describe_device()
477 	rc = gve_adminq_execute_cmd(priv, &aq_cmd);  in gve_adminq_describe_device()
484 	priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;  in gve_adminq_describe_device()
485 	priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE;  in gve_adminq_describe_device()
487 	rc = gve_process_device_options(priv, desc,  in gve_adminq_describe_device()
499 		priv->queue_format = GVE_DQO_RDA_FORMAT;  in gve_adminq_describe_device()
503 			device_printf(priv->dev,  in gve_adminq_describe_device()
508 		priv->queue_format = GVE_DQO_QPL_FORMAT;  in gve_adminq_describe_device()
512 			device_printf(priv->dev,  in gve_adminq_describe_device()
517 		priv->queue_format = GVE_GQI_QPL_FORMAT;  in gve_adminq_describe_device()
521 			device_printf(priv->dev,  in gve_adminq_describe_device()
524 		device_printf(priv->dev, "No compatible queue formats\n");  in gve_adminq_describe_device()
529         priv->num_event_counters = be16toh(desc->counters);  in gve_adminq_describe_device()
530 	priv->default_num_queues = be16toh(desc->default_num_queues);  in gve_adminq_describe_device()
531 	priv->tx_desc_cnt = be16toh(desc->tx_queue_entries);  in gve_adminq_describe_device()
532 	priv->rx_desc_cnt = be16toh(desc->rx_queue_entries);  in gve_adminq_describe_device()
533 	priv->rx_pages_per_qpl = be16toh(desc->rx_pages_per_qpl);  in gve_adminq_describe_device()
534 	priv->max_registered_pages = be64toh(desc->max_registered_pages);  in gve_adminq_describe_device()
535 	priv->max_mtu = be16toh(desc->mtu);  in gve_adminq_describe_device()
536 	priv->default_num_queues = be16toh(desc->default_num_queues);  in gve_adminq_describe_device()
537 	priv->supported_features =  supported_features_mask;  in gve_adminq_describe_device()
540 	priv->max_rx_desc_cnt = priv->rx_desc_cnt;  in gve_adminq_describe_device()
541 	priv->max_tx_desc_cnt = priv->tx_desc_cnt;  in gve_adminq_describe_device()
543 	gve_enable_supported_features(priv, supported_features_mask,  in gve_adminq_describe_device()
547 		priv->mac[i] = desc->mac[i];  in gve_adminq_describe_device()
556 gve_adminq_register_page_list(struct gve_priv *priv,  in gve_adminq_register_page_list()  argument
567 	err = gve_dma_alloc_coherent(priv, size, PAGE_SIZE, &dma);  in gve_adminq_register_page_list()
586 	err = gve_adminq_execute_cmd(priv, &cmd);  in gve_adminq_register_page_list()
592 gve_adminq_unregister_page_list(struct gve_priv *priv, uint32_t page_list_id)  in gve_adminq_unregister_page_list()  argument
601 	return (gve_adminq_execute_cmd(priv, &cmd));  in gve_adminq_unregister_page_list()
606 gve_adminq_configure_device_resources(struct gve_priv *priv)  in gve_adminq_configure_device_resources()  argument
610 	bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,  in gve_adminq_configure_device_resources()
612 	bus_dmamap_sync(priv->counter_array_mem.tag,  in gve_adminq_configure_device_resources()
613 	    priv->counter_array_mem.map, BUS_DMASYNC_PREREAD);  in gve_adminq_configure_device_resources()
618 		.counter_array = htobe64(priv->counter_array_mem.bus_addr),  in gve_adminq_configure_device_resources()
619 		.irq_db_addr = htobe64(priv->irqs_db_mem.bus_addr),  in gve_adminq_configure_device_resources()
620 		.num_counters = htobe32(priv->num_event_counters),  in gve_adminq_configure_device_resources()
621 		.num_irq_dbs = htobe32(priv->num_queues),  in gve_adminq_configure_device_resources()
624 		.queue_format = priv->queue_format,  in gve_adminq_configure_device_resources()
627 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_configure_device_resources()
631 gve_adminq_deconfigure_device_resources(struct gve_priv *priv)  in gve_adminq_deconfigure_device_resources()  argument
636 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_deconfigure_device_resources()
640 gve_adminq_verify_driver_compatibility(struct gve_priv *priv,  in gve_adminq_verify_driver_compatibility()  argument
652 	return (gve_adminq_execute_cmd(priv, &aq_cmd));  in gve_adminq_verify_driver_compatibility()
656 gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,  in gve_adminq_get_ptype_map_dqo()  argument
665 	err = gve_dma_alloc_coherent(priv, sizeof(*ptype_map), PAGE_SIZE, &dma);  in gve_adminq_get_ptype_map_dqo()
676 	err = gve_adminq_execute_cmd(priv, &aq_cmd);  in gve_adminq_get_ptype_map_dqo()
691 gve_adminq_alloc(struct gve_priv *priv)  in gve_adminq_alloc()  argument
695 	if (gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))  in gve_adminq_alloc()
698 	if (priv->aq_mem.cpu_addr == NULL) {  in gve_adminq_alloc()
699 		rc = gve_dma_alloc_coherent(priv, ADMINQ_SIZE, ADMINQ_SIZE,  in gve_adminq_alloc()
700 		    &priv->aq_mem);  in gve_adminq_alloc()
702 			device_printf(priv->dev, "Failed to allocate admin queue mem\n");  in gve_adminq_alloc()
707 	priv->adminq = priv->aq_mem.cpu_addr;  in gve_adminq_alloc()
708 	priv->adminq_bus_addr = priv->aq_mem.bus_addr;  in gve_adminq_alloc()
710 	if (priv->adminq == NULL)  in gve_adminq_alloc()
713 	priv->adminq_mask = ADMINQ_SLOTS - 1;  in gve_adminq_alloc()
714 	priv->adminq_prod_cnt = 0;  in gve_adminq_alloc()
715 	priv->adminq_cmd_fail = 0;  in gve_adminq_alloc()
716 	priv->adminq_timeouts = 0;  in gve_adminq_alloc()
717 	priv->adminq_describe_device_cnt = 0;  in gve_adminq_alloc()
718 	priv->adminq_cfg_device_resources_cnt = 0;  in gve_adminq_alloc()
719 	priv->adminq_register_page_list_cnt = 0;  in gve_adminq_alloc()
720 	priv->adminq_unregister_page_list_cnt = 0;  in gve_adminq_alloc()
721 	priv->adminq_create_tx_queue_cnt = 0;  in gve_adminq_alloc()
722 	priv->adminq_create_rx_queue_cnt = 0;  in gve_adminq_alloc()
723 	priv->adminq_destroy_tx_queue_cnt = 0;  in gve_adminq_alloc()
724 	priv->adminq_destroy_rx_queue_cnt = 0;  in gve_adminq_alloc()
725 	priv->adminq_dcfg_device_resources_cnt = 0;  in gve_adminq_alloc()
726 	priv->adminq_set_driver_parameter_cnt = 0;  in gve_adminq_alloc()
727 	priv->adminq_get_ptype_map_cnt = 0;  in gve_adminq_alloc()
729 	gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR,  in gve_adminq_alloc()
730 	    priv->adminq_bus_addr / ADMINQ_SIZE);  in gve_adminq_alloc()
732 	gve_set_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);  in gve_adminq_alloc()
737 gve_release_adminq(struct gve_priv *priv)  in gve_release_adminq()  argument
739 	if (!gve_get_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK))  in gve_release_adminq()
742 	gve_reg_bar_write_4(priv, GVE_REG_ADMINQ_ADDR, 0);  in gve_release_adminq()
743 	while (gve_reg_bar_read_4(priv, GVE_REG_ADMINQ_ADDR)) {  in gve_release_adminq()
744 		device_printf(priv->dev, "Waiting until admin queue is released.\n");  in gve_release_adminq()
748 	gve_dma_free_coherent(&priv->aq_mem);  in gve_release_adminq()
749 	priv->aq_mem = (struct gve_dma_handle){};  in gve_release_adminq()
750 	priv->adminq = 0;  in gve_release_adminq()
751 	priv->adminq_bus_addr = 0;  in gve_release_adminq()
753 	gve_clear_state_flag(priv, GVE_STATE_FLAG_ADMINQ_OK);  in gve_release_adminq()
756 		device_printf(priv->dev, "Admin queue released\n");  in gve_release_adminq()
760 gve_adminq_parse_err(struct gve_priv *priv, uint32_t opcode, uint32_t status)  in gve_adminq_parse_err()  argument
764 		device_printf(priv->dev, "AQ command(%u): failed with status %d\n", opcode, status);  in gve_adminq_parse_err()
765 		priv->adminq_cmd_fail++;  in gve_adminq_parse_err()
772 		device_printf(priv->dev,  in gve_adminq_parse_err()
806 		device_printf(priv->dev, "AQ command(%u): unknown status code %d\n",  in gve_adminq_parse_err()
813 gve_adminq_kick_cmd(struct gve_priv *priv, uint32_t prod_cnt)  in gve_adminq_kick_cmd()  argument
815 	gve_reg_bar_write_4(priv, ADMINQ_DOORBELL, prod_cnt);  in gve_adminq_kick_cmd()
820 gve_adminq_wait_for_cmd(struct gve_priv *priv, uint32_t prod_cnt)  in gve_adminq_wait_for_cmd()  argument
825 		if (gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER) == prod_cnt)  in gve_adminq_wait_for_cmd()
838 gve_adminq_kick_and_wait(struct gve_priv *priv)  in gve_adminq_kick_and_wait()  argument
846 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_kick_and_wait()
847 	head = priv->adminq_prod_cnt;  in gve_adminq_kick_and_wait()
849 	gve_adminq_kick_cmd(priv, head);  in gve_adminq_kick_and_wait()
850 	if (!gve_adminq_wait_for_cmd(priv, head)) {  in gve_adminq_kick_and_wait()
851 		device_printf(priv->dev, "AQ commands timed out, need to reset AQ\n");  in gve_adminq_kick_and_wait()
852 		priv->adminq_timeouts++;  in gve_adminq_kick_and_wait()
856 	    priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_POSTREAD);  in gve_adminq_kick_and_wait()
859 		cmd = &priv->adminq[i & priv->adminq_mask];  in gve_adminq_kick_and_wait()
862 		err = gve_adminq_parse_err(priv, opcode, status);  in gve_adminq_kick_and_wait()
875 gve_adminq_issue_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)  in gve_adminq_issue_cmd()  argument
882 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_issue_cmd()
885 	if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {  in gve_adminq_issue_cmd()
887 		err = gve_adminq_kick_and_wait(priv);  in gve_adminq_issue_cmd()
892 		tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_issue_cmd()
893 		if ((priv->adminq_prod_cnt - tail) > priv->adminq_mask) {  in gve_adminq_issue_cmd()
902 	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];  in gve_adminq_issue_cmd()
903 	priv->adminq_prod_cnt++;  in gve_adminq_issue_cmd()
908 	    priv->aq_mem.tag, priv->aq_mem.map, BUS_DMASYNC_PREWRITE);  in gve_adminq_issue_cmd()
914 		priv->adminq_describe_device_cnt++;  in gve_adminq_issue_cmd()
918 		priv->adminq_cfg_device_resources_cnt++;  in gve_adminq_issue_cmd()
922 		priv->adminq_register_page_list_cnt++;  in gve_adminq_issue_cmd()
926 		priv->adminq_unregister_page_list_cnt++;  in gve_adminq_issue_cmd()
930 		priv->adminq_create_tx_queue_cnt++;  in gve_adminq_issue_cmd()
934 		priv->adminq_create_rx_queue_cnt++;  in gve_adminq_issue_cmd()
938 		priv->adminq_destroy_tx_queue_cnt++;  in gve_adminq_issue_cmd()
942 		priv->adminq_destroy_rx_queue_cnt++;  in gve_adminq_issue_cmd()
946 		priv->adminq_dcfg_device_resources_cnt++;  in gve_adminq_issue_cmd()
950 		priv->adminq_set_driver_parameter_cnt++;  in gve_adminq_issue_cmd()
954 		priv->adminq_verify_driver_compatibility_cnt++;  in gve_adminq_issue_cmd()
958 		priv->adminq_get_ptype_map_cnt++;  in gve_adminq_issue_cmd()
962 		device_printf(priv->dev, "Unknown AQ command opcode %d\n", opcode);  in gve_adminq_issue_cmd()
975 gve_adminq_execute_cmd(struct gve_priv *priv, struct gve_adminq_command *cmd_orig)  in gve_adminq_execute_cmd()  argument
980 	tail = gve_reg_bar_read_4(priv, ADMINQ_EVENT_COUNTER);  in gve_adminq_execute_cmd()
981 	head = priv->adminq_prod_cnt;  in gve_adminq_execute_cmd()
985 	err = gve_adminq_issue_cmd(priv, cmd_orig);  in gve_adminq_execute_cmd()
988 	return (gve_adminq_kick_and_wait(priv));  in gve_adminq_execute_cmd()