1 /* Broadcom NetXtreme-C/E network driver.
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2018 Broadcom Limited
32 #include <linux/io-64-nonatomic-lo-hi.h>
71 	(((txr)->tx_napi_idx << TX_OPAQUE_RING_SHIFT) |			\
72 	 ((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (bp)->tx_ring_mask))
80 				 (bp)->tx_ring_mask)
182 	(le32_to_cpu((txcmp)->sq_cons_idx) & TX_CMP_SQ_CONS_IDX_MASK)
202 	(le32_to_cpu((tscmp)->tx_ts_cmp_ts_ns_lo) |			\
203 	 ((u64)(le32_to_cpu((tscmp)->tx_ts_cmp_flags_type) &		\
207 	(((tscmp)->tx_ts_cmp_flags_type & cpu_to_le32(TX_TS_CMP_FLAGS_ERROR)) &&\
208 	 ((tscmp)->tx_ts_cmp_errors_v & cpu_to_le32(TX_TS_CMP_TS_INVALID_ERR)))
262 	((rxcmp)->rx_cmp_len_flags_type & cpu_to_le32(RX_CMP_FLAGS_RSS_VALID))
267 	(((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_RSS_HASH_TYPE) >>\
271 	(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_FLAGS_ITYPES_MASK)
274 	((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_V3_RSS_EXT_OP_LEGACY) >>\
278 	((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_V3_RSS_EXT_OP_NEW) >>\
282 	(((bp)->rss_cap & BNXT_RSS_CAP_RSS_TCAM) ?		\
292 	((rxcmp)->rx_cmp_misc_v1 & cpu_to_le32(RX_CMP_METADATA1_VALID))
295 	(le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_METADATA1_TPID_SEL)
361 	    (((rxcmp1)->rx_cmp_flags2 &	RX_CMP_L4_CS_BITS) &&		\
362 	     !((rxcmp1)->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS))
365 	    ((le32_to_cpu((rxcmp1)->rx_cmp_flags2) &			\
369 	((le32_to_cpu((rxcmpl1)->rx_cmp_cfa_code_errors_v2) &		\
373 	((le32_to_cpu((rxcmp1)->rx_cmp_cfa_code_errors_v2) &		\
390 	((le32_to_cpu((rx_agg)->rx_agg_cmp_v) &		\
435 	((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type &		\
439 	(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
444 	(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
449 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
453 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
457 	((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type &		\
461 	((rx_tpa_start)->rx_tpa_start_cmp_misc_v1 &			\
465 	(le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
500 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
504 	(!!((rx_tpa_start)->rx_tpa_start_cmp_flags2 &		\
508 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
513 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
554 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
558 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
562 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
566 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
570 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
578 	((rx_tpa_end)->rx_tpa_end_cmp_len_flags_type &			\
582 	(!!((rx_tpa_end)->rx_tpa_end_cmp_tsdelta &			\
611 	((rx_tpa_end_ext)->rx_tpa_end_cmp_errors_v2 &			\
615 	((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) &	\
620 	((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) &	\
681 	(((cpr)->cp_ring_type << BNXT_NQ_HDL_TYPE_SHIFT) | (cpr)->cp_idx)
700 /* 64-bit doorbell */
728 #define INVALID_HW_RING_ID	((u16)-1)
745 /* The RXBD length is 16-bit so we can only support page sizes < 64K */
756 /* First RX buffer page in XDP multi-buf mode
758  * +-------------------------------------------------------------------------+
759  * | XDP_PACKET_HEADROOM | bp->rx_buf_use_size              | skb_shared_info|
760  * | (bp->rx_dma_offset) |                                  |                |
761  * +-------------------------------------------------------------------------+
764 	((unsigned int)PAGE_SIZE - VLAN_ETH_HLEN - NET_IP_ALIGN -	\
767 	(BNXT_MAX_PAGE_MODE_MTU_SBUF - \
777 #define MAX_TPA_P5_MASK	(MAX_TPA_P5 - 1)
808 #define BNXT_MAX_RX_DESC_CNT		(RX_DESC_CNT * MAX_RX_PAGES - 1)
809 #define BNXT_MAX_RX_DESC_CNT_JUM_ENA	(RX_DESC_CNT * MAX_RX_PAGES_AGG_ENA - 1)
810 #define BNXT_MAX_RX_JUM_DESC_CNT	(RX_DESC_CNT * MAX_RX_AGG_PAGES - 1)
811 #define BNXT_MAX_TX_DESC_CNT		(TX_DESC_CNT * MAX_TX_PAGES - 1)
818 #define RX_RING(bp, x)	(((x) & (bp)->rx_ring_mask) >> (BNXT_PAGE_SHIFT - 4))
819 #define RX_AGG_RING(bp, x)	(((x) & (bp)->rx_agg_ring_mask) >>	\
820 				 (BNXT_PAGE_SHIFT - 4))
821 #define RX_IDX(x)	((x) & (RX_DESC_CNT - 1))
823 #define TX_RING(bp, x)	(((x) & (bp)->tx_ring_mask) >> (BNXT_PAGE_SHIFT - 4))
824 #define TX_IDX(x)	((x) & (TX_DESC_CNT - 1))
826 #define CP_RING(x)	(((x) & ~(CP_DESC_CNT - 1)) >> (BNXT_PAGE_SHIFT - 4))
827 #define CP_IDX(x)	((x) & (CP_DESC_CNT - 1))
830 	(!!((txcmp)->tx_cmp_errors_v & cpu_to_le32(TX_CMP_V)) ==	\
831 	 !((raw_cons) & bp->cp_bit))
834 	(!!((rxcmp1)->rx_cmp_cfa_code_errors_v2 & cpu_to_le32(RX_CMP_V)) ==\
835 	 !((raw_cons) & bp->cp_bit))
838 	(!!((agg)->rx_agg_cmp_v & cpu_to_le32(RX_AGG_CMP_V)) ==	\
839 	 !((raw_cons) & bp->cp_bit))
842 	(!!((nqcmp)->v & cpu_to_le32(NQ_CN_V)) == !((raw_cons) & bp->cp_bit))
845 	(le32_to_cpu((txcmp)->tx_cmp_flags_type) & CMP_TYPE)
848 	(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_CMP_TYPE)
850 #define RING_RX(bp, idx)	((idx) & (bp)->rx_ring_mask)
853 #define RING_RX_AGG(bp, idx)	((idx) & (bp)->rx_agg_ring_mask)
856 #define RING_TX(bp, idx)	((idx) & (bp)->tx_ring_mask)
861 #define RING_CMP(idx)		((idx) & bp->cp_ring_mask)
958 #define DB_EPOCH(db, idx)	(((idx) & (db)->db_epoch_mask) <<	\
959 				 ((db)->db_epoch_shift))
963 #define DB_RING_IDX(db, idx)	(((idx) & (db)->db_ring_mask) |		\
1200 	for (iter = 0, txr = (bnapi)->tx_ring[0]; txr;	\
1201 	     txr = (iter < BNXT_MAX_TXR_PER_NAPI - 1) ?	\
1202 	     (bnapi)->tx_ring[++iter] : NULL)
1242 #define INVALID_STATS_CTX_ID	-1
1494 	__le32	preemphasis;  member
1539 #define BNXT_LINK_IS_UP(bp)	((bp)->link_info.link_state == BNXT_LINK_STATE_UP)
1641 	u32			preemphasis;  member
1706 	((link_info)->support_pam4_speeds)
1760 	((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ? 0x80000000 : 0xb1000000)
1826 	/* Stat counter mask (width) */
1856 #define MAX_CTX_BYTES_MASK	(MAX_CTX_BYTES - 1)
1960 #define BNXT_CTX_INV	((u16)-1)
2134 	if (!bs_trace->wrapped &&  in bnxt_bs_trace_check_wrap()
2135 	    *bs_trace->magic_byte != BNXT_TRACE_BUF_MAGIC_BYTE)  in bnxt_bs_trace_check_wrap()
2136 		bs_trace->wrapped = 1;  in bnxt_bs_trace_check_wrap()
2137 	bs_trace->last_offset = offset;  in bnxt_bs_trace_check_wrap()
2271 #define BNXT_PF(bp)		(!((bp)->flags & BNXT_FLAG_VF))
2272 #define BNXT_VF(bp)		((bp)->flags & BNXT_FLAG_VF)
2273 #define BNXT_NPAR(bp)		((bp)->port_partition_type)
2274 #define BNXT_MH(bp)		((bp)->flags & BNXT_FLAG_MULTI_HOST)
2277 				 ((bp)->phy_flags & BNXT_PHY_FL_SHARED_PORT_CFG))
2280 				 (bp)->link_info.phy_state == BNXT_PHY_STATE_ENABLED)
2281 #define BNXT_CHIP_TYPE_NITRO_A0(bp) ((bp)->flags & BNXT_FLAG_CHIP_NITRO_A0)
2282 #define BNXT_RX_PAGE_MODE(bp)	((bp)->flags & BNXT_FLAG_RX_PAGE_MODE)
2284 				 (!((bp)->flags & BNXT_FLAG_CHIP_P5_PLUS) ||\
2285 				  (bp)->max_tpa_v2) && !is_kdump_kernel())
2286 #define BNXT_RX_JUMBO_MODE(bp)	((bp)->flags & BNXT_FLAG_JUMBO)
2289 	((bp)->chip_num == CHIP_NUM_57608)
2292 	((bp)->chip_num == CHIP_NUM_57508 ||	\
2293 	 (bp)->chip_num == CHIP_NUM_57504 ||	\
2294 	 (bp)->chip_num == CHIP_NUM_57502)
2302 	(BNXT_CHIP_NUM_57X1X((bp)->chip_num) ||	\
2303 	 BNXT_CHIP_NUM_5745X((bp)->chip_num) ||	\
2304 	 BNXT_CHIP_NUM_588XX((bp)->chip_num) ||	\
2305 	 (BNXT_CHIP_NUM_58700((bp)->chip_num) &&	\
2310 	(BNXT_CHIP_NUM_57X0X((bp)->chip_num) ||	\
2428 	(test_bit(BNXT_STATE_FW_FATAL_COND, &(bp)->state) ||	\
2429 	 pci_channel_offline((bp)->pdev))
2488 #define BNXT_NEW_RM(bp)		((bp)->fw_cap & BNXT_FW_CAP_NEW_RM)
2490 				 ((bp)->fw_cap & BNXT_FW_CAP_PTP_RTC))
2492 	(BNXT_PF(bp) && ((bp)->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3))
2496 	 ((bp)->rss_cap & BNXT_RSS_CAP_MULTI_RSS_CTX))
2499 	 ((bp)->fw_cap & BNXT_FW_CAP_VNIC_RE_FLUSH))
2501 	((bp)->fw_cap & BNXT_FW_CAP_ENABLE_RDMA_SRIOV)
2503 	((bp)->fw_cap & BNXT_FW_CAP_ROCE_VF_RESC_MGMT_SUPPORTED)
2531 #define PHY_VER_STR_LEN         (FW_VER_STR_LEN - BC_HWRM_STR_LEN)
2538 #define BNXT_FW_MAJ(bp)		((bp)->fw_ver_code >> 48)
2539 #define BNXT_FW_BLD(bp)		(((bp)->fw_ver_code >> 16) & 0xffff)
2612 	/* ensure atomic 64-bit doorbell writes on 32-bit systems. */
2621 #define BNXT_NTP_FLTR_HASH_MASK	(BNXT_NTP_FLTR_HASH_SIZE - 1)
2631 #define BNXT_L2_FLTR_HASH_MASK	(BNXT_L2_FLTR_HASH_SIZE - 1)
2681 	/* devlink interface and vf-rep structs */
2685 	struct bnxt_vf_rep	**vf_reps; /* array of vf-rep ptrs */
2686 	u16			*cfa_code_map; /* cfa_code -> vf_idx map */
2768 	u32 used = READ_ONCE(txr->tx_prod) - READ_ONCE(txr->tx_cons);  in bnxt_tx_avail()
2770 	return bp->tx_ring_size - (used & bp->tx_ring_mask);  in bnxt_tx_avail()
2777 	spin_lock(&bp->db_lock);  in bnxt_writeq()
2779 	spin_unlock(&bp->db_lock);  in bnxt_writeq()
2789 	spin_lock(&bp->db_lock);  in bnxt_writeq_relaxed()
2791 	spin_unlock(&bp->db_lock);  in bnxt_writeq_relaxed()
2801 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {  in bnxt_db_write_relaxed()
2802 		bnxt_writeq_relaxed(bp, db->db_key64 | DB_RING_IDX(db, idx),  in bnxt_db_write_relaxed()
2803 				    db->doorbell);  in bnxt_db_write_relaxed()
2805 		u32 db_val = db->db_key32 | DB_RING_IDX(db, idx);  in bnxt_db_write_relaxed()
2807 		writel_relaxed(db_val, db->doorbell);  in bnxt_db_write_relaxed()
2808 		if (bp->flags & BNXT_FLAG_DOUBLE_DB)  in bnxt_db_write_relaxed()
2809 			writel_relaxed(db_val, db->doorbell);  in bnxt_db_write_relaxed()
2817 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {  in bnxt_db_write()
2818 		bnxt_writeq(bp, db->db_key64 | DB_RING_IDX(db, idx),  in bnxt_db_write()
2819 			    db->doorbell);  in bnxt_db_write()
2821 		u32 db_val = db->db_key32 | DB_RING_IDX(db, idx);  in bnxt_db_write()
2823 		writel(db_val, db->doorbell);  in bnxt_db_write()
2824 		if (bp->flags & BNXT_FLAG_DOUBLE_DB)  in bnxt_db_write()
2825 			writel(db_val, db->doorbell);  in bnxt_db_write()
2833 	return BNXT_PF(bp) && (bp->pf.active_vfs || bp->sriov_cfg);  in bnxt_sriov_cfg()