1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Texas Instruments ICSSG Ethernet driver 3 * 4 * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/ 5 * 6 */ 7 8 #ifndef __NET_TI_ICSSG_PRUETH_H 9 #define __NET_TI_ICSSG_PRUETH_H 10 11 #include <linux/bpf.h> 12 #include <linux/bpf_trace.h> 13 #include <linux/etherdevice.h> 14 #include <linux/genalloc.h> 15 #include <linux/if_vlan.h> 16 #include <linux/if_hsr.h> 17 #include <linux/interrupt.h> 18 #include <linux/kernel.h> 19 #include <linux/mfd/syscon.h> 20 #include <linux/module.h> 21 #include <linux/mutex.h> 22 #include <linux/net_tstamp.h> 23 #include <linux/of.h> 24 #include <linux/of_irq.h> 25 #include <linux/of_mdio.h> 26 #include <linux/of_net.h> 27 #include <linux/of_platform.h> 28 #include <linux/phy.h> 29 #include <linux/remoteproc/pruss.h> 30 #include <linux/pruss_driver.h> 31 #include <linux/ptp_clock_kernel.h> 32 #include <linux/remoteproc.h> 33 34 #include <linux/dma-mapping.h> 35 #include <linux/dma/ti-cppi5.h> 36 #include <linux/dma/k3-udma-glue.h> 37 38 #include <net/devlink.h> 39 #include <net/xdp.h> 40 #include <net/page_pool/helpers.h> 41 #include <net/xsk_buff_pool.h> 42 #include <net/xdp_sock_drv.h> 43 44 #include "icssg_config.h" 45 #include "icss_iep.h" 46 #include "icssg_switch_map.h" 47 48 #define PRUETH_MAX_MTU (2000 - ETH_HLEN - ETH_FCS_LEN) 49 #define PRUETH_MIN_PKT_SIZE (VLAN_ETH_ZLEN) 50 #define PRUETH_MAX_PKT_SIZE (PRUETH_MAX_MTU + ETH_HLEN + ETH_FCS_LEN) 51 52 #define ICSS_SLICE0 0 53 #define ICSS_SLICE1 1 54 55 #define ICSS_FW_PRU 0 56 #define ICSS_FW_RTU 1 57 58 #define ICSSG_MAX_RFLOWS 8 /* per slice */ 59 60 #define ICSSG_NUM_PA_STATS 32 61 #define ICSSG_NUM_MIIG_STATS 60 62 /* Number of ICSSG related stats */ 63 #define ICSSG_NUM_STATS (ICSSG_NUM_MIIG_STATS + ICSSG_NUM_PA_STATS) 64 #define ICSSG_NUM_STANDARD_STATS 31 65 #define ICSSG_NUM_ETHTOOL_STATS (ICSSG_NUM_STATS - ICSSG_NUM_STANDARD_STATS) 66 67 #define IEP_DEFAULT_CYCLE_TIME_NS 1000000 /* 1 ms */ 68 69 #define PRUETH_UNDIRECTED_PKT_DST_TAG 0 70 #define PRUETH_UNDIRECTED_PKT_TAG_INS BIT(30) 71 72 /* Firmware status codes */ 73 #define ICSS_HS_FW_READY 0x55555555 74 #define ICSS_HS_FW_DEAD 0xDEAD0000 /* lower 16 bits contain error code */ 75 76 /* Firmware command codes */ 77 #define ICSS_HS_CMD_BUSY 0x40000000 78 #define ICSS_HS_CMD_DONE 0x80000000 79 #define ICSS_HS_CMD_CANCEL 0x10000000 80 81 /* Firmware commands */ 82 #define ICSS_CMD_SPAD 0x20 83 #define ICSS_CMD_RXTX 0x10 84 #define ICSS_CMD_ADD_FDB 0x1 85 #define ICSS_CMD_DEL_FDB 0x2 86 #define ICSS_CMD_SET_RUN 0x4 87 #define ICSS_CMD_GET_FDB_SLOT 0x5 88 #define ICSS_CMD_ENABLE_VLAN 0x5 89 #define ICSS_CMD_DISABLE_VLAN 0x6 90 #define ICSS_CMD_ADD_FILTER 0x7 91 #define ICSS_CMD_ADD_MAC 0x8 92 93 /* VLAN Filtering Related MACROs */ 94 #define PRUETH_DFLT_VLAN_HSR 1 95 #define PRUETH_DFLT_VLAN_SW 1 96 #define PRUETH_DFLT_VLAN_MAC 0 97 #define MAX_VLAN_ID 256 98 99 /* In switch mode there are 3 real ports i.e. 3 mac addrs. 100 * however Linux sees only the host side port. The other 2 ports 101 * are the switch ports. 102 * In emac mode there are 2 real ports i.e. 2 mac addrs. 103 * Linux sees both the ports. 104 */ 105 enum prueth_port { 106 PRUETH_PORT_HOST = 0, /* host side port */ 107 PRUETH_PORT_MII0, /* physical port RG/SG MII 0 */ 108 PRUETH_PORT_MII1, /* physical port RG/SG MII 1 */ 109 PRUETH_PORT_INVALID, /* Invalid prueth port */ 110 }; 111 112 enum prueth_mac { 113 PRUETH_MAC0 = 0, 114 PRUETH_MAC1, 115 PRUETH_NUM_MACS, 116 PRUETH_MAC_INVALID, 117 }; 118 119 struct prueth_tx_chn { 120 struct device *dma_dev; 121 struct napi_struct napi_tx; 122 struct k3_cppi_desc_pool *desc_pool; 123 struct k3_udma_glue_tx_channel *tx_chn; 124 struct prueth_emac *emac; 125 u32 id; 126 u32 descs_num; 127 unsigned int irq; 128 char name[32]; 129 struct hrtimer tx_hrtimer; 130 unsigned long tx_pace_timeout_ns; 131 struct xsk_buff_pool *xsk_pool; 132 bool irq_disabled; 133 }; 134 135 struct prueth_rx_chn { 136 struct device *dev; 137 struct device *dma_dev; 138 struct k3_cppi_desc_pool *desc_pool; 139 struct k3_udma_glue_rx_channel *rx_chn; 140 u32 descs_num; 141 unsigned int irq[ICSSG_MAX_RFLOWS]; /* separate irq per flow */ 142 char name[32]; 143 struct page_pool *pg_pool; 144 struct xdp_rxq_info xdp_rxq; 145 struct xsk_buff_pool *xsk_pool; 146 bool irq_disabled; 147 }; 148 149 enum prueth_swdata_type { 150 PRUETH_SWDATA_INVALID = 0, 151 PRUETH_SWDATA_SKB, 152 PRUETH_SWDATA_PAGE, 153 PRUETH_SWDATA_CMD, 154 PRUETH_SWDATA_XDPF, 155 PRUETH_SWDATA_XSK, 156 }; 157 158 enum prueth_tx_buff_type { 159 PRUETH_TX_BUFF_TYPE_XDP_TX, 160 PRUETH_TX_BUFF_TYPE_XDP_NDO, 161 }; 162 163 struct prueth_swdata { 164 enum prueth_swdata_type type; 165 union prueth_data { 166 struct sk_buff *skb; 167 struct page *page; 168 u32 cmd; 169 struct xdp_frame *xdpf; 170 struct xdp_buff *xdp; 171 } data; 172 }; 173 174 /* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3) 175 * and lower three are lower priority channels or threads. 176 */ 177 #define PRUETH_MAX_TX_QUEUES 4 178 179 #define PRUETH_MAX_TX_TS_REQUESTS 50 /* Max simultaneous TX_TS requests */ 180 181 /* XDP BPF state */ 182 #define ICSSG_XDP_PASS 0 183 #define ICSSG_XDP_CONSUMED BIT(0) 184 #define ICSSG_XDP_TX BIT(1) 185 #define ICSSG_XDP_REDIR BIT(2) 186 187 /* Minimum coalesce time in usecs for both Tx and Rx */ 188 #define ICSSG_MIN_COALESCE_USECS 20 189 190 /* data for each emac port */ 191 struct prueth_emac { 192 bool is_sr1; 193 struct prueth *prueth; 194 struct net_device *ndev; 195 u8 mac_addr[6]; 196 struct napi_struct napi_rx; 197 u32 msg_enable; 198 199 int link; 200 int speed; 201 int duplex; 202 203 const char *phy_id; 204 struct device_node *phy_node; 205 phy_interface_t phy_if; 206 enum prueth_port port_id; 207 struct icss_iep *iep; 208 unsigned int rx_ts_enabled : 1; 209 unsigned int tx_ts_enabled : 1; 210 unsigned int half_duplex : 1; 211 212 /* DMA related */ 213 struct prueth_tx_chn tx_chns[PRUETH_MAX_TX_QUEUES]; 214 struct completion tdown_complete; 215 atomic_t tdown_cnt; 216 struct prueth_rx_chn rx_chns; 217 int rx_flow_id_base; 218 int tx_ch_num; 219 220 /* SR1.0 Management channel */ 221 struct prueth_rx_chn rx_mgm_chn; 222 int rx_mgm_flow_id_base; 223 224 spinlock_t lock; /* serialize access */ 225 226 /* TX HW Timestamping */ 227 /* TX TS cookie will be index to the tx_ts_skb array */ 228 struct sk_buff *tx_ts_skb[PRUETH_MAX_TX_TS_REQUESTS]; 229 atomic_t tx_ts_pending; 230 int tx_ts_irq; 231 232 u8 cmd_seq; 233 /* shutdown related */ 234 __le32 cmd_data[4]; 235 struct completion cmd_complete; 236 /* Mutex to serialize access to firmware command interface */ 237 struct mutex cmd_lock; 238 struct work_struct rx_mode_work; 239 struct workqueue_struct *cmd_wq; 240 241 struct pruss_mem_region dram; 242 243 bool offload_fwd_mark; 244 int port_vlan; 245 246 struct delayed_work stats_work; 247 u64 stats[ICSSG_NUM_MIIG_STATS]; 248 u64 pa_stats[ICSSG_NUM_PA_STATS]; 249 250 /* RX IRQ Coalescing Related */ 251 struct hrtimer rx_hrtimer; 252 unsigned long rx_pace_timeout_ns; 253 254 struct netdev_hw_addr_list vlan_mcast_list[MAX_VLAN_ID]; 255 struct bpf_prog *xdp_prog; 256 struct xdp_attachment_info xdpi; 257 int xsk_qid; 258 }; 259 260 /* The buf includes headroom compatible with both skb and xdpf */ 261 #define PRUETH_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN) 262 #define PRUETH_HEADROOM ALIGN(PRUETH_HEADROOM_NA, sizeof(long)) 263 264 /** 265 * struct prueth_pdata - PRUeth platform data 266 * @fdqring_mode: Free desc queue mode 267 * @quirk_10m_link_issue: 10M link detect errata 268 * @switch_mode: switch firmware support 269 * @banked_ms_ram: banked memory support 270 */ 271 struct prueth_pdata { 272 enum k3_ring_mode fdqring_mode; 273 u32 quirk_10m_link_issue:1; 274 u32 switch_mode:1; 275 u32 banked_ms_ram:1; 276 }; 277 278 struct icssg_firmwares { 279 const char *pru; 280 const char *rtu; 281 const char *txpru; 282 }; 283 284 /** 285 * struct prueth - PRUeth structure 286 * @dev: device 287 * @pruss: pruss handle 288 * @pru: rproc instances of PRUs 289 * @rtu: rproc instances of RTUs 290 * @txpru: rproc instances of TX_PRUs 291 * @shram: PRUSS shared RAM region 292 * @sram_pool: MSMC RAM pool for buffers 293 * @msmcram: MSMC RAM region 294 * @eth_node: DT node for the port 295 * @emac: private EMAC data structure 296 * @registered_netdevs: list of registered netdevs 297 * @miig_rt: regmap to mii_g_rt block 298 * @mii_rt: regmap to mii_rt block 299 * @pa_stats: regmap to pa_stats block 300 * @pru_id: ID for each of the PRUs 301 * @pdev: pointer to ICSSG platform device 302 * @pdata: pointer to platform data for ICSSG driver 303 * @icssg_hwcmdseq: seq counter or HWQ messages 304 * @emacs_initialized: num of EMACs/ext ports that are up/running 305 * @iep0: pointer to IEP0 device 306 * @iep1: pointer to IEP1 device 307 * @vlan_tbl: VLAN-FID table pointer 308 * @hw_bridge_dev: pointer to HW bridge net device 309 * @hsr_dev: pointer to the HSR net device 310 * @hsr_prp_version: enum to store the protocol version of hsr master 311 * @br_members: bitmask of bridge member ports 312 * @hsr_members: bitmask of hsr member ports 313 * @prueth_netdevice_nb: netdevice notifier block 314 * @prueth_switchdev_nb: switchdev notifier block 315 * @prueth_switchdev_bl_nb: switchdev blocking notifier block 316 * @is_switch_mode: flag to indicate if device is in Switch mode 317 * @is_hsr_offload_mode: flag to indicate if device is in hsr offload mode 318 * @is_switchmode_supported: indicates platform support for switch mode 319 * @switch_id: ID for mapping switch ports to bridge 320 * @default_vlan: Default VLAN for host 321 * @icssg_emac_firmwares: Firmware names for EMAC mode, indexed per MAC 322 * @icssg_switch_firmwares: Firmware names for SWITCH mode, indexed per MAC 323 * @icssg_hsr_firmwares: Firmware names for HSR mode, indexed per MAC 324 * @icssg_prp_firmwares: Firmware names for PRP mode, indexed per MAC 325 */ 326 struct prueth { 327 struct device *dev; 328 struct pruss *pruss; 329 struct rproc *pru[PRUSS_NUM_PRUS]; 330 struct rproc *rtu[PRUSS_NUM_PRUS]; 331 struct rproc *txpru[PRUSS_NUM_PRUS]; 332 struct pruss_mem_region shram; 333 struct gen_pool *sram_pool; 334 struct pruss_mem_region msmcram; 335 336 struct device_node *eth_node[PRUETH_NUM_MACS]; 337 struct prueth_emac *emac[PRUETH_NUM_MACS]; 338 struct net_device *registered_netdevs[PRUETH_NUM_MACS]; 339 struct regmap *miig_rt; 340 struct regmap *mii_rt; 341 struct regmap *pa_stats; 342 343 enum pruss_pru_id pru_id[PRUSS_NUM_PRUS]; 344 struct platform_device *pdev; 345 struct prueth_pdata pdata; 346 u8 icssg_hwcmdseq; 347 int emacs_initialized; 348 struct icss_iep *iep0; 349 struct icss_iep *iep1; 350 struct prueth_vlan_tbl *vlan_tbl; 351 352 struct net_device *hw_bridge_dev; 353 struct net_device *hsr_dev; 354 enum hsr_version hsr_prp_version; 355 u8 br_members; 356 u8 hsr_members; 357 struct notifier_block prueth_netdevice_nb; 358 struct notifier_block prueth_switchdev_nb; 359 struct notifier_block prueth_switchdev_bl_nb; 360 bool is_switch_mode; 361 bool is_hsr_offload_mode; 362 bool is_switchmode_supported; 363 unsigned char switch_id[MAX_PHYS_ITEM_ID_LEN]; 364 int default_vlan; 365 /** @vtbl_lock: Lock for vtbl in shared memory */ 366 spinlock_t vtbl_lock; 367 /** @stats_lock: Lock for reading icssg stats */ 368 spinlock_t stats_lock; 369 struct icssg_firmwares icssg_emac_firmwares[PRUETH_NUM_MACS]; 370 struct icssg_firmwares icssg_switch_firmwares[PRUETH_NUM_MACS]; 371 struct icssg_firmwares icssg_hsr_firmwares[PRUETH_NUM_MACS]; 372 struct icssg_firmwares icssg_prp_firmwares[PRUETH_NUM_MACS]; 373 }; 374 375 struct emac_tx_ts_response { 376 u32 reserved[2]; 377 u32 cookie; 378 u32 lo_ts; 379 u32 hi_ts; 380 }; 381 382 struct emac_tx_ts_response_sr1 { 383 __le32 lo_ts; 384 __le32 hi_ts; 385 __le32 reserved; 386 __le32 cookie; 387 }; 388 389 /* get PRUSS SLICE number from prueth_emac */ 390 static inline int prueth_emac_slice(struct prueth_emac *emac) 391 { 392 switch (emac->port_id) { 393 case PRUETH_PORT_MII0: 394 return ICSS_SLICE0; 395 case PRUETH_PORT_MII1: 396 return ICSS_SLICE1; 397 default: 398 return -EINVAL; 399 } 400 } 401 402 extern const struct ethtool_ops icssg_ethtool_ops; 403 extern const struct dev_pm_ops prueth_dev_pm_ops; 404 405 static inline u64 icssg_read_time(const void __iomem *addr) 406 { 407 u32 low, high; 408 409 do { 410 high = readl(addr + 4); 411 low = readl(addr); 412 } while (high != readl(addr + 4)); 413 414 return low + ((u64)high << 32); 415 } 416 417 /* Classifier helpers */ 418 void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac); 419 void icssg_class_set_host_mac_addr(struct regmap *miig_rt, const u8 *mac); 420 void icssg_class_disable(struct regmap *miig_rt, int slice); 421 void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti, 422 bool is_sr1); 423 void icssg_class_promiscuous_sr1(struct regmap *miig_rt, int slice); 424 void icssg_class_add_mcast_sr1(struct regmap *miig_rt, int slice, 425 struct net_device *ndev); 426 void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr); 427 428 /* config helpers */ 429 void icssg_config_ipg(struct prueth_emac *emac); 430 int icssg_config(struct prueth *prueth, struct prueth_emac *emac, 431 int slice); 432 int icssg_set_port_state(struct prueth_emac *emac, 433 enum icssg_port_state_cmd state); 434 void icssg_config_set_speed(struct prueth_emac *emac); 435 void icssg_config_half_duplex(struct prueth_emac *emac); 436 void icssg_init_emac_mode(struct prueth *prueth); 437 void icssg_init_fw_offload_mode(struct prueth *prueth); 438 439 /* Buffer queue helpers */ 440 int icssg_queue_pop(struct prueth *prueth, u8 queue); 441 void icssg_queue_push(struct prueth *prueth, int queue, u16 addr); 442 u32 icssg_queue_level(struct prueth *prueth, int queue); 443 444 int icssg_send_fdb_msg(struct prueth_emac *emac, struct mgmt_cmd *cmd, 445 struct mgmt_cmd_rsp *rsp); 446 int icssg_fdb_add_del(struct prueth_emac *emac, const unsigned char *addr, 447 u8 vid, u8 fid_c2, bool add); 448 int icssg_fdb_lookup(struct prueth_emac *emac, const unsigned char *addr, 449 u8 vid); 450 void icssg_vtbl_modify(struct prueth_emac *emac, u8 vid, u8 port_mask, 451 u8 untag_mask, bool add); 452 u16 icssg_get_pvid(struct prueth_emac *emac); 453 void icssg_set_pvid(struct prueth *prueth, u8 vid, u8 port); 454 int emac_fdb_flow_id_updated(struct prueth_emac *emac); 455 #define prueth_napi_to_tx_chn(pnapi) \ 456 container_of(pnapi, struct prueth_tx_chn, napi_tx) 457 458 void icssg_stats_work_handler(struct work_struct *work); 459 void emac_update_hardware_stats(struct prueth_emac *emac); 460 int emac_get_stat_by_name(struct prueth_emac *emac, char *stat_name); 461 462 /* Common functions */ 463 void prueth_cleanup_rx_chns(struct prueth_emac *emac, 464 struct prueth_rx_chn *rx_chn, 465 int max_rflows); 466 void prueth_cleanup_tx_chns(struct prueth_emac *emac); 467 void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num); 468 void prueth_xmit_free(struct prueth_tx_chn *tx_chn, 469 struct cppi5_host_desc_t *desc); 470 int emac_tx_complete_packets(struct prueth_emac *emac, int chn, 471 int budget, bool *tdown); 472 int prueth_ndev_add_tx_napi(struct prueth_emac *emac); 473 int prueth_init_tx_chns(struct prueth_emac *emac); 474 int prueth_init_rx_chns(struct prueth_emac *emac, 475 struct prueth_rx_chn *rx_chn, 476 char *name, u32 max_rflows, 477 u32 max_desc_num); 478 int prueth_dma_rx_push_mapped(struct prueth_emac *emac, 479 struct prueth_rx_chn *rx_chn, 480 struct page *page, u32 buf_len); 481 unsigned int prueth_rxbuf_total_len(unsigned int len); 482 void emac_rx_timestamp(struct prueth_emac *emac, 483 struct sk_buff *skb, u32 *psdata); 484 enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev); 485 irqreturn_t prueth_rx_irq(int irq, void *dev_id); 486 void prueth_cleanup_tx_ts(struct prueth_emac *emac); 487 int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget); 488 int prueth_prepare_rx_chan(struct prueth_emac *emac, 489 struct prueth_rx_chn *chn, 490 int buf_size); 491 void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num, 492 bool free_skb); 493 void prueth_reset_rx_chan(struct prueth_rx_chn *chn, 494 int num_flows, bool disable); 495 void icssg_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue); 496 int icssg_ndo_get_ts_config(struct net_device *ndev, 497 struct kernel_hwtstamp_config *config); 498 int icssg_ndo_set_ts_config(struct net_device *ndev, 499 struct kernel_hwtstamp_config *config, 500 struct netlink_ext_ack *extack); 501 void icssg_ndo_get_stats64(struct net_device *ndev, 502 struct rtnl_link_stats64 *stats); 503 int icssg_ndo_get_phys_port_name(struct net_device *ndev, char *name, 504 size_t len); 505 int prueth_node_port(struct device_node *eth_node); 506 int prueth_node_mac(struct device_node *eth_node); 507 void prueth_netdev_exit(struct prueth *prueth, 508 struct device_node *eth_node); 509 int prueth_get_cores(struct prueth *prueth, int slice, bool is_sr1); 510 void prueth_put_cores(struct prueth *prueth, int slice); 511 512 /* Revision specific helper */ 513 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns); 514 u32 emac_xmit_xdp_frame(struct prueth_emac *emac, 515 struct xdp_frame *xdpf, 516 unsigned int q_idx, 517 enum prueth_tx_buff_type buff_type); 518 void prueth_rx_cleanup(void *data, dma_addr_t desc_dma); 519 void prueth_tx_cleanup(void *data, dma_addr_t desc_dma); 520 int prueth_xsk_wakeup(struct net_device *ndev, u32 qid, u32 flags); 521 static inline bool prueth_xdp_is_enabled(struct prueth_emac *emac) 522 { 523 return !!READ_ONCE(emac->xdp_prog); 524 } 525 526 #endif /* __NET_TI_ICSSG_PRUETH_H */ 527