1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This driver adds support for HNS3 PMU iEP device. Related perf events are 4 * bandwidth, latency, packet rate, interrupt rate etc. 5 * 6 * Copyright (C) 2022 HiSilicon Limited 7 */ 8 #include <linux/bitfield.h> 9 #include <linux/bitmap.h> 10 #include <linux/bug.h> 11 #include <linux/cpuhotplug.h> 12 #include <linux/cpumask.h> 13 #include <linux/delay.h> 14 #include <linux/device.h> 15 #include <linux/err.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/io-64-nonatomic-hi-lo.h> 19 #include <linux/irq.h> 20 #include <linux/kernel.h> 21 #include <linux/list.h> 22 #include <linux/module.h> 23 #include <linux/pci.h> 24 #include <linux/pci-epf.h> 25 #include <linux/perf_event.h> 26 #include <linux/smp.h> 27 28 /* registers offset address */ 29 #define HNS3_PMU_REG_GLOBAL_CTRL 0x0000 30 #define HNS3_PMU_REG_CLOCK_FREQ 0x0020 31 #define HNS3_PMU_REG_BDF 0x0fe0 32 #define HNS3_PMU_REG_VERSION 0x0fe4 33 #define HNS3_PMU_REG_DEVICE_ID 0x0fe8 34 35 #define HNS3_PMU_REG_EVENT_OFFSET 0x1000 36 #define HNS3_PMU_REG_EVENT_SIZE 0x1000 37 #define HNS3_PMU_REG_EVENT_CTRL_LOW 0x00 38 #define HNS3_PMU_REG_EVENT_CTRL_HIGH 0x04 39 #define HNS3_PMU_REG_EVENT_INTR_STATUS 0x08 40 #define HNS3_PMU_REG_EVENT_INTR_MASK 0x0c 41 #define HNS3_PMU_REG_EVENT_COUNTER 0x10 42 #define HNS3_PMU_REG_EVENT_EXT_COUNTER 0x18 43 #define HNS3_PMU_REG_EVENT_QID_CTRL 0x28 44 #define HNS3_PMU_REG_EVENT_QID_PARA 0x2c 45 46 #define HNS3_PMU_FILTER_SUPPORT_GLOBAL BIT(0) 47 #define HNS3_PMU_FILTER_SUPPORT_PORT BIT(1) 48 #define HNS3_PMU_FILTER_SUPPORT_PORT_TC BIT(2) 49 #define HNS3_PMU_FILTER_SUPPORT_FUNC BIT(3) 50 #define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE BIT(4) 51 #define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR BIT(5) 52 53 #define HNS3_PMU_FILTER_ALL_TC 0xf 54 #define HNS3_PMU_FILTER_ALL_QUEUE 0xffff 55 56 #define HNS3_PMU_CTRL_SUBEVENT_S 4 57 #define HNS3_PMU_CTRL_FILTER_MODE_S 24 58 59 #define HNS3_PMU_GLOBAL_START BIT(0) 60 61 #define HNS3_PMU_EVENT_STATUS_RESET BIT(11) 62 #define HNS3_PMU_EVENT_EN BIT(12) 63 #define HNS3_PMU_EVENT_OVERFLOW_RESTART BIT(15) 64 65 #define HNS3_PMU_QID_PARA_FUNC_S 0 66 #define HNS3_PMU_QID_PARA_QUEUE_S 16 67 68 #define HNS3_PMU_QID_CTRL_REQ_ENABLE BIT(0) 69 #define HNS3_PMU_QID_CTRL_DONE BIT(1) 70 #define HNS3_PMU_QID_CTRL_MISS BIT(2) 71 72 #define HNS3_PMU_INTR_MASK_OVERFLOW BIT(1) 73 74 #define HNS3_PMU_MAX_HW_EVENTS 8 75 76 /* 77 * Each hardware event contains two registers (counter and ext_counter) for 78 * bandwidth, packet rate, latency and interrupt rate. These two registers will 79 * be triggered to run at the same when a hardware event is enabled. The meaning 80 * of counter and ext_counter of different event type are different, their 81 * meaning show as follow: 82 * 83 * +----------------+------------------+---------------+ 84 * | event type | counter | ext_counter | 85 * +----------------+------------------+---------------+ 86 * | bandwidth | byte number | cycle number | 87 * +----------------+------------------+---------------+ 88 * | packet rate | packet number | cycle number | 89 * +----------------+------------------+---------------+ 90 * | latency | cycle number | packet number | 91 * +----------------+------------------+---------------+ 92 * | interrupt rate | interrupt number | cycle number | 93 * +----------------+------------------+---------------+ 94 * 95 * The cycle number indicates increment of counter of hardware timer, the 96 * frequency of hardware timer can be read from hw_clk_freq file. 97 * 98 * Performance of each hardware event is calculated by: counter / ext_counter. 99 * 100 * Since processing of data is preferred to be done in userspace, we expose 101 * ext_counter as a separate event for userspace and use bit 16 to indicate it. 102 * For example, event 0x00001 and 0x10001 are actually one event for hardware 103 * because bit 0-15 are same. If the bit 16 of one event is 0 means to read 104 * counter register, otherwise means to read ext_counter register. 105 */ 106 /* bandwidth events */ 107 #define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM 0x00001 108 #define HNS3_PMU_EVT_BW_SSU_EGU_TIME 0x10001 109 #define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM 0x00002 110 #define HNS3_PMU_EVT_BW_SSU_RPU_TIME 0x10002 111 #define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM 0x00003 112 #define HNS3_PMU_EVT_BW_SSU_ROCE_TIME 0x10003 113 #define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM 0x00004 114 #define HNS3_PMU_EVT_BW_ROCE_SSU_TIME 0x10004 115 #define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM 0x00005 116 #define HNS3_PMU_EVT_BW_TPU_SSU_TIME 0x10005 117 #define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM 0x00006 118 #define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME 0x10006 119 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM 0x00008 120 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME 0x10008 121 #define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM 0x00009 122 #define HNS3_PMU_EVT_BW_WR_FBD_TIME 0x10009 123 #define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM 0x0000a 124 #define HNS3_PMU_EVT_BW_WR_EBD_TIME 0x1000a 125 #define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM 0x0000b 126 #define HNS3_PMU_EVT_BW_RD_FBD_TIME 0x1000b 127 #define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM 0x0000c 128 #define HNS3_PMU_EVT_BW_RD_EBD_TIME 0x1000c 129 #define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM 0x0000d 130 #define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME 0x1000d 131 #define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM 0x0000e 132 #define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME 0x1000e 133 #define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM 0x0000f 134 #define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME 0x1000f 135 #define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM 0x00010 136 #define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME 0x10010 137 138 /* packet rate events */ 139 #define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM 0x00100 140 #define HNS3_PMU_EVT_PPS_IGU_SSU_TIME 0x10100 141 #define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM 0x00101 142 #define HNS3_PMU_EVT_PPS_SSU_EGU_TIME 0x10101 143 #define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM 0x00102 144 #define HNS3_PMU_EVT_PPS_SSU_RPU_TIME 0x10102 145 #define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM 0x00103 146 #define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME 0x10103 147 #define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM 0x00104 148 #define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME 0x10104 149 #define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM 0x00105 150 #define HNS3_PMU_EVT_PPS_TPU_SSU_TIME 0x10105 151 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM 0x00106 152 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME 0x10106 153 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM 0x00107 154 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME 0x10107 155 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM 0x00108 156 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME 0x10108 157 #define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM 0x00109 158 #define HNS3_PMU_EVT_PPS_WR_FBD_TIME 0x10109 159 #define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM 0x0010a 160 #define HNS3_PMU_EVT_PPS_WR_EBD_TIME 0x1010a 161 #define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM 0x0010b 162 #define HNS3_PMU_EVT_PPS_RD_FBD_TIME 0x1010b 163 #define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM 0x0010c 164 #define HNS3_PMU_EVT_PPS_RD_EBD_TIME 0x1010c 165 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM 0x0010d 166 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME 0x1010d 167 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM 0x0010e 168 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME 0x1010e 169 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM 0x0010f 170 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME 0x1010f 171 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM 0x00110 172 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME 0x10110 173 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM 0x00111 174 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME 0x10111 175 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM 0x00112 176 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME 0x10112 177 178 /* latency events */ 179 #define HNS3_PMU_EVT_DLY_TX_PUSH_TIME 0x00202 180 #define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM 0x10202 181 #define HNS3_PMU_EVT_DLY_TX_TIME 0x00204 182 #define HNS3_PMU_EVT_DLY_TX_PACKET_NUM 0x10204 183 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME 0x00206 184 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM 0x10206 185 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME 0x00207 186 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM 0x10207 187 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME 0x00208 188 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM 0x10208 189 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME 0x00209 190 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM 0x10209 191 #define HNS3_PMU_EVT_DLY_RPU_TIME 0x0020e 192 #define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM 0x1020e 193 #define HNS3_PMU_EVT_DLY_TPU_TIME 0x0020f 194 #define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM 0x1020f 195 #define HNS3_PMU_EVT_DLY_RPE_TIME 0x00210 196 #define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM 0x10210 197 #define HNS3_PMU_EVT_DLY_TPE_TIME 0x00211 198 #define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM 0x10211 199 #define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME 0x00212 200 #define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM 0x10212 201 #define HNS3_PMU_EVT_DLY_WR_FBD_TIME 0x00213 202 #define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM 0x10213 203 #define HNS3_PMU_EVT_DLY_WR_EBD_TIME 0x00214 204 #define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM 0x10214 205 #define HNS3_PMU_EVT_DLY_RD_FBD_TIME 0x00215 206 #define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM 0x10215 207 #define HNS3_PMU_EVT_DLY_RD_EBD_TIME 0x00216 208 #define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM 0x10216 209 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME 0x00217 210 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM 0x10217 211 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME 0x00218 212 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM 0x10218 213 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME 0x00219 214 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM 0x10219 215 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME 0x0021a 216 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM 0x1021a 217 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME 0x0021c 218 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM 0x1021c 219 220 /* interrupt rate events */ 221 #define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM 0x00300 222 #define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME 0x10300 223 224 /* filter mode supported by each bandwidth event */ 225 #define HNS3_PMU_FILTER_BW_SSU_EGU 0x07 226 #define HNS3_PMU_FILTER_BW_SSU_RPU 0x1f 227 #define HNS3_PMU_FILTER_BW_SSU_ROCE 0x0f 228 #define HNS3_PMU_FILTER_BW_ROCE_SSU 0x0f 229 #define HNS3_PMU_FILTER_BW_TPU_SSU 0x1f 230 #define HNS3_PMU_FILTER_BW_RPU_RCBRX 0x11 231 #define HNS3_PMU_FILTER_BW_RCBTX_TXSCH 0x11 232 #define HNS3_PMU_FILTER_BW_WR_FBD 0x1b 233 #define HNS3_PMU_FILTER_BW_WR_EBD 0x11 234 #define HNS3_PMU_FILTER_BW_RD_FBD 0x01 235 #define HNS3_PMU_FILTER_BW_RD_EBD 0x1b 236 #define HNS3_PMU_FILTER_BW_RD_PAY_M0 0x01 237 #define HNS3_PMU_FILTER_BW_RD_PAY_M1 0x01 238 #define HNS3_PMU_FILTER_BW_WR_PAY_M0 0x01 239 #define HNS3_PMU_FILTER_BW_WR_PAY_M1 0x01 240 241 /* filter mode supported by each packet rate event */ 242 #define HNS3_PMU_FILTER_PPS_IGU_SSU 0x07 243 #define HNS3_PMU_FILTER_PPS_SSU_EGU 0x07 244 #define HNS3_PMU_FILTER_PPS_SSU_RPU 0x1f 245 #define HNS3_PMU_FILTER_PPS_SSU_ROCE 0x0f 246 #define HNS3_PMU_FILTER_PPS_ROCE_SSU 0x0f 247 #define HNS3_PMU_FILTER_PPS_TPU_SSU 0x1f 248 #define HNS3_PMU_FILTER_PPS_RPU_RCBRX 0x11 249 #define HNS3_PMU_FILTER_PPS_RCBTX_TPU 0x1f 250 #define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH 0x11 251 #define HNS3_PMU_FILTER_PPS_WR_FBD 0x1b 252 #define HNS3_PMU_FILTER_PPS_WR_EBD 0x11 253 #define HNS3_PMU_FILTER_PPS_RD_FBD 0x01 254 #define HNS3_PMU_FILTER_PPS_RD_EBD 0x1b 255 #define HNS3_PMU_FILTER_PPS_RD_PAY_M0 0x01 256 #define HNS3_PMU_FILTER_PPS_RD_PAY_M1 0x01 257 #define HNS3_PMU_FILTER_PPS_WR_PAY_M0 0x01 258 #define HNS3_PMU_FILTER_PPS_WR_PAY_M1 0x01 259 #define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE 0x01 260 #define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE 0x01 261 262 /* filter mode supported by each latency event */ 263 #define HNS3_PMU_FILTER_DLY_TX_PUSH 0x01 264 #define HNS3_PMU_FILTER_DLY_TX 0x01 265 #define HNS3_PMU_FILTER_DLY_SSU_TX_NIC 0x07 266 #define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE 0x07 267 #define HNS3_PMU_FILTER_DLY_SSU_RX_NIC 0x07 268 #define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE 0x07 269 #define HNS3_PMU_FILTER_DLY_RPU 0x11 270 #define HNS3_PMU_FILTER_DLY_TPU 0x1f 271 #define HNS3_PMU_FILTER_DLY_RPE 0x01 272 #define HNS3_PMU_FILTER_DLY_TPE 0x0b 273 #define HNS3_PMU_FILTER_DLY_TPE_PUSH 0x1b 274 #define HNS3_PMU_FILTER_DLY_WR_FBD 0x1b 275 #define HNS3_PMU_FILTER_DLY_WR_EBD 0x11 276 #define HNS3_PMU_FILTER_DLY_RD_FBD 0x01 277 #define HNS3_PMU_FILTER_DLY_RD_EBD 0x1b 278 #define HNS3_PMU_FILTER_DLY_RD_PAY_M0 0x01 279 #define HNS3_PMU_FILTER_DLY_RD_PAY_M1 0x01 280 #define HNS3_PMU_FILTER_DLY_WR_PAY_M0 0x01 281 #define HNS3_PMU_FILTER_DLY_WR_PAY_M1 0x01 282 #define HNS3_PMU_FILTER_DLY_MSIX_WRITE 0x01 283 284 /* filter mode supported by each interrupt rate event */ 285 #define HNS3_PMU_FILTER_INTR_MSIX_NIC 0x01 286 287 enum hns3_pmu_hw_filter_mode { 288 HNS3_PMU_HW_FILTER_GLOBAL, 289 HNS3_PMU_HW_FILTER_PORT, 290 HNS3_PMU_HW_FILTER_PORT_TC, 291 HNS3_PMU_HW_FILTER_FUNC, 292 HNS3_PMU_HW_FILTER_FUNC_QUEUE, 293 HNS3_PMU_HW_FILTER_FUNC_INTR, 294 }; 295 296 struct hns3_pmu_event_attr { 297 u32 event; 298 u16 filter_support; 299 }; 300 301 struct hns3_pmu { 302 struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS]; 303 struct hlist_node node; 304 struct pci_dev *pdev; 305 struct pmu pmu; 306 void __iomem *base; 307 int irq; 308 int on_cpu; 309 u32 identifier; 310 u32 hw_clk_freq; /* hardware clock frequency of PMU */ 311 /* maximum and minimum bdf allowed by PMU */ 312 u16 bdf_min; 313 u16 bdf_max; 314 }; 315 316 #define to_hns3_pmu(p) (container_of((p), struct hns3_pmu, pmu)) 317 318 #define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) 319 320 #define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff) 321 #define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07)) 322 #define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func)) 323 324 #define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end) \ 325 static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \ 326 { \ 327 return FIELD_GET(GENMASK_ULL(_end, _start), \ 328 event->attr._config); \ 329 } 330 331 HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7); 332 HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15); 333 HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16); 334 HNS3_PMU_FILTER_ATTR(port, config1, 0, 3); 335 HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7); 336 HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23); 337 HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39); 338 HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51); 339 HNS3_PMU_FILTER_ATTR(global, config1, 52, 52); 340 341 #define HNS3_BW_EVT_BYTE_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 342 HNS3_PMU_EVT_BW_##_name##_BYTE_NUM, \ 343 HNS3_PMU_FILTER_BW_##_name}) 344 #define HNS3_BW_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 345 HNS3_PMU_EVT_BW_##_name##_TIME, \ 346 HNS3_PMU_FILTER_BW_##_name}) 347 #define HNS3_PPS_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 348 HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM, \ 349 HNS3_PMU_FILTER_PPS_##_name}) 350 #define HNS3_PPS_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 351 HNS3_PMU_EVT_PPS_##_name##_TIME, \ 352 HNS3_PMU_FILTER_PPS_##_name}) 353 #define HNS3_DLY_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 354 HNS3_PMU_EVT_DLY_##_name##_TIME, \ 355 HNS3_PMU_FILTER_DLY_##_name}) 356 #define HNS3_DLY_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 357 HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM, \ 358 HNS3_PMU_FILTER_DLY_##_name}) 359 #define HNS3_INTR_EVT_INTR_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 360 HNS3_PMU_EVT_PPS_##_name##_INTR_NUM, \ 361 HNS3_PMU_FILTER_INTR_##_name}) 362 #define HNS3_INTR_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 363 HNS3_PMU_EVT_PPS_##_name##_TIME, \ 364 HNS3_PMU_FILTER_INTR_##_name}) 365 366 static ssize_t hns3_pmu_event_show(struct device *dev, 367 struct device_attribute *attr, char *buf) 368 { 369 struct hns3_pmu_event_attr *event; 370 struct dev_ext_attribute *eattr; 371 372 eattr = container_of(attr, struct dev_ext_attribute, attr); 373 event = eattr->var; 374 375 return sysfs_emit(buf, "config=0x%x\n", event->event); 376 } 377 378 static ssize_t hns3_pmu_filter_mode_show(struct device *dev, 379 struct device_attribute *attr, 380 char *buf) 381 { 382 struct hns3_pmu_event_attr *event; 383 struct dev_ext_attribute *eattr; 384 int len; 385 386 eattr = container_of(attr, struct dev_ext_attribute, attr); 387 event = eattr->var; 388 389 len = sysfs_emit_at(buf, 0, "filter mode supported: "); 390 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL) 391 len += sysfs_emit_at(buf, len, "global "); 392 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT) 393 len += sysfs_emit_at(buf, len, "port "); 394 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC) 395 len += sysfs_emit_at(buf, len, "port-tc "); 396 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC) 397 len += sysfs_emit_at(buf, len, "func "); 398 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE) 399 len += sysfs_emit_at(buf, len, "func-queue "); 400 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR) 401 len += sysfs_emit_at(buf, len, "func-intr "); 402 403 len += sysfs_emit_at(buf, len, "\n"); 404 405 return len; 406 } 407 408 #define HNS3_PMU_ATTR(_name, _func, _config) \ 409 (&((struct dev_ext_attribute[]) { \ 410 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ 411 })[0].attr.attr) 412 413 #define HNS3_PMU_FORMAT_ATTR(_name, _format) \ 414 HNS3_PMU_ATTR(_name, device_show_string, _format) 415 #define HNS3_PMU_EVENT_ATTR(_name, _event) \ 416 HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event) 417 #define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \ 418 HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event) 419 420 #define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \ 421 HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \ 422 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro)) 423 #define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \ 424 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \ 425 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro)) 426 #define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \ 427 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \ 428 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro)) 429 #define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \ 430 HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \ 431 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro)) 432 433 #define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \ 434 HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \ 435 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro)) 436 #define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \ 437 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \ 438 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro)) 439 #define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \ 440 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \ 441 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro)) 442 #define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \ 443 HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \ 444 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro)) 445 446 static u8 hns3_pmu_hw_filter_modes[] = { 447 HNS3_PMU_HW_FILTER_GLOBAL, 448 HNS3_PMU_HW_FILTER_PORT, 449 HNS3_PMU_HW_FILTER_PORT_TC, 450 HNS3_PMU_HW_FILTER_FUNC, 451 HNS3_PMU_HW_FILTER_FUNC_QUEUE, 452 HNS3_PMU_HW_FILTER_FUNC_INTR, 453 }; 454 455 #define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \ 456 ((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)]) 457 458 static ssize_t identifier_show(struct device *dev, 459 struct device_attribute *attr, char *buf) 460 { 461 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 462 463 return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier); 464 } 465 static DEVICE_ATTR_RO(identifier); 466 467 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 468 char *buf) 469 { 470 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 471 472 return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu); 473 } 474 static DEVICE_ATTR_RO(cpumask); 475 476 static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr, 477 char *buf) 478 { 479 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 480 u16 bdf = hns3_pmu->bdf_min; 481 482 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf), 483 PCI_SLOT(bdf), PCI_FUNC(bdf)); 484 } 485 static DEVICE_ATTR_RO(bdf_min); 486 487 static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr, 488 char *buf) 489 { 490 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 491 u16 bdf = hns3_pmu->bdf_max; 492 493 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf), 494 PCI_SLOT(bdf), PCI_FUNC(bdf)); 495 } 496 static DEVICE_ATTR_RO(bdf_max); 497 498 static ssize_t hw_clk_freq_show(struct device *dev, 499 struct device_attribute *attr, char *buf) 500 { 501 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 502 503 return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq); 504 } 505 static DEVICE_ATTR_RO(hw_clk_freq); 506 507 static struct attribute *hns3_pmu_events_attr[] = { 508 /* bandwidth events */ 509 HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU), 510 HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU), 511 HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE), 512 HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU), 513 HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU), 514 HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX), 515 HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH), 516 HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD), 517 HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD), 518 HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD), 519 HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD), 520 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0), 521 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1), 522 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0), 523 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1), 524 525 /* packet rate events */ 526 HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU), 527 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU), 528 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU), 529 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE), 530 HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU), 531 HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU), 532 HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX), 533 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU), 534 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH), 535 HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD), 536 HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD), 537 HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD), 538 HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD), 539 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0), 540 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1), 541 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0), 542 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1), 543 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE), 544 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE), 545 546 /* latency events */ 547 HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH), 548 HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX), 549 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC), 550 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE), 551 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC), 552 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE), 553 HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU), 554 HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU), 555 HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE), 556 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE), 557 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH), 558 HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD), 559 HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD), 560 HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD), 561 HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD), 562 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0), 563 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1), 564 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0), 565 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1), 566 HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE), 567 568 /* interrupt rate events */ 569 HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC), 570 571 NULL 572 }; 573 574 static struct attribute *hns3_pmu_filter_mode_attr[] = { 575 /* bandwidth events */ 576 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU), 577 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU), 578 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE), 579 HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU), 580 HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU), 581 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX), 582 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH), 583 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD), 584 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD), 585 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD), 586 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD), 587 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0), 588 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1), 589 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0), 590 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1), 591 592 /* packet rate events */ 593 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU), 594 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU), 595 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU), 596 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE), 597 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU), 598 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU), 599 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX), 600 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU), 601 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH), 602 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD), 603 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD), 604 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD), 605 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD), 606 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0), 607 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1), 608 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0), 609 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1), 610 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE), 611 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE), 612 613 /* latency events */ 614 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH), 615 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX), 616 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC), 617 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE), 618 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC), 619 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE), 620 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU), 621 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU), 622 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE), 623 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE), 624 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH), 625 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD), 626 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD), 627 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD), 628 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD), 629 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0), 630 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1), 631 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0), 632 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1), 633 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE), 634 635 /* interrupt rate events */ 636 HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC), 637 638 NULL 639 }; 640 641 static struct attribute_group hns3_pmu_events_group = { 642 .name = "events", 643 .attrs = hns3_pmu_events_attr, 644 }; 645 646 static struct attribute_group hns3_pmu_filter_mode_group = { 647 .name = "filtermode", 648 .attrs = hns3_pmu_filter_mode_attr, 649 }; 650 651 static struct attribute *hns3_pmu_format_attr[] = { 652 HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"), 653 HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"), 654 HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"), 655 HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"), 656 HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"), 657 HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"), 658 HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"), 659 HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"), 660 HNS3_PMU_FORMAT_ATTR(global, "config1:52"), 661 NULL 662 }; 663 664 static struct attribute_group hns3_pmu_format_group = { 665 .name = "format", 666 .attrs = hns3_pmu_format_attr, 667 }; 668 669 static struct attribute *hns3_pmu_cpumask_attrs[] = { 670 &dev_attr_cpumask.attr, 671 NULL 672 }; 673 674 static struct attribute_group hns3_pmu_cpumask_attr_group = { 675 .attrs = hns3_pmu_cpumask_attrs, 676 }; 677 678 static struct attribute *hns3_pmu_identifier_attrs[] = { 679 &dev_attr_identifier.attr, 680 NULL 681 }; 682 683 static struct attribute_group hns3_pmu_identifier_attr_group = { 684 .attrs = hns3_pmu_identifier_attrs, 685 }; 686 687 static struct attribute *hns3_pmu_bdf_range_attrs[] = { 688 &dev_attr_bdf_min.attr, 689 &dev_attr_bdf_max.attr, 690 NULL 691 }; 692 693 static struct attribute_group hns3_pmu_bdf_range_attr_group = { 694 .attrs = hns3_pmu_bdf_range_attrs, 695 }; 696 697 static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = { 698 &dev_attr_hw_clk_freq.attr, 699 NULL 700 }; 701 702 static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = { 703 .attrs = hns3_pmu_hw_clk_freq_attrs, 704 }; 705 706 static const struct attribute_group *hns3_pmu_attr_groups[] = { 707 &hns3_pmu_events_group, 708 &hns3_pmu_filter_mode_group, 709 &hns3_pmu_format_group, 710 &hns3_pmu_cpumask_attr_group, 711 &hns3_pmu_identifier_attr_group, 712 &hns3_pmu_bdf_range_attr_group, 713 &hns3_pmu_hw_clk_freq_attr_group, 714 NULL 715 }; 716 717 static u32 hns3_pmu_get_event(struct perf_event *event) 718 { 719 return hns3_pmu_get_ext_counter_used(event) << 16 | 720 hns3_pmu_get_event_type(event) << 8 | 721 hns3_pmu_get_subevent(event); 722 } 723 724 static u32 hns3_pmu_get_real_event(struct perf_event *event) 725 { 726 return hns3_pmu_get_event_type(event) << 8 | 727 hns3_pmu_get_subevent(event); 728 } 729 730 static u32 hns3_pmu_get_offset(u32 offset, u32 idx) 731 { 732 return offset + HNS3_PMU_REG_EVENT_OFFSET + 733 HNS3_PMU_REG_EVENT_SIZE * idx; 734 } 735 736 static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx) 737 { 738 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 739 740 return readl(hns3_pmu->base + offset); 741 } 742 743 static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx, 744 u32 val) 745 { 746 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 747 748 writel(val, hns3_pmu->base + offset); 749 } 750 751 static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx) 752 { 753 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 754 755 return readq(hns3_pmu->base + offset); 756 } 757 758 static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx, 759 u64 val) 760 { 761 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 762 763 writeq(val, hns3_pmu->base + offset); 764 } 765 766 static bool hns3_pmu_cmp_event(struct perf_event *target, 767 struct perf_event *event) 768 { 769 return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event); 770 } 771 772 static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu, 773 struct perf_event *event) 774 { 775 struct perf_event *sibling; 776 int hw_event_used = 0; 777 int idx; 778 779 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 780 sibling = hns3_pmu->hw_events[idx]; 781 if (!sibling) 782 continue; 783 784 hw_event_used++; 785 786 if (!hns3_pmu_cmp_event(sibling, event)) 787 continue; 788 789 /* Related events is used in group */ 790 if (sibling->group_leader == event->group_leader) 791 return idx; 792 } 793 794 /* No related event and all hardware events are used up */ 795 if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS) 796 return -EBUSY; 797 798 /* No related event and there is extra hardware events can be use */ 799 return -ENOENT; 800 } 801 802 static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu) 803 { 804 int idx; 805 806 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 807 if (!hns3_pmu->hw_events[idx]) 808 return idx; 809 } 810 811 return -EBUSY; 812 } 813 814 static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf) 815 { 816 struct pci_dev *pdev; 817 818 if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) { 819 pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf); 820 return false; 821 } 822 823 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus), 824 PCI_BUS_NUM(bdf), 825 GET_PCI_DEVFN(bdf)); 826 if (!pdev) { 827 pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf); 828 return false; 829 } 830 831 pci_dev_put(pdev); 832 return true; 833 } 834 835 static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf, 836 u16 queue) 837 { 838 u32 val; 839 840 val = GET_PCI_DEVFN(bdf); 841 val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S; 842 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val); 843 } 844 845 static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx) 846 { 847 bool queue_id_valid = false; 848 u32 reg_qid_ctrl, val; 849 int err; 850 851 /* enable queue id request */ 852 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 853 HNS3_PMU_QID_CTRL_REQ_ENABLE); 854 855 reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx); 856 err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val, 857 val & HNS3_PMU_QID_CTRL_DONE, 1, 1000); 858 if (err == -ETIMEDOUT) { 859 pci_err(hns3_pmu->pdev, "QID request timeout!\n"); 860 goto out; 861 } 862 863 queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS); 864 865 out: 866 /* disable qid request and clear status */ 867 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0); 868 869 return queue_id_valid; 870 } 871 872 static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf, 873 u16 queue) 874 { 875 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue); 876 877 return hns3_pmu_qid_req_start(hns3_pmu, idx); 878 } 879 880 static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event) 881 { 882 struct hns3_pmu_event_attr *pmu_event; 883 struct dev_ext_attribute *eattr; 884 struct device_attribute *dattr; 885 struct attribute *attr; 886 u32 i; 887 888 for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) { 889 attr = hns3_pmu_events_attr[i]; 890 dattr = container_of(attr, struct device_attribute, attr); 891 eattr = container_of(dattr, struct dev_ext_attribute, attr); 892 pmu_event = eattr->var; 893 894 if (event == pmu_event->event) 895 return pmu_event; 896 } 897 898 return NULL; 899 } 900 901 static int hns3_pmu_set_func_mode(struct perf_event *event, 902 struct hns3_pmu *hns3_pmu) 903 { 904 struct hw_perf_event *hwc = &event->hw; 905 u16 bdf = hns3_pmu_get_bdf(event); 906 907 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf)) 908 return -ENOENT; 909 910 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC); 911 912 return 0; 913 } 914 915 static int hns3_pmu_set_func_queue_mode(struct perf_event *event, 916 struct hns3_pmu *hns3_pmu) 917 { 918 u16 queue_id = hns3_pmu_get_queue(event); 919 struct hw_perf_event *hwc = &event->hw; 920 u16 bdf = hns3_pmu_get_bdf(event); 921 922 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf)) 923 return -ENOENT; 924 925 if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) { 926 pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id); 927 return -ENOENT; 928 } 929 930 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE); 931 932 return 0; 933 } 934 935 static bool 936 hns3_pmu_is_enabled_global_mode(struct perf_event *event, 937 struct hns3_pmu_event_attr *pmu_event) 938 { 939 u8 global = hns3_pmu_get_global(event); 940 941 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)) 942 return false; 943 944 return global; 945 } 946 947 static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event, 948 struct hns3_pmu_event_attr *pmu_event) 949 { 950 u16 queue_id = hns3_pmu_get_queue(event); 951 u16 bdf = hns3_pmu_get_bdf(event); 952 953 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)) 954 return false; 955 else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE) 956 return false; 957 958 return bdf; 959 } 960 961 static bool 962 hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event, 963 struct hns3_pmu_event_attr *pmu_event) 964 { 965 u16 queue_id = hns3_pmu_get_queue(event); 966 u16 bdf = hns3_pmu_get_bdf(event); 967 968 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)) 969 return false; 970 else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE) 971 return false; 972 973 return bdf; 974 } 975 976 static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event, 977 struct hns3_pmu_event_attr *pmu_event) 978 { 979 u8 tc_id = hns3_pmu_get_tc(event); 980 981 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)) 982 return false; 983 984 return tc_id == HNS3_PMU_FILTER_ALL_TC; 985 } 986 987 static bool 988 hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event, 989 struct hns3_pmu_event_attr *pmu_event) 990 { 991 u8 tc_id = hns3_pmu_get_tc(event); 992 993 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)) 994 return false; 995 996 return tc_id != HNS3_PMU_FILTER_ALL_TC; 997 } 998 999 static bool 1000 hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event, 1001 struct hns3_pmu *hns3_pmu, 1002 struct hns3_pmu_event_attr *pmu_event) 1003 { 1004 u16 bdf = hns3_pmu_get_bdf(event); 1005 1006 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)) 1007 return false; 1008 1009 return hns3_pmu_valid_bdf(hns3_pmu, bdf); 1010 } 1011 1012 static int hns3_pmu_select_filter_mode(struct perf_event *event, 1013 struct hns3_pmu *hns3_pmu) 1014 { 1015 u32 event_id = hns3_pmu_get_event(event); 1016 struct hw_perf_event *hwc = &event->hw; 1017 struct hns3_pmu_event_attr *pmu_event; 1018 1019 pmu_event = hns3_pmu_get_pmu_event(event_id); 1020 if (!pmu_event) { 1021 pci_err(hns3_pmu->pdev, "Invalid pmu event\n"); 1022 return -ENOENT; 1023 } 1024 1025 if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) { 1026 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL); 1027 return 0; 1028 } 1029 1030 if (hns3_pmu_is_enabled_func_mode(event, pmu_event)) 1031 return hns3_pmu_set_func_mode(event, hns3_pmu); 1032 1033 if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event)) 1034 return hns3_pmu_set_func_queue_mode(event, hns3_pmu); 1035 1036 if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) { 1037 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT); 1038 return 0; 1039 } 1040 1041 if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) { 1042 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC); 1043 return 0; 1044 } 1045 1046 if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) { 1047 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR); 1048 return 0; 1049 } 1050 1051 return -ENOENT; 1052 } 1053 1054 static bool hns3_pmu_validate_event_group(struct perf_event *event) 1055 { 1056 struct perf_event *sibling, *leader = event->group_leader; 1057 struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS]; 1058 int counters = 1; 1059 int num; 1060 1061 event_group[0] = leader; 1062 if (!is_software_event(leader)) { 1063 if (leader->pmu != event->pmu) 1064 return false; 1065 1066 if (leader != event && !hns3_pmu_cmp_event(leader, event)) 1067 event_group[counters++] = event; 1068 } 1069 1070 for_each_sibling_event(sibling, event->group_leader) { 1071 if (is_software_event(sibling)) 1072 continue; 1073 1074 if (sibling->pmu != event->pmu) 1075 return false; 1076 1077 for (num = 0; num < counters; num++) { 1078 /* 1079 * If we find a related event, then it's a valid group 1080 * since we don't need to allocate a new counter for it. 1081 */ 1082 if (hns3_pmu_cmp_event(event_group[num], sibling)) 1083 break; 1084 } 1085 1086 /* 1087 * Otherwise it's a new event but if there's no available counter, 1088 * fail the check since we cannot schedule all the events in 1089 * the group simultaneously. 1090 */ 1091 if (num == HNS3_PMU_MAX_HW_EVENTS) 1092 return false; 1093 1094 if (num == counters) 1095 event_group[counters++] = sibling; 1096 } 1097 1098 return true; 1099 } 1100 1101 static u32 hns3_pmu_get_filter_condition(struct perf_event *event) 1102 { 1103 struct hw_perf_event *hwc = &event->hw; 1104 u16 intr_id = hns3_pmu_get_intr(event); 1105 u8 port_id = hns3_pmu_get_port(event); 1106 u16 bdf = hns3_pmu_get_bdf(event); 1107 u8 tc_id = hns3_pmu_get_tc(event); 1108 u8 filter_mode; 1109 1110 filter_mode = *(u8 *)hwc->addr_filters; 1111 switch (filter_mode) { 1112 case HNS3_PMU_HW_FILTER_PORT: 1113 return FILTER_CONDITION_PORT(port_id); 1114 case HNS3_PMU_HW_FILTER_PORT_TC: 1115 return FILTER_CONDITION_PORT_TC(port_id, tc_id); 1116 case HNS3_PMU_HW_FILTER_FUNC: 1117 case HNS3_PMU_HW_FILTER_FUNC_QUEUE: 1118 return GET_PCI_DEVFN(bdf); 1119 case HNS3_PMU_HW_FILTER_FUNC_INTR: 1120 return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id); 1121 default: 1122 break; 1123 } 1124 1125 return 0; 1126 } 1127 1128 static void hns3_pmu_config_filter(struct perf_event *event) 1129 { 1130 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1131 u8 event_type = hns3_pmu_get_event_type(event); 1132 u8 subevent_id = hns3_pmu_get_subevent(event); 1133 u16 queue_id = hns3_pmu_get_queue(event); 1134 struct hw_perf_event *hwc = &event->hw; 1135 u8 filter_mode = *(u8 *)hwc->addr_filters; 1136 u16 bdf = hns3_pmu_get_bdf(event); 1137 u32 idx = hwc->idx; 1138 u32 val; 1139 1140 val = event_type; 1141 val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S; 1142 val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S; 1143 val |= HNS3_PMU_EVENT_OVERFLOW_RESTART; 1144 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1145 1146 val = hns3_pmu_get_filter_condition(event); 1147 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val); 1148 1149 if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE) 1150 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id); 1151 } 1152 1153 static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu, 1154 struct hw_perf_event *hwc) 1155 { 1156 u32 idx = hwc->idx; 1157 u32 val; 1158 1159 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1160 val |= HNS3_PMU_EVENT_EN; 1161 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1162 } 1163 1164 static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu, 1165 struct hw_perf_event *hwc) 1166 { 1167 u32 idx = hwc->idx; 1168 u32 val; 1169 1170 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1171 val &= ~HNS3_PMU_EVENT_EN; 1172 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1173 } 1174 1175 static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu, 1176 struct hw_perf_event *hwc) 1177 { 1178 u32 idx = hwc->idx; 1179 u32 val; 1180 1181 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx); 1182 val &= ~HNS3_PMU_INTR_MASK_OVERFLOW; 1183 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val); 1184 } 1185 1186 static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu, 1187 struct hw_perf_event *hwc) 1188 { 1189 u32 idx = hwc->idx; 1190 u32 val; 1191 1192 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx); 1193 val |= HNS3_PMU_INTR_MASK_OVERFLOW; 1194 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val); 1195 } 1196 1197 static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx) 1198 { 1199 u32 val; 1200 1201 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1202 val |= HNS3_PMU_EVENT_STATUS_RESET; 1203 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1204 1205 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1206 val &= ~HNS3_PMU_EVENT_STATUS_RESET; 1207 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1208 } 1209 1210 static u64 hns3_pmu_read_counter(struct perf_event *event) 1211 { 1212 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1213 1214 return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx); 1215 } 1216 1217 static void hns3_pmu_write_counter(struct perf_event *event, u64 value) 1218 { 1219 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1220 u32 idx = event->hw.idx; 1221 1222 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value); 1223 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value); 1224 } 1225 1226 static void hns3_pmu_init_counter(struct perf_event *event) 1227 { 1228 struct hw_perf_event *hwc = &event->hw; 1229 1230 local64_set(&hwc->prev_count, 0); 1231 hns3_pmu_write_counter(event, 0); 1232 } 1233 1234 static int hns3_pmu_event_init(struct perf_event *event) 1235 { 1236 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1237 struct hw_perf_event *hwc = &event->hw; 1238 int idx; 1239 int ret; 1240 1241 if (event->attr.type != event->pmu->type) 1242 return -ENOENT; 1243 1244 /* Sampling is not supported */ 1245 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1246 return -EOPNOTSUPP; 1247 1248 event->cpu = hns3_pmu->on_cpu; 1249 1250 idx = hns3_pmu_get_event_idx(hns3_pmu); 1251 if (idx < 0) { 1252 pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n", 1253 HNS3_PMU_MAX_HW_EVENTS); 1254 return -EBUSY; 1255 } 1256 1257 hwc->idx = idx; 1258 1259 ret = hns3_pmu_select_filter_mode(event, hns3_pmu); 1260 if (ret) { 1261 pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret); 1262 return ret; 1263 } 1264 1265 if (!hns3_pmu_validate_event_group(event)) { 1266 pci_err(hns3_pmu->pdev, "Invalid event group.\n"); 1267 return -EINVAL; 1268 } 1269 1270 if (hns3_pmu_get_ext_counter_used(event)) 1271 hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER; 1272 else 1273 hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER; 1274 1275 return 0; 1276 } 1277 1278 static void hns3_pmu_read(struct perf_event *event) 1279 { 1280 struct hw_perf_event *hwc = &event->hw; 1281 u64 new_cnt, prev_cnt, delta; 1282 1283 do { 1284 prev_cnt = local64_read(&hwc->prev_count); 1285 new_cnt = hns3_pmu_read_counter(event); 1286 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != 1287 prev_cnt); 1288 1289 delta = new_cnt - prev_cnt; 1290 local64_add(delta, &event->count); 1291 } 1292 1293 static void hns3_pmu_start(struct perf_event *event, int flags) 1294 { 1295 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1296 struct hw_perf_event *hwc = &event->hw; 1297 1298 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 1299 return; 1300 1301 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 1302 hwc->state = 0; 1303 1304 hns3_pmu_config_filter(event); 1305 hns3_pmu_init_counter(event); 1306 hns3_pmu_enable_intr(hns3_pmu, hwc); 1307 hns3_pmu_enable_counter(hns3_pmu, hwc); 1308 1309 perf_event_update_userpage(event); 1310 } 1311 1312 static void hns3_pmu_stop(struct perf_event *event, int flags) 1313 { 1314 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1315 struct hw_perf_event *hwc = &event->hw; 1316 1317 hns3_pmu_disable_counter(hns3_pmu, hwc); 1318 hns3_pmu_disable_intr(hns3_pmu, hwc); 1319 1320 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1321 hwc->state |= PERF_HES_STOPPED; 1322 1323 if (hwc->state & PERF_HES_UPTODATE) 1324 return; 1325 1326 /* Read hardware counter and update the perf counter statistics */ 1327 hns3_pmu_read(event); 1328 hwc->state |= PERF_HES_UPTODATE; 1329 } 1330 1331 static int hns3_pmu_add(struct perf_event *event, int flags) 1332 { 1333 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1334 struct hw_perf_event *hwc = &event->hw; 1335 int idx; 1336 1337 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1338 1339 /* Check all working events to find a related event. */ 1340 idx = hns3_pmu_find_related_event_idx(hns3_pmu, event); 1341 if (idx < 0 && idx != -ENOENT) 1342 return idx; 1343 1344 /* Current event shares an enabled hardware event with related event */ 1345 if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) { 1346 hwc->idx = idx; 1347 goto start_count; 1348 } 1349 1350 idx = hns3_pmu_get_event_idx(hns3_pmu); 1351 if (idx < 0) 1352 return idx; 1353 1354 hwc->idx = idx; 1355 hns3_pmu->hw_events[idx] = event; 1356 1357 start_count: 1358 if (flags & PERF_EF_START) 1359 hns3_pmu_start(event, PERF_EF_RELOAD); 1360 1361 return 0; 1362 } 1363 1364 static void hns3_pmu_del(struct perf_event *event, int flags) 1365 { 1366 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1367 struct hw_perf_event *hwc = &event->hw; 1368 1369 hns3_pmu_stop(event, PERF_EF_UPDATE); 1370 hns3_pmu->hw_events[hwc->idx] = NULL; 1371 perf_event_update_userpage(event); 1372 } 1373 1374 static void hns3_pmu_enable(struct pmu *pmu) 1375 { 1376 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu); 1377 u32 val; 1378 1379 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1380 val |= HNS3_PMU_GLOBAL_START; 1381 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1382 } 1383 1384 static void hns3_pmu_disable(struct pmu *pmu) 1385 { 1386 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu); 1387 u32 val; 1388 1389 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1390 val &= ~HNS3_PMU_GLOBAL_START; 1391 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1392 } 1393 1394 static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu) 1395 { 1396 u16 device_id; 1397 char *name; 1398 u32 val; 1399 1400 hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2]; 1401 if (!hns3_pmu->base) { 1402 pci_err(pdev, "ioremap failed\n"); 1403 return -ENOMEM; 1404 } 1405 1406 hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ); 1407 1408 val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF); 1409 hns3_pmu->bdf_min = val & 0xffff; 1410 hns3_pmu->bdf_max = val >> 16; 1411 1412 val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID); 1413 device_id = val & 0xffff; 1414 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id); 1415 if (!name) 1416 return -ENOMEM; 1417 1418 hns3_pmu->pdev = pdev; 1419 hns3_pmu->on_cpu = -1; 1420 hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION); 1421 hns3_pmu->pmu = (struct pmu) { 1422 .name = name, 1423 .module = THIS_MODULE, 1424 .parent = &pdev->dev, 1425 .event_init = hns3_pmu_event_init, 1426 .pmu_enable = hns3_pmu_enable, 1427 .pmu_disable = hns3_pmu_disable, 1428 .add = hns3_pmu_add, 1429 .del = hns3_pmu_del, 1430 .start = hns3_pmu_start, 1431 .stop = hns3_pmu_stop, 1432 .read = hns3_pmu_read, 1433 .task_ctx_nr = perf_invalid_context, 1434 .attr_groups = hns3_pmu_attr_groups, 1435 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1436 }; 1437 1438 return 0; 1439 } 1440 1441 static irqreturn_t hns3_pmu_irq(int irq, void *data) 1442 { 1443 struct hns3_pmu *hns3_pmu = data; 1444 u32 intr_status, idx; 1445 1446 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 1447 intr_status = hns3_pmu_readl(hns3_pmu, 1448 HNS3_PMU_REG_EVENT_INTR_STATUS, 1449 idx); 1450 1451 /* 1452 * As each counter will restart from 0 when it is overflowed, 1453 * extra processing is no need, just clear interrupt status. 1454 */ 1455 if (intr_status) 1456 hns3_pmu_clear_intr_status(hns3_pmu, idx); 1457 } 1458 1459 return IRQ_HANDLED; 1460 } 1461 1462 static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 1463 { 1464 struct hns3_pmu *hns3_pmu; 1465 1466 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node); 1467 if (!hns3_pmu) 1468 return -ENODEV; 1469 1470 if (hns3_pmu->on_cpu == -1) { 1471 hns3_pmu->on_cpu = cpu; 1472 irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu)); 1473 } 1474 1475 return 0; 1476 } 1477 1478 static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1479 { 1480 struct hns3_pmu *hns3_pmu; 1481 unsigned int target; 1482 1483 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node); 1484 if (!hns3_pmu) 1485 return -ENODEV; 1486 1487 /* Nothing to do if this CPU doesn't own the PMU */ 1488 if (hns3_pmu->on_cpu != cpu) 1489 return 0; 1490 1491 /* Choose a new CPU from all online cpus */ 1492 target = cpumask_any_but(cpu_online_mask, cpu); 1493 if (target >= nr_cpu_ids) 1494 return 0; 1495 1496 perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target); 1497 hns3_pmu->on_cpu = target; 1498 irq_set_affinity(hns3_pmu->irq, cpumask_of(target)); 1499 1500 return 0; 1501 } 1502 1503 static void hns3_pmu_free_irq(void *data) 1504 { 1505 struct pci_dev *pdev = data; 1506 1507 pci_free_irq_vectors(pdev); 1508 } 1509 1510 static int hns3_pmu_irq_register(struct pci_dev *pdev, 1511 struct hns3_pmu *hns3_pmu) 1512 { 1513 int irq, ret; 1514 1515 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); 1516 if (ret < 0) { 1517 pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret); 1518 return ret; 1519 } 1520 1521 ret = devm_add_action_or_reset(&pdev->dev, hns3_pmu_free_irq, pdev); 1522 if (ret) { 1523 pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret); 1524 return ret; 1525 } 1526 1527 irq = pci_irq_vector(pdev, 0); 1528 ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0, 1529 hns3_pmu->pmu.name, hns3_pmu); 1530 if (ret) { 1531 pci_err(pdev, "failed to register irq, ret = %d.\n", ret); 1532 return ret; 1533 } 1534 1535 hns3_pmu->irq = irq; 1536 1537 return 0; 1538 } 1539 1540 static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu) 1541 { 1542 int ret; 1543 1544 ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu); 1545 if (ret) 1546 return ret; 1547 1548 ret = hns3_pmu_irq_register(pdev, hns3_pmu); 1549 if (ret) 1550 return ret; 1551 1552 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1553 &hns3_pmu->node); 1554 if (ret) { 1555 pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret); 1556 return ret; 1557 } 1558 1559 ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1); 1560 if (ret) { 1561 pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret); 1562 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1563 &hns3_pmu->node); 1564 } 1565 1566 return ret; 1567 } 1568 1569 static void hns3_pmu_uninit_pmu(struct pci_dev *pdev) 1570 { 1571 struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev); 1572 1573 perf_pmu_unregister(&hns3_pmu->pmu); 1574 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1575 &hns3_pmu->node); 1576 } 1577 1578 static int hns3_pmu_init_dev(struct pci_dev *pdev) 1579 { 1580 int ret; 1581 1582 ret = pcim_enable_device(pdev); 1583 if (ret) { 1584 pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret); 1585 return ret; 1586 } 1587 1588 ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu"); 1589 if (ret < 0) { 1590 pci_err(pdev, "failed to request pci region, ret = %d.\n", ret); 1591 return ret; 1592 } 1593 1594 pci_set_master(pdev); 1595 1596 return 0; 1597 } 1598 1599 static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1600 { 1601 struct hns3_pmu *hns3_pmu; 1602 int ret; 1603 1604 hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL); 1605 if (!hns3_pmu) 1606 return -ENOMEM; 1607 1608 ret = hns3_pmu_init_dev(pdev); 1609 if (ret) 1610 return ret; 1611 1612 ret = hns3_pmu_init_pmu(pdev, hns3_pmu); 1613 if (ret) { 1614 pci_clear_master(pdev); 1615 return ret; 1616 } 1617 1618 pci_set_drvdata(pdev, hns3_pmu); 1619 1620 return ret; 1621 } 1622 1623 static void hns3_pmu_remove(struct pci_dev *pdev) 1624 { 1625 hns3_pmu_uninit_pmu(pdev); 1626 pci_clear_master(pdev); 1627 pci_set_drvdata(pdev, NULL); 1628 } 1629 1630 static const struct pci_device_id hns3_pmu_ids[] = { 1631 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) }, 1632 { 0, } 1633 }; 1634 MODULE_DEVICE_TABLE(pci, hns3_pmu_ids); 1635 1636 static struct pci_driver hns3_pmu_driver = { 1637 .name = "hns3_pmu", 1638 .id_table = hns3_pmu_ids, 1639 .probe = hns3_pmu_probe, 1640 .remove = hns3_pmu_remove, 1641 }; 1642 1643 static int __init hns3_pmu_module_init(void) 1644 { 1645 int ret; 1646 1647 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1648 "AP_PERF_ARM_HNS3_PMU_ONLINE", 1649 hns3_pmu_online_cpu, 1650 hns3_pmu_offline_cpu); 1651 if (ret) { 1652 pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret); 1653 return ret; 1654 } 1655 1656 ret = pci_register_driver(&hns3_pmu_driver); 1657 if (ret) { 1658 pr_err("failed to register pci driver, ret = %d.\n", ret); 1659 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE); 1660 } 1661 1662 return ret; 1663 } 1664 module_init(hns3_pmu_module_init); 1665 1666 static void __exit hns3_pmu_module_exit(void) 1667 { 1668 pci_unregister_driver(&hns3_pmu_driver); 1669 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE); 1670 } 1671 module_exit(hns3_pmu_module_exit); 1672 1673 MODULE_DESCRIPTION("HNS3 PMU driver"); 1674 MODULE_LICENSE("GPL v2"); 1675