1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <stdio.h> 8 #include <string.h> 9 #include <endian.h> 10 #include <byteswap.h> 11 #include <linux/bitops.h> 12 #include <stdarg.h> 13 #include <linux/kernel.h> 14 #include <linux/unaligned.h> 15 16 #include "arm-spe-pkt-decoder.h" 17 18 static const char * const arm_spe_packet_name[] = { 19 [ARM_SPE_PAD] = "PAD", 20 [ARM_SPE_END] = "END", 21 [ARM_SPE_TIMESTAMP] = "TS", 22 [ARM_SPE_ADDRESS] = "ADDR", 23 [ARM_SPE_COUNTER] = "LAT", 24 [ARM_SPE_CONTEXT] = "CONTEXT", 25 [ARM_SPE_OP_TYPE] = "OP-TYPE", 26 [ARM_SPE_EVENTS] = "EVENTS", 27 [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", 28 }; 29 30 const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) 31 { 32 return arm_spe_packet_name[type]; 33 } 34 35 /* 36 * Extracts the field "sz" from header bits and converts to bytes: 37 * 00 : byte (1) 38 * 01 : halfword (2) 39 * 10 : word (4) 40 * 11 : doubleword (8) 41 */ 42 static unsigned int arm_spe_payload_len(unsigned char hdr) 43 { 44 return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); 45 } 46 47 static int arm_spe_get_payload(const unsigned char *buf, size_t len, 48 unsigned char ext_hdr, 49 struct arm_spe_pkt *packet) 50 { 51 size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); 52 53 if (len < 1 + ext_hdr + payload_len) 54 return ARM_SPE_NEED_MORE_BYTES; 55 56 buf += 1 + ext_hdr; 57 58 switch (payload_len) { 59 case 1: packet->payload = *(uint8_t *)buf; break; 60 case 2: packet->payload = get_unaligned_le16(buf); break; 61 case 4: packet->payload = get_unaligned_le32(buf); break; 62 case 8: packet->payload = get_unaligned_le64(buf); break; 63 default: return ARM_SPE_BAD_PACKET; 64 } 65 66 return 1 + ext_hdr + payload_len; 67 } 68 69 static int arm_spe_get_pad(struct arm_spe_pkt *packet) 70 { 71 packet->type = ARM_SPE_PAD; 72 return 1; 73 } 74 75 static int arm_spe_get_alignment(const unsigned char *buf, size_t len, 76 struct arm_spe_pkt *packet) 77 { 78 unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); 79 80 if (len < alignment) 81 return ARM_SPE_NEED_MORE_BYTES; 82 83 packet->type = ARM_SPE_PAD; 84 return alignment - (((uintptr_t)buf) & (alignment - 1)); 85 } 86 87 static int arm_spe_get_end(struct arm_spe_pkt *packet) 88 { 89 packet->type = ARM_SPE_END; 90 return 1; 91 } 92 93 static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, 94 struct arm_spe_pkt *packet) 95 { 96 packet->type = ARM_SPE_TIMESTAMP; 97 return arm_spe_get_payload(buf, len, 0, packet); 98 } 99 100 static int arm_spe_get_events(const unsigned char *buf, size_t len, 101 struct arm_spe_pkt *packet) 102 { 103 packet->type = ARM_SPE_EVENTS; 104 105 /* we use index to identify Events with a less number of 106 * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, 107 * LLC-REFILL, and REMOTE-ACCESS events are identified if 108 * index > 1. 109 */ 110 packet->index = arm_spe_payload_len(buf[0]); 111 112 return arm_spe_get_payload(buf, len, 0, packet); 113 } 114 115 static int arm_spe_get_data_source(const unsigned char *buf, size_t len, 116 struct arm_spe_pkt *packet) 117 { 118 packet->type = ARM_SPE_DATA_SOURCE; 119 return arm_spe_get_payload(buf, len, 0, packet); 120 } 121 122 static int arm_spe_get_context(const unsigned char *buf, size_t len, 123 struct arm_spe_pkt *packet) 124 { 125 packet->type = ARM_SPE_CONTEXT; 126 packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); 127 return arm_spe_get_payload(buf, len, 0, packet); 128 } 129 130 static int arm_spe_get_op_type(const unsigned char *buf, size_t len, 131 struct arm_spe_pkt *packet) 132 { 133 packet->type = ARM_SPE_OP_TYPE; 134 packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); 135 return arm_spe_get_payload(buf, len, 0, packet); 136 } 137 138 static int arm_spe_get_counter(const unsigned char *buf, size_t len, 139 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 140 { 141 packet->type = ARM_SPE_COUNTER; 142 143 if (ext_hdr) 144 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 145 else 146 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 147 148 return arm_spe_get_payload(buf, len, ext_hdr, packet); 149 } 150 151 static int arm_spe_get_addr(const unsigned char *buf, size_t len, 152 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 153 { 154 packet->type = ARM_SPE_ADDRESS; 155 156 if (ext_hdr) 157 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 158 else 159 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 160 161 return arm_spe_get_payload(buf, len, ext_hdr, packet); 162 } 163 164 static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, 165 struct arm_spe_pkt *packet) 166 { 167 unsigned int hdr; 168 unsigned char ext_hdr = 0; 169 170 memset(packet, 0, sizeof(struct arm_spe_pkt)); 171 172 if (!len) 173 return ARM_SPE_NEED_MORE_BYTES; 174 175 hdr = buf[0]; 176 177 if (hdr == SPE_HEADER0_PAD) 178 return arm_spe_get_pad(packet); 179 180 if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ 181 return arm_spe_get_end(packet); 182 183 if (hdr == SPE_HEADER0_TIMESTAMP) 184 return arm_spe_get_timestamp(buf, len, packet); 185 186 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) 187 return arm_spe_get_events(buf, len, packet); 188 189 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) 190 return arm_spe_get_data_source(buf, len, packet); 191 192 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) 193 return arm_spe_get_context(buf, len, packet); 194 195 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) 196 return arm_spe_get_op_type(buf, len, packet); 197 198 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { 199 /* 16-bit extended format header */ 200 if (len == 1) 201 return ARM_SPE_BAD_PACKET; 202 203 ext_hdr = 1; 204 hdr = buf[1]; 205 if (hdr == SPE_HEADER1_ALIGNMENT) 206 return arm_spe_get_alignment(buf, len, packet); 207 } 208 209 /* 210 * The short format header's byte 0 or the extended format header's 211 * byte 1 has been assigned to 'hdr', which uses the same encoding for 212 * address packet and counter packet, so don't need to distinguish if 213 * it's short format or extended format and handle in once. 214 */ 215 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) 216 return arm_spe_get_addr(buf, len, ext_hdr, packet); 217 218 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) 219 return arm_spe_get_counter(buf, len, ext_hdr, packet); 220 221 return ARM_SPE_BAD_PACKET; 222 } 223 224 int arm_spe_get_packet(const unsigned char *buf, size_t len, 225 struct arm_spe_pkt *packet) 226 { 227 int ret; 228 229 ret = arm_spe_do_get_packet(buf, len, packet); 230 /* put multiple consecutive PADs on the same line, up to 231 * the fixed-width output format of 16 bytes per line. 232 */ 233 if (ret > 0 && packet->type == ARM_SPE_PAD) { 234 while (ret < 16 && len > (size_t)ret && !buf[ret]) 235 ret += 1; 236 } 237 return ret; 238 } 239 240 static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, 241 const char *fmt, ...) 242 { 243 va_list ap; 244 int ret; 245 246 /* Bail out if any error occurred */ 247 if (err && *err) 248 return *err; 249 250 va_start(ap, fmt); 251 ret = vsnprintf(*buf_p, *blen, fmt, ap); 252 va_end(ap); 253 254 if (ret < 0) { 255 if (err && !*err) 256 *err = ret; 257 258 /* 259 * A return value of *blen or more means that the output was 260 * truncated and the buffer is overrun. 261 */ 262 } else if ((size_t)ret >= *blen) { 263 (*buf_p)[*blen - 1] = '\0'; 264 265 /* 266 * Set *err to 'ret' to avoid overflow if tries to 267 * fill this buffer sequentially. 268 */ 269 if (err && !*err) 270 *err = ret; 271 } else { 272 *buf_p += ret; 273 *blen -= ret; 274 } 275 276 return ret; 277 } 278 279 static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, 280 char *buf, size_t buf_len) 281 { 282 u64 payload = packet->payload; 283 int err = 0; 284 285 arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); 286 287 if (payload & BIT(EV_EXCEPTION_GEN)) 288 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); 289 if (payload & BIT(EV_RETIRED)) 290 arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); 291 if (payload & BIT(EV_L1D_ACCESS)) 292 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); 293 if (payload & BIT(EV_L1D_REFILL)) 294 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); 295 if (payload & BIT(EV_TLB_ACCESS)) 296 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); 297 if (payload & BIT(EV_TLB_WALK)) 298 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); 299 if (payload & BIT(EV_NOT_TAKEN)) 300 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); 301 if (payload & BIT(EV_MISPRED)) 302 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); 303 if (payload & BIT(EV_LLC_ACCESS)) 304 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); 305 if (payload & BIT(EV_LLC_MISS)) 306 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); 307 if (payload & BIT(EV_REMOTE_ACCESS)) 308 arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); 309 if (payload & BIT(EV_ALIGNMENT)) 310 arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); 311 if (payload & BIT(EV_TRANSACTIONAL)) 312 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TXN"); 313 if (payload & BIT(EV_PARTIAL_PREDICATE)) 314 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); 315 if (payload & BIT(EV_EMPTY_PREDICATE)) 316 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); 317 if (payload & BIT(EV_L2D_ACCESS)) 318 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-ACCESS"); 319 if (payload & BIT(EV_L2D_MISS)) 320 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-MISS"); 321 if (payload & BIT(EV_CACHE_DATA_MODIFIED)) 322 arm_spe_pkt_out_string(&err, &buf, &buf_len, " HITM"); 323 if (payload & BIT(EV_RECENTLY_FETCHED)) 324 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LFB"); 325 if (payload & BIT(EV_DATA_SNOOPED)) 326 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SNOOPED"); 327 if (payload & BIT(EV_STREAMING_SVE_MODE)) 328 arm_spe_pkt_out_string(&err, &buf, &buf_len, " STREAMING-SVE"); 329 if (payload & BIT(EV_SMCU)) 330 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SMCU"); 331 332 return err; 333 } 334 335 static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, 336 char *buf, size_t buf_len) 337 { 338 u64 payload = packet->payload; 339 int err = 0; 340 341 switch (packet->index) { 342 case SPE_OP_PKT_HDR_CLASS_OTHER: 343 if (SPE_OP_PKT_OTHER_SUBCLASS_SVE(payload)) { 344 arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); 345 346 /* SVE effective vector length */ 347 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 348 SPE_OP_PKG_SVE_EVL(payload)); 349 350 if (payload & SPE_OP_PKT_SVE_FP) 351 arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); 352 if (payload & SPE_OP_PKT_SVE_PRED) 353 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 354 } else { 355 arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); 356 arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", 357 payload & SPE_OP_PKT_COND ? 358 "COND-SELECT" : "INSN-OTHER"); 359 } 360 break; 361 case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: 362 arm_spe_pkt_out_string(&err, &buf, &buf_len, 363 payload & 0x1 ? "ST" : "LD"); 364 365 if (SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(payload)) { 366 if (payload & SPE_OP_PKT_AT) 367 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); 368 if (payload & SPE_OP_PKT_EXCL) 369 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); 370 if (payload & SPE_OP_PKT_AR) 371 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); 372 } else if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) { 373 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); 374 } else if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) { 375 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); 376 } else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) { 377 arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); 378 } else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) { 379 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); 380 } else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) { 381 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); 382 } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) { 383 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); 384 } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) { 385 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); 386 } else if (SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(payload)) { 387 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-SME-REG"); 388 389 /* SVE effective vector length */ 390 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 391 SPE_OP_PKG_SVE_EVL(payload)); 392 393 if (payload & SPE_OP_PKT_SVE_PRED) 394 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 395 if (payload & SPE_OP_PKT_SVE_SG) 396 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); 397 } else if (SPE_OP_PKT_LDST_SUBCLASS_GCS(payload)) { 398 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); 399 if (payload & SPE_OP_PKT_GCS_COMM) 400 arm_spe_pkt_out_string(&err, &buf, &buf_len, " COMM"); 401 } 402 break; 403 case SPE_OP_PKT_HDR_CLASS_BR_ERET: 404 arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); 405 406 if (payload & SPE_OP_PKT_COND) 407 arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); 408 if (payload & SPE_OP_PKT_INDIRECT_BRANCH) 409 arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); 410 if (payload & SPE_OP_PKT_GCS) 411 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); 412 if (SPE_OP_PKT_CR_BL(payload)) 413 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-BL"); 414 if (SPE_OP_PKT_CR_RET(payload)) 415 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-RET"); 416 if (SPE_OP_PKT_CR_NON_BL_RET(payload)) 417 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-NON-BL-RET"); 418 break; 419 default: 420 /* Unknown index */ 421 err = -1; 422 break; 423 } 424 425 return err; 426 } 427 428 static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, 429 char *buf, size_t buf_len) 430 { 431 int ns, el, idx = packet->index; 432 int ch, pat; 433 u64 payload = packet->payload; 434 int err = 0; 435 static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; 436 437 switch (idx) { 438 case SPE_ADDR_PKT_HDR_INDEX_INS: 439 case SPE_ADDR_PKT_HDR_INDEX_BRANCH: 440 case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: 441 ns = !!SPE_ADDR_PKT_GET_NS(payload); 442 el = SPE_ADDR_PKT_GET_EL(payload); 443 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 444 arm_spe_pkt_out_string(&err, &buf, &buf_len, 445 "%s 0x%llx el%d ns=%d", 446 idx_name[idx], payload, el, ns); 447 break; 448 case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: 449 arm_spe_pkt_out_string(&err, &buf, &buf_len, 450 "VA 0x%llx", payload); 451 break; 452 case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: 453 ns = !!SPE_ADDR_PKT_GET_NS(payload); 454 ch = !!SPE_ADDR_PKT_GET_CH(payload); 455 pat = SPE_ADDR_PKT_GET_PAT(payload); 456 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 457 arm_spe_pkt_out_string(&err, &buf, &buf_len, 458 "PA 0x%llx ns=%d ch=%d pat=%x", 459 payload, ns, ch, pat); 460 break; 461 default: 462 /* Unknown index */ 463 err = -1; 464 break; 465 } 466 467 return err; 468 } 469 470 static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, 471 char *buf, size_t buf_len) 472 { 473 u64 payload = packet->payload; 474 const char *name = arm_spe_pkt_name(packet->type); 475 int err = 0; 476 477 arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, 478 (unsigned short)payload); 479 480 switch (packet->index) { 481 case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: 482 arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); 483 break; 484 case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: 485 arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); 486 break; 487 case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: 488 arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); 489 break; 490 default: 491 break; 492 } 493 494 return err; 495 } 496 497 int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, 498 size_t buf_len) 499 { 500 int idx = packet->index; 501 unsigned long long payload = packet->payload; 502 const char *name = arm_spe_pkt_name(packet->type); 503 char *buf_orig = buf; 504 size_t blen = buf_len; 505 int err = 0; 506 507 switch (packet->type) { 508 case ARM_SPE_BAD: 509 case ARM_SPE_PAD: 510 case ARM_SPE_END: 511 arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); 512 break; 513 case ARM_SPE_EVENTS: 514 err = arm_spe_pkt_desc_event(packet, buf, buf_len); 515 break; 516 case ARM_SPE_OP_TYPE: 517 err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); 518 break; 519 case ARM_SPE_DATA_SOURCE: 520 case ARM_SPE_TIMESTAMP: 521 arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); 522 break; 523 case ARM_SPE_ADDRESS: 524 err = arm_spe_pkt_desc_addr(packet, buf, buf_len); 525 break; 526 case ARM_SPE_CONTEXT: 527 arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", 528 name, (unsigned long)payload, idx + 1); 529 break; 530 case ARM_SPE_COUNTER: 531 err = arm_spe_pkt_desc_counter(packet, buf, buf_len); 532 break; 533 default: 534 /* Unknown packet type */ 535 err = -1; 536 break; 537 } 538 539 /* Output raw data if detect any error */ 540 if (err) { 541 err = 0; 542 arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", 543 name, payload, packet->index); 544 } 545 546 return err; 547 } 548