1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <stdio.h> 8 #include <string.h> 9 #include <endian.h> 10 #include <byteswap.h> 11 #include <linux/bitops.h> 12 #include <stdarg.h> 13 #include <linux/kernel.h> 14 #include <linux/unaligned.h> 15 16 #include "arm-spe-pkt-decoder.h" 17 18 static const char * const arm_spe_packet_name[] = { 19 [ARM_SPE_PAD] = "PAD", 20 [ARM_SPE_END] = "END", 21 [ARM_SPE_TIMESTAMP] = "TS", 22 [ARM_SPE_ADDRESS] = "ADDR", 23 [ARM_SPE_COUNTER] = "LAT", 24 [ARM_SPE_CONTEXT] = "CONTEXT", 25 [ARM_SPE_OP_TYPE] = "OP-TYPE", 26 [ARM_SPE_EVENTS] = "EVENTS", 27 [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", 28 }; 29 30 const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) 31 { 32 return arm_spe_packet_name[type]; 33 } 34 35 /* 36 * Extracts the field "sz" from header bits and converts to bytes: 37 * 00 : byte (1) 38 * 01 : halfword (2) 39 * 10 : word (4) 40 * 11 : doubleword (8) 41 */ 42 static unsigned int arm_spe_payload_len(unsigned char hdr) 43 { 44 return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); 45 } 46 47 static int arm_spe_get_payload(const unsigned char *buf, size_t len, 48 unsigned char ext_hdr, 49 struct arm_spe_pkt *packet) 50 { 51 size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); 52 53 if (len < 1 + ext_hdr + payload_len) 54 return ARM_SPE_NEED_MORE_BYTES; 55 56 buf += 1 + ext_hdr; 57 58 switch (payload_len) { 59 case 1: packet->payload = *(uint8_t *)buf; break; 60 case 2: packet->payload = get_unaligned_le16(buf); break; 61 case 4: packet->payload = get_unaligned_le32(buf); break; 62 case 8: packet->payload = get_unaligned_le64(buf); break; 63 default: return ARM_SPE_BAD_PACKET; 64 } 65 66 return 1 + ext_hdr + payload_len; 67 } 68 69 static int arm_spe_get_pad(struct arm_spe_pkt *packet) 70 { 71 packet->type = ARM_SPE_PAD; 72 return 1; 73 } 74 75 static int arm_spe_get_alignment(const unsigned char *buf, size_t len, 76 struct arm_spe_pkt *packet) 77 { 78 unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); 79 80 if (len < alignment) 81 return ARM_SPE_NEED_MORE_BYTES; 82 83 packet->type = ARM_SPE_PAD; 84 return alignment - (((uintptr_t)buf) & (alignment - 1)); 85 } 86 87 static int arm_spe_get_end(struct arm_spe_pkt *packet) 88 { 89 packet->type = ARM_SPE_END; 90 return 1; 91 } 92 93 static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, 94 struct arm_spe_pkt *packet) 95 { 96 packet->type = ARM_SPE_TIMESTAMP; 97 return arm_spe_get_payload(buf, len, 0, packet); 98 } 99 100 static int arm_spe_get_events(const unsigned char *buf, size_t len, 101 struct arm_spe_pkt *packet) 102 { 103 packet->type = ARM_SPE_EVENTS; 104 105 /* we use index to identify Events with a less number of 106 * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, 107 * LLC-REFILL, and REMOTE-ACCESS events are identified if 108 * index > 1. 109 */ 110 packet->index = arm_spe_payload_len(buf[0]); 111 112 return arm_spe_get_payload(buf, len, 0, packet); 113 } 114 115 static int arm_spe_get_data_source(const unsigned char *buf, size_t len, 116 struct arm_spe_pkt *packet) 117 { 118 packet->type = ARM_SPE_DATA_SOURCE; 119 return arm_spe_get_payload(buf, len, 0, packet); 120 } 121 122 static int arm_spe_get_context(const unsigned char *buf, size_t len, 123 struct arm_spe_pkt *packet) 124 { 125 packet->type = ARM_SPE_CONTEXT; 126 packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); 127 return arm_spe_get_payload(buf, len, 0, packet); 128 } 129 130 static int arm_spe_get_op_type(const unsigned char *buf, size_t len, 131 struct arm_spe_pkt *packet) 132 { 133 packet->type = ARM_SPE_OP_TYPE; 134 packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); 135 return arm_spe_get_payload(buf, len, 0, packet); 136 } 137 138 static int arm_spe_get_counter(const unsigned char *buf, size_t len, 139 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 140 { 141 packet->type = ARM_SPE_COUNTER; 142 143 if (ext_hdr) 144 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 145 else 146 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 147 148 return arm_spe_get_payload(buf, len, ext_hdr, packet); 149 } 150 151 static int arm_spe_get_addr(const unsigned char *buf, size_t len, 152 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 153 { 154 packet->type = ARM_SPE_ADDRESS; 155 156 if (ext_hdr) 157 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 158 else 159 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 160 161 return arm_spe_get_payload(buf, len, ext_hdr, packet); 162 } 163 164 static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, 165 struct arm_spe_pkt *packet) 166 { 167 unsigned int hdr; 168 unsigned char ext_hdr = 0; 169 170 memset(packet, 0, sizeof(struct arm_spe_pkt)); 171 172 if (!len) 173 return ARM_SPE_NEED_MORE_BYTES; 174 175 hdr = buf[0]; 176 177 if (hdr == SPE_HEADER0_PAD) 178 return arm_spe_get_pad(packet); 179 180 if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ 181 return arm_spe_get_end(packet); 182 183 if (hdr == SPE_HEADER0_TIMESTAMP) 184 return arm_spe_get_timestamp(buf, len, packet); 185 186 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) 187 return arm_spe_get_events(buf, len, packet); 188 189 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) 190 return arm_spe_get_data_source(buf, len, packet); 191 192 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) 193 return arm_spe_get_context(buf, len, packet); 194 195 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) 196 return arm_spe_get_op_type(buf, len, packet); 197 198 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { 199 /* 16-bit extended format header */ 200 if (len == 1) 201 return ARM_SPE_BAD_PACKET; 202 203 ext_hdr = 1; 204 hdr = buf[1]; 205 if (hdr == SPE_HEADER1_ALIGNMENT) 206 return arm_spe_get_alignment(buf, len, packet); 207 } 208 209 /* 210 * The short format header's byte 0 or the extended format header's 211 * byte 1 has been assigned to 'hdr', which uses the same encoding for 212 * address packet and counter packet, so don't need to distinguish if 213 * it's short format or extended format and handle in once. 214 */ 215 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) 216 return arm_spe_get_addr(buf, len, ext_hdr, packet); 217 218 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) 219 return arm_spe_get_counter(buf, len, ext_hdr, packet); 220 221 return ARM_SPE_BAD_PACKET; 222 } 223 224 int arm_spe_get_packet(const unsigned char *buf, size_t len, 225 struct arm_spe_pkt *packet) 226 { 227 int ret; 228 229 ret = arm_spe_do_get_packet(buf, len, packet); 230 /* put multiple consecutive PADs on the same line, up to 231 * the fixed-width output format of 16 bytes per line. 232 */ 233 if (ret > 0 && packet->type == ARM_SPE_PAD) { 234 while (ret < 16 && len > (size_t)ret && !buf[ret]) 235 ret += 1; 236 } 237 return ret; 238 } 239 240 static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, 241 const char *fmt, ...) 242 { 243 va_list ap; 244 int ret; 245 246 /* Bail out if any error occurred */ 247 if (err && *err) 248 return *err; 249 250 va_start(ap, fmt); 251 ret = vsnprintf(*buf_p, *blen, fmt, ap); 252 va_end(ap); 253 254 if (ret < 0) { 255 if (err && !*err) 256 *err = ret; 257 258 /* 259 * A return value of *blen or more means that the output was 260 * truncated and the buffer is overrun. 261 */ 262 } else if ((size_t)ret >= *blen) { 263 (*buf_p)[*blen - 1] = '\0'; 264 265 /* 266 * Set *err to 'ret' to avoid overflow if tries to 267 * fill this buffer sequentially. 268 */ 269 if (err && !*err) 270 *err = ret; 271 } else { 272 *buf_p += ret; 273 *blen -= ret; 274 } 275 276 return ret; 277 } 278 279 static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, 280 char *buf, size_t buf_len) 281 { 282 u64 payload = packet->payload; 283 int err = 0; 284 285 arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); 286 287 if (payload & BIT(EV_EXCEPTION_GEN)) 288 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); 289 if (payload & BIT(EV_RETIRED)) 290 arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); 291 if (payload & BIT(EV_L1D_ACCESS)) 292 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); 293 if (payload & BIT(EV_L1D_REFILL)) 294 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); 295 if (payload & BIT(EV_TLB_ACCESS)) 296 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); 297 if (payload & BIT(EV_TLB_WALK)) 298 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); 299 if (payload & BIT(EV_NOT_TAKEN)) 300 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); 301 if (payload & BIT(EV_MISPRED)) 302 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); 303 if (payload & BIT(EV_LLC_ACCESS)) 304 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); 305 if (payload & BIT(EV_LLC_MISS)) 306 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); 307 if (payload & BIT(EV_REMOTE_ACCESS)) 308 arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); 309 if (payload & BIT(EV_ALIGNMENT)) 310 arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); 311 if (payload & BIT(EV_TRANSACTIONAL)) 312 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TXN"); 313 if (payload & BIT(EV_PARTIAL_PREDICATE)) 314 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); 315 if (payload & BIT(EV_EMPTY_PREDICATE)) 316 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); 317 if (payload & BIT(EV_L2D_ACCESS)) 318 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-ACCESS"); 319 if (payload & BIT(EV_L2D_MISS)) 320 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-MISS"); 321 if (payload & BIT(EV_CACHE_DATA_MODIFIED)) 322 arm_spe_pkt_out_string(&err, &buf, &buf_len, " HITM"); 323 if (payload & BIT(EV_RECENTLY_FETCHED)) 324 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LFB"); 325 if (payload & BIT(EV_DATA_SNOOPED)) 326 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SNOOPED"); 327 if (payload & BIT(EV_STREAMING_SVE_MODE)) 328 arm_spe_pkt_out_string(&err, &buf, &buf_len, " STREAMING-SVE"); 329 if (payload & BIT(EV_SMCU)) 330 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SMCU"); 331 332 return err; 333 } 334 335 static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, 336 char *buf, size_t buf_len) 337 { 338 u64 payload = packet->payload; 339 int err = 0; 340 341 switch (packet->index) { 342 case SPE_OP_PKT_HDR_CLASS_OTHER: 343 if (SPE_OP_PKT_IS_OTHER_SVE_OP(payload)) { 344 arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); 345 346 /* SVE effective vector length */ 347 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 348 SPE_OP_PKG_SVE_EVL(payload)); 349 350 if (payload & SPE_OP_PKT_SVE_FP) 351 arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); 352 if (payload & SPE_OP_PKT_SVE_PRED) 353 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 354 } else { 355 arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); 356 arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", 357 payload & SPE_OP_PKT_COND ? 358 "COND-SELECT" : "INSN-OTHER"); 359 } 360 break; 361 case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: 362 arm_spe_pkt_out_string(&err, &buf, &buf_len, 363 payload & 0x1 ? "ST" : "LD"); 364 365 if (SPE_OP_PKT_IS_LDST_ATOMIC(payload)) { 366 if (payload & SPE_OP_PKT_AT) 367 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); 368 if (payload & SPE_OP_PKT_EXCL) 369 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); 370 if (payload & SPE_OP_PKT_AR) 371 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); 372 } 373 374 switch (SPE_OP_PKT_LDST_SUBCLASS_GET(payload)) { 375 case SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP: 376 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); 377 break; 378 case SPE_OP_PKT_LDST_SUBCLASS_GP_REG: 379 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); 380 break; 381 case SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG: 382 arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); 383 break; 384 case SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG: 385 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); 386 break; 387 case SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG: 388 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); 389 break; 390 case SPE_OP_PKT_LDST_SUBCLASS_MEMCPY: 391 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); 392 break; 393 case SPE_OP_PKT_LDST_SUBCLASS_MEMSET: 394 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); 395 break; 396 default: 397 break; 398 } 399 400 if (SPE_OP_PKT_IS_LDST_SVE(payload)) { 401 /* SVE effective vector length */ 402 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 403 SPE_OP_PKG_SVE_EVL(payload)); 404 405 if (payload & SPE_OP_PKT_SVE_PRED) 406 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 407 if (payload & SPE_OP_PKT_SVE_SG) 408 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); 409 } 410 break; 411 case SPE_OP_PKT_HDR_CLASS_BR_ERET: 412 arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); 413 414 if (payload & SPE_OP_PKT_COND) 415 arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); 416 if (payload & SPE_OP_PKT_INDIRECT_BRANCH) 417 arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); 418 if (payload & SPE_OP_PKT_GCS) 419 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); 420 if (SPE_OP_PKT_CR_BL(payload)) 421 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-BL"); 422 if (SPE_OP_PKT_CR_RET(payload)) 423 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-RET"); 424 if (SPE_OP_PKT_CR_NON_BL_RET(payload)) 425 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-NON-BL-RET"); 426 break; 427 default: 428 /* Unknown index */ 429 err = -1; 430 break; 431 } 432 433 return err; 434 } 435 436 static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, 437 char *buf, size_t buf_len) 438 { 439 int ns, el, idx = packet->index; 440 int ch, pat; 441 u64 payload = packet->payload; 442 int err = 0; 443 static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; 444 445 switch (idx) { 446 case SPE_ADDR_PKT_HDR_INDEX_INS: 447 case SPE_ADDR_PKT_HDR_INDEX_BRANCH: 448 case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: 449 ns = !!SPE_ADDR_PKT_GET_NS(payload); 450 el = SPE_ADDR_PKT_GET_EL(payload); 451 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 452 arm_spe_pkt_out_string(&err, &buf, &buf_len, 453 "%s 0x%llx el%d ns=%d", 454 idx_name[idx], payload, el, ns); 455 break; 456 case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: 457 arm_spe_pkt_out_string(&err, &buf, &buf_len, 458 "VA 0x%llx", payload); 459 break; 460 case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: 461 ns = !!SPE_ADDR_PKT_GET_NS(payload); 462 ch = !!SPE_ADDR_PKT_GET_CH(payload); 463 pat = SPE_ADDR_PKT_GET_PAT(payload); 464 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 465 arm_spe_pkt_out_string(&err, &buf, &buf_len, 466 "PA 0x%llx ns=%d ch=%d pat=%x", 467 payload, ns, ch, pat); 468 break; 469 default: 470 /* Unknown index */ 471 err = -1; 472 break; 473 } 474 475 return err; 476 } 477 478 static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, 479 char *buf, size_t buf_len) 480 { 481 u64 payload = packet->payload; 482 const char *name = arm_spe_pkt_name(packet->type); 483 int err = 0; 484 485 arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, 486 (unsigned short)payload); 487 488 switch (packet->index) { 489 case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: 490 arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); 491 break; 492 case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: 493 arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); 494 break; 495 case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: 496 arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); 497 break; 498 default: 499 break; 500 } 501 502 return err; 503 } 504 505 int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, 506 size_t buf_len) 507 { 508 int idx = packet->index; 509 unsigned long long payload = packet->payload; 510 const char *name = arm_spe_pkt_name(packet->type); 511 char *buf_orig = buf; 512 size_t blen = buf_len; 513 int err = 0; 514 515 switch (packet->type) { 516 case ARM_SPE_BAD: 517 case ARM_SPE_PAD: 518 case ARM_SPE_END: 519 arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); 520 break; 521 case ARM_SPE_EVENTS: 522 err = arm_spe_pkt_desc_event(packet, buf, buf_len); 523 break; 524 case ARM_SPE_OP_TYPE: 525 err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); 526 break; 527 case ARM_SPE_DATA_SOURCE: 528 case ARM_SPE_TIMESTAMP: 529 arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); 530 break; 531 case ARM_SPE_ADDRESS: 532 err = arm_spe_pkt_desc_addr(packet, buf, buf_len); 533 break; 534 case ARM_SPE_CONTEXT: 535 arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", 536 name, (unsigned long)payload, idx + 1); 537 break; 538 case ARM_SPE_COUNTER: 539 err = arm_spe_pkt_desc_counter(packet, buf, buf_len); 540 break; 541 default: 542 /* Unknown packet type */ 543 err = -1; 544 break; 545 } 546 547 /* Output raw data if detect any error */ 548 if (err) { 549 err = 0; 550 arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", 551 name, payload, packet->index); 552 } 553 554 return err; 555 } 556