1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <stdio.h> 8 #include <string.h> 9 #include <endian.h> 10 #include <byteswap.h> 11 #include <linux/bitops.h> 12 #include <stdarg.h> 13 #include <linux/kernel.h> 14 #include <linux/unaligned.h> 15 16 #include "arm-spe-pkt-decoder.h" 17 18 static const char * const arm_spe_packet_name[] = { 19 [ARM_SPE_PAD] = "PAD", 20 [ARM_SPE_END] = "END", 21 [ARM_SPE_TIMESTAMP] = "TS", 22 [ARM_SPE_ADDRESS] = "ADDR", 23 [ARM_SPE_COUNTER] = "LAT", 24 [ARM_SPE_CONTEXT] = "CONTEXT", 25 [ARM_SPE_OP_TYPE] = "OP-TYPE", 26 [ARM_SPE_EVENTS] = "EVENTS", 27 [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", 28 }; 29 30 const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) 31 { 32 return arm_spe_packet_name[type]; 33 } 34 35 /* 36 * Extracts the field "sz" from header bits and converts to bytes: 37 * 00 : byte (1) 38 * 01 : halfword (2) 39 * 10 : word (4) 40 * 11 : doubleword (8) 41 */ 42 static unsigned int arm_spe_payload_len(unsigned char hdr) 43 { 44 return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); 45 } 46 47 static int arm_spe_get_payload(const unsigned char *buf, size_t len, 48 unsigned char ext_hdr, 49 struct arm_spe_pkt *packet) 50 { 51 size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); 52 53 if (len < 1 + ext_hdr + payload_len) 54 return ARM_SPE_NEED_MORE_BYTES; 55 56 buf += 1 + ext_hdr; 57 58 switch (payload_len) { 59 case 1: packet->payload = *(uint8_t *)buf; break; 60 case 2: packet->payload = get_unaligned_le16(buf); break; 61 case 4: packet->payload = get_unaligned_le32(buf); break; 62 case 8: packet->payload = get_unaligned_le64(buf); break; 63 default: return ARM_SPE_BAD_PACKET; 64 } 65 66 return 1 + ext_hdr + payload_len; 67 } 68 69 static int arm_spe_get_pad(struct arm_spe_pkt *packet) 70 { 71 packet->type = ARM_SPE_PAD; 72 return 1; 73 } 74 75 static int arm_spe_get_alignment(const unsigned char *buf, size_t len, 76 struct arm_spe_pkt *packet) 77 { 78 unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); 79 80 if (len < alignment) 81 return ARM_SPE_NEED_MORE_BYTES; 82 83 packet->type = ARM_SPE_PAD; 84 return alignment - (((uintptr_t)buf) & (alignment - 1)); 85 } 86 87 static int arm_spe_get_end(struct arm_spe_pkt *packet) 88 { 89 packet->type = ARM_SPE_END; 90 return 1; 91 } 92 93 static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, 94 struct arm_spe_pkt *packet) 95 { 96 packet->type = ARM_SPE_TIMESTAMP; 97 return arm_spe_get_payload(buf, len, 0, packet); 98 } 99 100 static int arm_spe_get_events(const unsigned char *buf, size_t len, 101 struct arm_spe_pkt *packet) 102 { 103 packet->type = ARM_SPE_EVENTS; 104 105 /* we use index to identify Events with a less number of 106 * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, 107 * LLC-REFILL, and REMOTE-ACCESS events are identified if 108 * index > 1. 109 */ 110 packet->index = arm_spe_payload_len(buf[0]); 111 112 return arm_spe_get_payload(buf, len, 0, packet); 113 } 114 115 static int arm_spe_get_data_source(const unsigned char *buf, size_t len, 116 struct arm_spe_pkt *packet) 117 { 118 packet->type = ARM_SPE_DATA_SOURCE; 119 return arm_spe_get_payload(buf, len, 0, packet); 120 } 121 122 static int arm_spe_get_context(const unsigned char *buf, size_t len, 123 struct arm_spe_pkt *packet) 124 { 125 packet->type = ARM_SPE_CONTEXT; 126 packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); 127 return arm_spe_get_payload(buf, len, 0, packet); 128 } 129 130 static int arm_spe_get_op_type(const unsigned char *buf, size_t len, 131 struct arm_spe_pkt *packet) 132 { 133 packet->type = ARM_SPE_OP_TYPE; 134 packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); 135 return arm_spe_get_payload(buf, len, 0, packet); 136 } 137 138 static int arm_spe_get_counter(const unsigned char *buf, size_t len, 139 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 140 { 141 packet->type = ARM_SPE_COUNTER; 142 143 if (ext_hdr) 144 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 145 else 146 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 147 148 return arm_spe_get_payload(buf, len, ext_hdr, packet); 149 } 150 151 static int arm_spe_get_addr(const unsigned char *buf, size_t len, 152 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 153 { 154 packet->type = ARM_SPE_ADDRESS; 155 156 if (ext_hdr) 157 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 158 else 159 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 160 161 return arm_spe_get_payload(buf, len, ext_hdr, packet); 162 } 163 164 static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, 165 struct arm_spe_pkt *packet) 166 { 167 unsigned int hdr; 168 unsigned char ext_hdr = 0; 169 170 memset(packet, 0, sizeof(struct arm_spe_pkt)); 171 172 if (!len) 173 return ARM_SPE_NEED_MORE_BYTES; 174 175 hdr = buf[0]; 176 177 if (hdr == SPE_HEADER0_PAD) 178 return arm_spe_get_pad(packet); 179 180 if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ 181 return arm_spe_get_end(packet); 182 183 if (hdr == SPE_HEADER0_TIMESTAMP) 184 return arm_spe_get_timestamp(buf, len, packet); 185 186 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) 187 return arm_spe_get_events(buf, len, packet); 188 189 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) 190 return arm_spe_get_data_source(buf, len, packet); 191 192 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) 193 return arm_spe_get_context(buf, len, packet); 194 195 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) 196 return arm_spe_get_op_type(buf, len, packet); 197 198 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { 199 /* 16-bit extended format header */ 200 if (len == 1) 201 return ARM_SPE_BAD_PACKET; 202 203 ext_hdr = 1; 204 hdr = buf[1]; 205 if (hdr == SPE_HEADER1_ALIGNMENT) 206 return arm_spe_get_alignment(buf, len, packet); 207 } 208 209 /* 210 * The short format header's byte 0 or the extended format header's 211 * byte 1 has been assigned to 'hdr', which uses the same encoding for 212 * address packet and counter packet, so don't need to distinguish if 213 * it's short format or extended format and handle in once. 214 */ 215 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) 216 return arm_spe_get_addr(buf, len, ext_hdr, packet); 217 218 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) 219 return arm_spe_get_counter(buf, len, ext_hdr, packet); 220 221 return ARM_SPE_BAD_PACKET; 222 } 223 224 int arm_spe_get_packet(const unsigned char *buf, size_t len, 225 struct arm_spe_pkt *packet) 226 { 227 int ret; 228 229 ret = arm_spe_do_get_packet(buf, len, packet); 230 /* put multiple consecutive PADs on the same line, up to 231 * the fixed-width output format of 16 bytes per line. 232 */ 233 if (ret > 0 && packet->type == ARM_SPE_PAD) { 234 while (ret < 16 && len > (size_t)ret && !buf[ret]) 235 ret += 1; 236 } 237 return ret; 238 } 239 240 static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, 241 const char *fmt, ...) 242 { 243 va_list ap; 244 int ret; 245 246 /* Bail out if any error occurred */ 247 if (err && *err) 248 return *err; 249 250 va_start(ap, fmt); 251 ret = vsnprintf(*buf_p, *blen, fmt, ap); 252 va_end(ap); 253 254 if (ret < 0) { 255 if (err && !*err) 256 *err = ret; 257 258 /* 259 * A return value of *blen or more means that the output was 260 * truncated and the buffer is overrun. 261 */ 262 } else if ((size_t)ret >= *blen) { 263 (*buf_p)[*blen - 1] = '\0'; 264 265 /* 266 * Set *err to 'ret' to avoid overflow if tries to 267 * fill this buffer sequentially. 268 */ 269 if (err && !*err) 270 *err = ret; 271 } else { 272 *buf_p += ret; 273 *blen -= ret; 274 } 275 276 return ret; 277 } 278 279 static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, 280 char *buf, size_t buf_len) 281 { 282 u64 payload = packet->payload; 283 int err = 0; 284 285 arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); 286 287 if (payload & BIT(EV_EXCEPTION_GEN)) 288 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); 289 if (payload & BIT(EV_RETIRED)) 290 arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); 291 if (payload & BIT(EV_L1D_ACCESS)) 292 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); 293 if (payload & BIT(EV_L1D_REFILL)) 294 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); 295 if (payload & BIT(EV_TLB_ACCESS)) 296 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); 297 if (payload & BIT(EV_TLB_WALK)) 298 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); 299 if (payload & BIT(EV_NOT_TAKEN)) 300 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); 301 if (payload & BIT(EV_MISPRED)) 302 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); 303 if (payload & BIT(EV_LLC_ACCESS)) 304 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); 305 if (payload & BIT(EV_LLC_MISS)) 306 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); 307 if (payload & BIT(EV_REMOTE_ACCESS)) 308 arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); 309 if (payload & BIT(EV_ALIGNMENT)) 310 arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); 311 if (payload & BIT(EV_TRANSACTIONAL)) 312 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TXN"); 313 if (payload & BIT(EV_PARTIAL_PREDICATE)) 314 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); 315 if (payload & BIT(EV_EMPTY_PREDICATE)) 316 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); 317 318 return err; 319 } 320 321 static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, 322 char *buf, size_t buf_len) 323 { 324 u64 payload = packet->payload; 325 int err = 0; 326 327 switch (packet->index) { 328 case SPE_OP_PKT_HDR_CLASS_OTHER: 329 if (SPE_OP_PKT_IS_OTHER_SVE_OP(payload)) { 330 arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); 331 332 /* SVE effective vector length */ 333 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 334 SPE_OP_PKG_SVE_EVL(payload)); 335 336 if (payload & SPE_OP_PKT_SVE_FP) 337 arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); 338 if (payload & SPE_OP_PKT_SVE_PRED) 339 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 340 } else { 341 arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); 342 arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", 343 payload & SPE_OP_PKT_COND ? 344 "COND-SELECT" : "INSN-OTHER"); 345 } 346 break; 347 case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: 348 arm_spe_pkt_out_string(&err, &buf, &buf_len, 349 payload & 0x1 ? "ST" : "LD"); 350 351 if (SPE_OP_PKT_IS_LDST_ATOMIC(payload)) { 352 if (payload & SPE_OP_PKT_AT) 353 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); 354 if (payload & SPE_OP_PKT_EXCL) 355 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); 356 if (payload & SPE_OP_PKT_AR) 357 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); 358 } 359 360 switch (SPE_OP_PKT_LDST_SUBCLASS_GET(payload)) { 361 case SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP: 362 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); 363 break; 364 case SPE_OP_PKT_LDST_SUBCLASS_GP_REG: 365 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); 366 break; 367 case SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG: 368 arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); 369 break; 370 case SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG: 371 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); 372 break; 373 case SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG: 374 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); 375 break; 376 case SPE_OP_PKT_LDST_SUBCLASS_MEMCPY: 377 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); 378 break; 379 case SPE_OP_PKT_LDST_SUBCLASS_MEMSET: 380 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); 381 break; 382 default: 383 break; 384 } 385 386 if (SPE_OP_PKT_IS_LDST_SVE(payload)) { 387 /* SVE effective vector length */ 388 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 389 SPE_OP_PKG_SVE_EVL(payload)); 390 391 if (payload & SPE_OP_PKT_SVE_PRED) 392 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 393 if (payload & SPE_OP_PKT_SVE_SG) 394 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); 395 } 396 break; 397 case SPE_OP_PKT_HDR_CLASS_BR_ERET: 398 arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); 399 400 if (payload & SPE_OP_PKT_COND) 401 arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); 402 if (payload & SPE_OP_PKT_INDIRECT_BRANCH) 403 arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); 404 if (payload & SPE_OP_PKT_GCS) 405 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); 406 if (SPE_OP_PKT_CR_BL(payload)) 407 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-BL"); 408 if (SPE_OP_PKT_CR_RET(payload)) 409 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-RET"); 410 if (SPE_OP_PKT_CR_NON_BL_RET(payload)) 411 arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-NON-BL-RET"); 412 break; 413 default: 414 /* Unknown index */ 415 err = -1; 416 break; 417 } 418 419 return err; 420 } 421 422 static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, 423 char *buf, size_t buf_len) 424 { 425 int ns, el, idx = packet->index; 426 int ch, pat; 427 u64 payload = packet->payload; 428 int err = 0; 429 static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; 430 431 switch (idx) { 432 case SPE_ADDR_PKT_HDR_INDEX_INS: 433 case SPE_ADDR_PKT_HDR_INDEX_BRANCH: 434 case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: 435 ns = !!SPE_ADDR_PKT_GET_NS(payload); 436 el = SPE_ADDR_PKT_GET_EL(payload); 437 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 438 arm_spe_pkt_out_string(&err, &buf, &buf_len, 439 "%s 0x%llx el%d ns=%d", 440 idx_name[idx], payload, el, ns); 441 break; 442 case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: 443 arm_spe_pkt_out_string(&err, &buf, &buf_len, 444 "VA 0x%llx", payload); 445 break; 446 case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: 447 ns = !!SPE_ADDR_PKT_GET_NS(payload); 448 ch = !!SPE_ADDR_PKT_GET_CH(payload); 449 pat = SPE_ADDR_PKT_GET_PAT(payload); 450 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 451 arm_spe_pkt_out_string(&err, &buf, &buf_len, 452 "PA 0x%llx ns=%d ch=%d pat=%x", 453 payload, ns, ch, pat); 454 break; 455 default: 456 /* Unknown index */ 457 err = -1; 458 break; 459 } 460 461 return err; 462 } 463 464 static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, 465 char *buf, size_t buf_len) 466 { 467 u64 payload = packet->payload; 468 const char *name = arm_spe_pkt_name(packet->type); 469 int err = 0; 470 471 arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, 472 (unsigned short)payload); 473 474 switch (packet->index) { 475 case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: 476 arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); 477 break; 478 case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: 479 arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); 480 break; 481 case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: 482 arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); 483 break; 484 default: 485 break; 486 } 487 488 return err; 489 } 490 491 int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, 492 size_t buf_len) 493 { 494 int idx = packet->index; 495 unsigned long long payload = packet->payload; 496 const char *name = arm_spe_pkt_name(packet->type); 497 char *buf_orig = buf; 498 size_t blen = buf_len; 499 int err = 0; 500 501 switch (packet->type) { 502 case ARM_SPE_BAD: 503 case ARM_SPE_PAD: 504 case ARM_SPE_END: 505 arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); 506 break; 507 case ARM_SPE_EVENTS: 508 err = arm_spe_pkt_desc_event(packet, buf, buf_len); 509 break; 510 case ARM_SPE_OP_TYPE: 511 err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); 512 break; 513 case ARM_SPE_DATA_SOURCE: 514 case ARM_SPE_TIMESTAMP: 515 arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); 516 break; 517 case ARM_SPE_ADDRESS: 518 err = arm_spe_pkt_desc_addr(packet, buf, buf_len); 519 break; 520 case ARM_SPE_CONTEXT: 521 arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", 522 name, (unsigned long)payload, idx + 1); 523 break; 524 case ARM_SPE_COUNTER: 525 err = arm_spe_pkt_desc_counter(packet, buf, buf_len); 526 break; 527 default: 528 /* Unknown packet type */ 529 err = -1; 530 break; 531 } 532 533 /* Output raw data if detect any error */ 534 if (err) { 535 err = 0; 536 arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", 537 name, payload, packet->index); 538 } 539 540 return err; 541 } 542