1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Arm Statistical Profiling Extensions (SPE) support 4 * Copyright (c) 2017-2018, Arm Ltd. 5 */ 6 7 #include <stdio.h> 8 #include <string.h> 9 #include <endian.h> 10 #include <byteswap.h> 11 #include <linux/bitops.h> 12 #include <stdarg.h> 13 #include <linux/kernel.h> 14 #include <asm-generic/unaligned.h> 15 16 #include "arm-spe-pkt-decoder.h" 17 18 static const char * const arm_spe_packet_name[] = { 19 [ARM_SPE_PAD] = "PAD", 20 [ARM_SPE_END] = "END", 21 [ARM_SPE_TIMESTAMP] = "TS", 22 [ARM_SPE_ADDRESS] = "ADDR", 23 [ARM_SPE_COUNTER] = "LAT", 24 [ARM_SPE_CONTEXT] = "CONTEXT", 25 [ARM_SPE_OP_TYPE] = "OP-TYPE", 26 [ARM_SPE_EVENTS] = "EVENTS", 27 [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", 28 }; 29 30 const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) 31 { 32 return arm_spe_packet_name[type]; 33 } 34 35 /* 36 * Extracts the field "sz" from header bits and converts to bytes: 37 * 00 : byte (1) 38 * 01 : halfword (2) 39 * 10 : word (4) 40 * 11 : doubleword (8) 41 */ 42 static unsigned int arm_spe_payload_len(unsigned char hdr) 43 { 44 return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); 45 } 46 47 static int arm_spe_get_payload(const unsigned char *buf, size_t len, 48 unsigned char ext_hdr, 49 struct arm_spe_pkt *packet) 50 { 51 size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); 52 53 if (len < 1 + ext_hdr + payload_len) 54 return ARM_SPE_NEED_MORE_BYTES; 55 56 buf += 1 + ext_hdr; 57 58 switch (payload_len) { 59 case 1: packet->payload = *(uint8_t *)buf; break; 60 case 2: packet->payload = get_unaligned_le16(buf); break; 61 case 4: packet->payload = get_unaligned_le32(buf); break; 62 case 8: packet->payload = get_unaligned_le64(buf); break; 63 default: return ARM_SPE_BAD_PACKET; 64 } 65 66 return 1 + ext_hdr + payload_len; 67 } 68 69 static int arm_spe_get_pad(struct arm_spe_pkt *packet) 70 { 71 packet->type = ARM_SPE_PAD; 72 return 1; 73 } 74 75 static int arm_spe_get_alignment(const unsigned char *buf, size_t len, 76 struct arm_spe_pkt *packet) 77 { 78 unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); 79 80 if (len < alignment) 81 return ARM_SPE_NEED_MORE_BYTES; 82 83 packet->type = ARM_SPE_PAD; 84 return alignment - (((uintptr_t)buf) & (alignment - 1)); 85 } 86 87 static int arm_spe_get_end(struct arm_spe_pkt *packet) 88 { 89 packet->type = ARM_SPE_END; 90 return 1; 91 } 92 93 static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, 94 struct arm_spe_pkt *packet) 95 { 96 packet->type = ARM_SPE_TIMESTAMP; 97 return arm_spe_get_payload(buf, len, 0, packet); 98 } 99 100 static int arm_spe_get_events(const unsigned char *buf, size_t len, 101 struct arm_spe_pkt *packet) 102 { 103 packet->type = ARM_SPE_EVENTS; 104 105 /* we use index to identify Events with a less number of 106 * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, 107 * LLC-REFILL, and REMOTE-ACCESS events are identified if 108 * index > 1. 109 */ 110 packet->index = arm_spe_payload_len(buf[0]); 111 112 return arm_spe_get_payload(buf, len, 0, packet); 113 } 114 115 static int arm_spe_get_data_source(const unsigned char *buf, size_t len, 116 struct arm_spe_pkt *packet) 117 { 118 packet->type = ARM_SPE_DATA_SOURCE; 119 return arm_spe_get_payload(buf, len, 0, packet); 120 } 121 122 static int arm_spe_get_context(const unsigned char *buf, size_t len, 123 struct arm_spe_pkt *packet) 124 { 125 packet->type = ARM_SPE_CONTEXT; 126 packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); 127 return arm_spe_get_payload(buf, len, 0, packet); 128 } 129 130 static int arm_spe_get_op_type(const unsigned char *buf, size_t len, 131 struct arm_spe_pkt *packet) 132 { 133 packet->type = ARM_SPE_OP_TYPE; 134 packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); 135 return arm_spe_get_payload(buf, len, 0, packet); 136 } 137 138 static int arm_spe_get_counter(const unsigned char *buf, size_t len, 139 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 140 { 141 packet->type = ARM_SPE_COUNTER; 142 143 if (ext_hdr) 144 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 145 else 146 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 147 148 return arm_spe_get_payload(buf, len, ext_hdr, packet); 149 } 150 151 static int arm_spe_get_addr(const unsigned char *buf, size_t len, 152 const unsigned char ext_hdr, struct arm_spe_pkt *packet) 153 { 154 packet->type = ARM_SPE_ADDRESS; 155 156 if (ext_hdr) 157 packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); 158 else 159 packet->index = SPE_HDR_SHORT_INDEX(buf[0]); 160 161 return arm_spe_get_payload(buf, len, ext_hdr, packet); 162 } 163 164 static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, 165 struct arm_spe_pkt *packet) 166 { 167 unsigned int hdr; 168 unsigned char ext_hdr = 0; 169 170 memset(packet, 0, sizeof(struct arm_spe_pkt)); 171 172 if (!len) 173 return ARM_SPE_NEED_MORE_BYTES; 174 175 hdr = buf[0]; 176 177 if (hdr == SPE_HEADER0_PAD) 178 return arm_spe_get_pad(packet); 179 180 if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ 181 return arm_spe_get_end(packet); 182 183 if (hdr == SPE_HEADER0_TIMESTAMP) 184 return arm_spe_get_timestamp(buf, len, packet); 185 186 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) 187 return arm_spe_get_events(buf, len, packet); 188 189 if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) 190 return arm_spe_get_data_source(buf, len, packet); 191 192 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) 193 return arm_spe_get_context(buf, len, packet); 194 195 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) 196 return arm_spe_get_op_type(buf, len, packet); 197 198 if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { 199 /* 16-bit extended format header */ 200 if (len == 1) 201 return ARM_SPE_BAD_PACKET; 202 203 ext_hdr = 1; 204 hdr = buf[1]; 205 if (hdr == SPE_HEADER1_ALIGNMENT) 206 return arm_spe_get_alignment(buf, len, packet); 207 } 208 209 /* 210 * The short format header's byte 0 or the extended format header's 211 * byte 1 has been assigned to 'hdr', which uses the same encoding for 212 * address packet and counter packet, so don't need to distinguish if 213 * it's short format or extended format and handle in once. 214 */ 215 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) 216 return arm_spe_get_addr(buf, len, ext_hdr, packet); 217 218 if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) 219 return arm_spe_get_counter(buf, len, ext_hdr, packet); 220 221 return ARM_SPE_BAD_PACKET; 222 } 223 224 int arm_spe_get_packet(const unsigned char *buf, size_t len, 225 struct arm_spe_pkt *packet) 226 { 227 int ret; 228 229 ret = arm_spe_do_get_packet(buf, len, packet); 230 /* put multiple consecutive PADs on the same line, up to 231 * the fixed-width output format of 16 bytes per line. 232 */ 233 if (ret > 0 && packet->type == ARM_SPE_PAD) { 234 while (ret < 16 && len > (size_t)ret && !buf[ret]) 235 ret += 1; 236 } 237 return ret; 238 } 239 240 static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, 241 const char *fmt, ...) 242 { 243 va_list ap; 244 int ret; 245 246 /* Bail out if any error occurred */ 247 if (err && *err) 248 return *err; 249 250 va_start(ap, fmt); 251 ret = vsnprintf(*buf_p, *blen, fmt, ap); 252 va_end(ap); 253 254 if (ret < 0) { 255 if (err && !*err) 256 *err = ret; 257 258 /* 259 * A return value of *blen or more means that the output was 260 * truncated and the buffer is overrun. 261 */ 262 } else if ((size_t)ret >= *blen) { 263 (*buf_p)[*blen - 1] = '\0'; 264 265 /* 266 * Set *err to 'ret' to avoid overflow if tries to 267 * fill this buffer sequentially. 268 */ 269 if (err && !*err) 270 *err = ret; 271 } else { 272 *buf_p += ret; 273 *blen -= ret; 274 } 275 276 return ret; 277 } 278 279 static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, 280 char *buf, size_t buf_len) 281 { 282 u64 payload = packet->payload; 283 int err = 0; 284 285 arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); 286 287 if (payload & BIT(EV_EXCEPTION_GEN)) 288 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); 289 if (payload & BIT(EV_RETIRED)) 290 arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); 291 if (payload & BIT(EV_L1D_ACCESS)) 292 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); 293 if (payload & BIT(EV_L1D_REFILL)) 294 arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); 295 if (payload & BIT(EV_TLB_ACCESS)) 296 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); 297 if (payload & BIT(EV_TLB_WALK)) 298 arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); 299 if (payload & BIT(EV_NOT_TAKEN)) 300 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); 301 if (payload & BIT(EV_MISPRED)) 302 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); 303 if (payload & BIT(EV_LLC_ACCESS)) 304 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); 305 if (payload & BIT(EV_LLC_MISS)) 306 arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); 307 if (payload & BIT(EV_REMOTE_ACCESS)) 308 arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); 309 if (payload & BIT(EV_ALIGNMENT)) 310 arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); 311 if (payload & BIT(EV_PARTIAL_PREDICATE)) 312 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); 313 if (payload & BIT(EV_EMPTY_PREDICATE)) 314 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); 315 316 return err; 317 } 318 319 static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, 320 char *buf, size_t buf_len) 321 { 322 u64 payload = packet->payload; 323 int err = 0; 324 325 switch (packet->index) { 326 case SPE_OP_PKT_HDR_CLASS_OTHER: 327 if (SPE_OP_PKT_IS_OTHER_SVE_OP(payload)) { 328 arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); 329 330 /* SVE effective vector length */ 331 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 332 SPE_OP_PKG_SVE_EVL(payload)); 333 334 if (payload & SPE_OP_PKT_SVE_FP) 335 arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); 336 if (payload & SPE_OP_PKT_SVE_PRED) 337 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 338 } else { 339 arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); 340 arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", 341 payload & SPE_OP_PKT_COND ? 342 "COND-SELECT" : "INSN-OTHER"); 343 } 344 break; 345 case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: 346 arm_spe_pkt_out_string(&err, &buf, &buf_len, 347 payload & 0x1 ? "ST" : "LD"); 348 349 if (SPE_OP_PKT_IS_LDST_ATOMIC(payload)) { 350 if (payload & SPE_OP_PKT_AT) 351 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); 352 if (payload & SPE_OP_PKT_EXCL) 353 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); 354 if (payload & SPE_OP_PKT_AR) 355 arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); 356 } 357 358 switch (SPE_OP_PKT_LDST_SUBCLASS_GET(payload)) { 359 case SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP: 360 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); 361 break; 362 case SPE_OP_PKT_LDST_SUBCLASS_GP_REG: 363 arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); 364 break; 365 case SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG: 366 arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); 367 break; 368 case SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG: 369 arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); 370 break; 371 case SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG: 372 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); 373 break; 374 case SPE_OP_PKT_LDST_SUBCLASS_MEMCPY: 375 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); 376 break; 377 case SPE_OP_PKT_LDST_SUBCLASS_MEMSET: 378 arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); 379 break; 380 default: 381 break; 382 } 383 384 if (SPE_OP_PKT_IS_LDST_SVE(payload)) { 385 /* SVE effective vector length */ 386 arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", 387 SPE_OP_PKG_SVE_EVL(payload)); 388 389 if (payload & SPE_OP_PKT_SVE_PRED) 390 arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); 391 if (payload & SPE_OP_PKT_SVE_SG) 392 arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); 393 } 394 break; 395 case SPE_OP_PKT_HDR_CLASS_BR_ERET: 396 arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); 397 398 if (payload & SPE_OP_PKT_COND) 399 arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); 400 401 if (SPE_OP_PKT_IS_INDIRECT_BRANCH(payload)) 402 arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); 403 404 break; 405 default: 406 /* Unknown index */ 407 err = -1; 408 break; 409 } 410 411 return err; 412 } 413 414 static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, 415 char *buf, size_t buf_len) 416 { 417 int ns, el, idx = packet->index; 418 int ch, pat; 419 u64 payload = packet->payload; 420 int err = 0; 421 static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; 422 423 switch (idx) { 424 case SPE_ADDR_PKT_HDR_INDEX_INS: 425 case SPE_ADDR_PKT_HDR_INDEX_BRANCH: 426 case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: 427 ns = !!SPE_ADDR_PKT_GET_NS(payload); 428 el = SPE_ADDR_PKT_GET_EL(payload); 429 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 430 arm_spe_pkt_out_string(&err, &buf, &buf_len, 431 "%s 0x%llx el%d ns=%d", 432 idx_name[idx], payload, el, ns); 433 break; 434 case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: 435 arm_spe_pkt_out_string(&err, &buf, &buf_len, 436 "VA 0x%llx", payload); 437 break; 438 case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: 439 ns = !!SPE_ADDR_PKT_GET_NS(payload); 440 ch = !!SPE_ADDR_PKT_GET_CH(payload); 441 pat = SPE_ADDR_PKT_GET_PAT(payload); 442 payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); 443 arm_spe_pkt_out_string(&err, &buf, &buf_len, 444 "PA 0x%llx ns=%d ch=%d pat=%x", 445 payload, ns, ch, pat); 446 break; 447 default: 448 /* Unknown index */ 449 err = -1; 450 break; 451 } 452 453 return err; 454 } 455 456 static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, 457 char *buf, size_t buf_len) 458 { 459 u64 payload = packet->payload; 460 const char *name = arm_spe_pkt_name(packet->type); 461 int err = 0; 462 463 arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, 464 (unsigned short)payload); 465 466 switch (packet->index) { 467 case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: 468 arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); 469 break; 470 case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: 471 arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); 472 break; 473 case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: 474 arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); 475 break; 476 default: 477 break; 478 } 479 480 return err; 481 } 482 483 int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, 484 size_t buf_len) 485 { 486 int idx = packet->index; 487 unsigned long long payload = packet->payload; 488 const char *name = arm_spe_pkt_name(packet->type); 489 char *buf_orig = buf; 490 size_t blen = buf_len; 491 int err = 0; 492 493 switch (packet->type) { 494 case ARM_SPE_BAD: 495 case ARM_SPE_PAD: 496 case ARM_SPE_END: 497 arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); 498 break; 499 case ARM_SPE_EVENTS: 500 err = arm_spe_pkt_desc_event(packet, buf, buf_len); 501 break; 502 case ARM_SPE_OP_TYPE: 503 err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); 504 break; 505 case ARM_SPE_DATA_SOURCE: 506 case ARM_SPE_TIMESTAMP: 507 arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); 508 break; 509 case ARM_SPE_ADDRESS: 510 err = arm_spe_pkt_desc_addr(packet, buf, buf_len); 511 break; 512 case ARM_SPE_CONTEXT: 513 arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", 514 name, (unsigned long)payload, idx + 1); 515 break; 516 case ARM_SPE_COUNTER: 517 err = arm_spe_pkt_desc_counter(packet, buf, buf_len); 518 break; 519 default: 520 /* Unknown packet type */ 521 err = -1; 522 break; 523 } 524 525 /* Output raw data if detect any error */ 526 if (err) { 527 err = 0; 528 arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", 529 name, payload, packet->index); 530 } 531 532 return err; 533 } 534