1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2013 Freescale Semiconductor, Inc. 5 */ 6 7 #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ 8 9 #include "fsl_pamu.h" 10 11 #include <linux/cleanup.h> 12 #include <linux/fsl/guts.h> 13 #include <linux/interrupt.h> 14 #include <linux/genalloc.h> 15 #include <linux/of_address.h> 16 #include <linux/of_irq.h> 17 #include <linux/platform_device.h> 18 19 #include <asm/mpc85xx.h> 20 21 /* define indexes for each operation mapping scenario */ 22 #define OMI_QMAN 0x00 23 #define OMI_FMAN 0x01 24 #define OMI_QMAN_PRIV 0x02 25 #define OMI_CAAM 0x03 26 27 #define make64(high, low) (((u64)(high) << 32) | (low)) 28 29 struct pamu_isr_data { 30 void __iomem *pamu_reg_base; /* Base address of PAMU regs */ 31 unsigned int count; /* The number of PAMUs */ 32 }; 33 34 static struct paace *ppaact; 35 static struct paace *spaact; 36 37 static bool probed; /* Has PAMU been probed? */ 38 39 /* 40 * Table for matching compatible strings, for device tree 41 * guts node, for QorIQ SOCs. 42 * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 43 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" 44 * string would be used. 45 */ 46 static const struct of_device_id guts_device_ids[] = { 47 { .compatible = "fsl,qoriq-device-config-1.0", }, 48 { .compatible = "fsl,qoriq-device-config-2.0", }, 49 {} 50 }; 51 52 /* 53 * Table for matching compatible strings, for device tree 54 * L3 cache controller node. 55 * "fsl,t4240-l3-cache-controller" corresponds to T4, 56 * "fsl,b4860-l3-cache-controller" corresponds to B4 & 57 * "fsl,p4080-l3-cache-controller" corresponds to other, 58 * SOCs. 59 */ 60 static const struct of_device_id l3_device_ids[] = { 61 { .compatible = "fsl,t4240-l3-cache-controller", }, 62 { .compatible = "fsl,b4860-l3-cache-controller", }, 63 { .compatible = "fsl,p4080-l3-cache-controller", }, 64 {} 65 }; 66 67 /* maximum subwindows permitted per liodn */ 68 static u32 max_subwindow_count; 69 70 /** 71 * pamu_get_ppaace() - Return the primary PACCE 72 * @liodn: liodn PAACT index for desired PAACE 73 * 74 * Returns the ppace pointer upon success else return 75 * null. 76 */ 77 static struct paace *pamu_get_ppaace(int liodn) 78 { 79 if (!ppaact || liodn >= PAACE_NUMBER_ENTRIES) { 80 pr_debug("PPAACT doesn't exist\n"); 81 return NULL; 82 } 83 84 return &ppaact[liodn]; 85 } 86 87 /** 88 * pamu_enable_liodn() - Set valid bit of PACCE 89 * @liodn: liodn PAACT index for desired PAACE 90 * 91 * Returns 0 upon success else error code < 0 returned 92 */ 93 int pamu_enable_liodn(int liodn) 94 { 95 struct paace *ppaace; 96 97 ppaace = pamu_get_ppaace(liodn); 98 if (!ppaace) { 99 pr_debug("Invalid primary paace entry\n"); 100 return -ENOENT; 101 } 102 103 if (!get_bf(ppaace->addr_bitfields, PPAACE_AF_WSE)) { 104 pr_debug("liodn %d not configured\n", liodn); 105 return -EINVAL; 106 } 107 108 /* Ensure that all other stores to the ppaace complete first */ 109 mb(); 110 111 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_VALID); 112 mb(); 113 114 return 0; 115 } 116 117 /** 118 * pamu_disable_liodn() - Clears valid bit of PACCE 119 * @liodn: liodn PAACT index for desired PAACE 120 * 121 * Returns 0 upon success else error code < 0 returned 122 */ 123 int pamu_disable_liodn(int liodn) 124 { 125 struct paace *ppaace; 126 127 ppaace = pamu_get_ppaace(liodn); 128 if (!ppaace) { 129 pr_debug("Invalid primary paace entry\n"); 130 return -ENOENT; 131 } 132 133 set_bf(ppaace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); 134 mb(); 135 136 return 0; 137 } 138 139 /* Derive the window size encoding for a particular PAACE entry */ 140 static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) 141 { 142 /* Bug if not a power of 2 */ 143 BUG_ON(addrspace_size & (addrspace_size - 1)); 144 145 /* window size is 2^(WSE+1) bytes */ 146 return fls64(addrspace_size) - 2; 147 } 148 149 /* 150 * Set the PAACE type as primary and set the coherency required domain 151 * attribute 152 */ 153 static void pamu_init_ppaace(struct paace *ppaace) 154 { 155 set_bf(ppaace->addr_bitfields, PAACE_AF_PT, PAACE_PT_PRIMARY); 156 157 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 158 PAACE_M_COHERENCE_REQ); 159 } 160 161 /* 162 * Function used for updating stash destination for the coressponding 163 * LIODN. 164 */ 165 int pamu_update_paace_stash(int liodn, u32 value) 166 { 167 struct paace *paace; 168 169 paace = pamu_get_ppaace(liodn); 170 if (!paace) { 171 pr_debug("Invalid liodn entry\n"); 172 return -ENOENT; 173 } 174 set_bf(paace->impl_attr, PAACE_IA_CID, value); 175 176 mb(); 177 178 return 0; 179 } 180 181 /** 182 * pamu_config_ppaace() - Sets up PPAACE entry for specified liodn 183 * 184 * @liodn: Logical IO device number 185 * @omi: Operation mapping index -- if ~omi == 0 then omi not defined 186 * @stashid: cache stash id for associated cpu -- if ~stashid == 0 then 187 * stashid not defined 188 * @prot: window permissions 189 * 190 * Returns 0 upon success else error code < 0 returned 191 */ 192 int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) 193 { 194 struct paace *ppaace; 195 196 ppaace = pamu_get_ppaace(liodn); 197 if (!ppaace) 198 return -ENOENT; 199 200 /* window size is 2^(WSE+1) bytes */ 201 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 202 map_addrspace_size_to_wse(1ULL << 36)); 203 204 pamu_init_ppaace(ppaace); 205 206 ppaace->wbah = 0; 207 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); 208 209 /* set up operation mapping if it's configured */ 210 if (omi < OME_NUMBER_ENTRIES) { 211 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 212 ppaace->op_encode.index_ot.omi = omi; 213 } else if (~omi != 0) { 214 pr_debug("bad operation mapping index: %d\n", omi); 215 return -ENODEV; 216 } 217 218 /* configure stash id */ 219 if (~stashid != 0) 220 set_bf(ppaace->impl_attr, PAACE_IA_CID, stashid); 221 222 set_bf(ppaace->impl_attr, PAACE_IA_ATM, PAACE_ATM_WINDOW_XLATE); 223 ppaace->twbah = 0; 224 set_bf(ppaace->win_bitfields, PAACE_WIN_TWBAL, 0); 225 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, prot); 226 set_bf(ppaace->impl_attr, PAACE_IA_WCE, 0); 227 set_bf(ppaace->addr_bitfields, PPAACE_AF_MW, 0); 228 mb(); 229 230 return 0; 231 } 232 233 /** 234 * get_ome_index() - Returns the index in the operation mapping table 235 * for device. 236 * @omi_index: pointer for storing the index value 237 * @dev: target device 238 * 239 */ 240 void get_ome_index(u32 *omi_index, struct device *dev) 241 { 242 if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) 243 *omi_index = OMI_QMAN; 244 if (of_device_is_compatible(dev->of_node, "fsl,qman")) 245 *omi_index = OMI_QMAN_PRIV; 246 } 247 248 /** 249 * get_stash_id - Returns stash destination id corresponding to a 250 * cache type and vcpu. 251 * @stash_dest_hint: L1, L2 or L3 252 * @vcpu: vpcu target for a particular cache type. 253 * 254 * Returs stash on success or ~(u32)0 on failure. 255 * 256 */ 257 u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) 258 { 259 const u32 *prop; 260 struct device_node *node; 261 u32 cache_level; 262 int len, found = 0; 263 int i; 264 265 /* Fastpath, exit early if L3/CPC cache is target for stashing */ 266 if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { 267 node = of_find_matching_node(NULL, l3_device_ids); 268 if (node) { 269 prop = of_get_property(node, "cache-stash-id", NULL); 270 if (!prop) { 271 pr_debug("missing cache-stash-id at %pOF\n", 272 node); 273 of_node_put(node); 274 return ~(u32)0; 275 } 276 of_node_put(node); 277 return be32_to_cpup(prop); 278 } 279 return ~(u32)0; 280 } 281 282 for_each_of_cpu_node(node) { 283 prop = of_get_property(node, "reg", &len); 284 for (i = 0; i < len / sizeof(u32); i++) { 285 if (be32_to_cpup(&prop[i]) == vcpu) { 286 found = 1; 287 goto found_cpu_node; 288 } 289 } 290 } 291 found_cpu_node: 292 293 /* find the hwnode that represents the cache */ 294 for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { 295 if (stash_dest_hint == cache_level) { 296 prop = of_get_property(node, "cache-stash-id", NULL); 297 if (!prop) { 298 pr_debug("missing cache-stash-id at %pOF\n", 299 node); 300 of_node_put(node); 301 return ~(u32)0; 302 } 303 of_node_put(node); 304 return be32_to_cpup(prop); 305 } 306 307 prop = of_get_property(node, "next-level-cache", NULL); 308 if (!prop) { 309 pr_debug("can't find next-level-cache at %pOF\n", node); 310 of_node_put(node); 311 return ~(u32)0; /* can't traverse any further */ 312 } 313 of_node_put(node); 314 315 /* advance to next node in cache hierarchy */ 316 node = of_find_node_by_phandle(*prop); 317 if (!node) { 318 pr_debug("Invalid node for cache hierarchy\n"); 319 return ~(u32)0; 320 } 321 } 322 323 pr_debug("stash dest not found for %d on vcpu %d\n", 324 stash_dest_hint, vcpu); 325 return ~(u32)0; 326 } 327 328 /* Identify if the PAACT table entry belongs to QMAN, BMAN or QMAN Portal */ 329 #define QMAN_PAACE 1 330 #define QMAN_PORTAL_PAACE 2 331 #define BMAN_PAACE 3 332 333 /* 334 * Setup operation mapping and stash destinations for QMAN and QMAN portal. 335 * Memory accesses to QMAN and BMAN private memory need not be coherent, so 336 * clear the PAACE entry coherency attribute for them. 337 */ 338 static void setup_qbman_paace(struct paace *ppaace, int paace_type) 339 { 340 switch (paace_type) { 341 case QMAN_PAACE: 342 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 343 ppaace->op_encode.index_ot.omi = OMI_QMAN_PRIV; 344 /* setup QMAN Private data stashing for the L3 cache */ 345 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 346 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 347 0); 348 break; 349 case QMAN_PORTAL_PAACE: 350 set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); 351 ppaace->op_encode.index_ot.omi = OMI_QMAN; 352 /* Set DQRR and Frame stashing for the L3 cache */ 353 set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); 354 break; 355 case BMAN_PAACE: 356 set_bf(ppaace->domain_attr.to_host.coherency_required, PAACE_DA_HOST_CR, 357 0); 358 break; 359 } 360 } 361 362 /* 363 * Setup the operation mapping table for various devices. This is a static 364 * table where each table index corresponds to a particular device. PAMU uses 365 * this table to translate device transaction to appropriate corenet 366 * transaction. 367 */ 368 static void setup_omt(struct ome *omt) 369 { 370 struct ome *ome; 371 372 /* Configure OMI_QMAN */ 373 ome = &omt[OMI_QMAN]; 374 375 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 376 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 377 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 378 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSAO; 379 380 ome->moe[IOE_DIRECT0_IDX] = EOE_VALID | EOE_LDEC; 381 ome->moe[IOE_DIRECT1_IDX] = EOE_VALID | EOE_LDECPE; 382 383 /* Configure OMI_FMAN */ 384 ome = &omt[OMI_FMAN]; 385 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 386 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 387 388 /* Configure OMI_QMAN private */ 389 ome = &omt[OMI_QMAN_PRIV]; 390 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READ; 391 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 392 ome->moe[IOE_EREAD0_IDX] = EOE_VALID | EOE_RSA; 393 ome->moe[IOE_EWRITE0_IDX] = EOE_VALID | EOE_WWSA; 394 395 /* Configure OMI_CAAM */ 396 ome = &omt[OMI_CAAM]; 397 ome->moe[IOE_READ_IDX] = EOE_VALID | EOE_READI; 398 ome->moe[IOE_WRITE_IDX] = EOE_VALID | EOE_WRITE; 399 } 400 401 /* 402 * Get the maximum number of PAACT table entries 403 * and subwindows supported by PAMU 404 */ 405 static void get_pamu_cap_values(unsigned long pamu_reg_base) 406 { 407 u32 pc_val; 408 409 pc_val = in_be32((u32 *)(pamu_reg_base + PAMU_PC3)); 410 /* Maximum number of subwindows per liodn */ 411 max_subwindow_count = 1 << (1 + PAMU_PC3_MWCE(pc_val)); 412 } 413 414 /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ 415 static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, 416 phys_addr_t ppaact_phys, phys_addr_t spaact_phys, 417 phys_addr_t omt_phys) 418 { 419 u32 *pc; 420 struct pamu_mmap_regs *pamu_regs; 421 422 pc = (u32 *) (pamu_reg_base + PAMU_PC); 423 pamu_regs = (struct pamu_mmap_regs *) 424 (pamu_reg_base + PAMU_MMAP_REGS_BASE); 425 426 /* set up pointers to corenet control blocks */ 427 428 out_be32(&pamu_regs->ppbah, upper_32_bits(ppaact_phys)); 429 out_be32(&pamu_regs->ppbal, lower_32_bits(ppaact_phys)); 430 ppaact_phys = ppaact_phys + PAACT_SIZE; 431 out_be32(&pamu_regs->pplah, upper_32_bits(ppaact_phys)); 432 out_be32(&pamu_regs->pplal, lower_32_bits(ppaact_phys)); 433 434 out_be32(&pamu_regs->spbah, upper_32_bits(spaact_phys)); 435 out_be32(&pamu_regs->spbal, lower_32_bits(spaact_phys)); 436 spaact_phys = spaact_phys + SPAACT_SIZE; 437 out_be32(&pamu_regs->splah, upper_32_bits(spaact_phys)); 438 out_be32(&pamu_regs->splal, lower_32_bits(spaact_phys)); 439 440 out_be32(&pamu_regs->obah, upper_32_bits(omt_phys)); 441 out_be32(&pamu_regs->obal, lower_32_bits(omt_phys)); 442 omt_phys = omt_phys + OMT_SIZE; 443 out_be32(&pamu_regs->olah, upper_32_bits(omt_phys)); 444 out_be32(&pamu_regs->olal, lower_32_bits(omt_phys)); 445 446 /* 447 * set PAMU enable bit, 448 * allow ppaact & omt to be cached 449 * & enable PAMU access violation interrupts. 450 */ 451 452 out_be32((u32 *)(pamu_reg_base + PAMU_PICS), 453 PAMU_ACCESS_VIOLATION_ENABLE); 454 out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); 455 return 0; 456 } 457 458 /* Enable all device LIODNS */ 459 static void setup_liodns(void) 460 { 461 int i, len; 462 struct paace *ppaace; 463 struct device_node *node = NULL; 464 const u32 *prop; 465 466 for_each_node_with_property(node, "fsl,liodn") { 467 prop = of_get_property(node, "fsl,liodn", &len); 468 for (i = 0; i < len / sizeof(u32); i++) { 469 int liodn; 470 471 liodn = be32_to_cpup(&prop[i]); 472 if (liodn >= PAACE_NUMBER_ENTRIES) { 473 pr_debug("Invalid LIODN value %d\n", liodn); 474 continue; 475 } 476 ppaace = pamu_get_ppaace(liodn); 477 pamu_init_ppaace(ppaace); 478 /* window size is 2^(WSE+1) bytes */ 479 set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, 35); 480 ppaace->wbah = 0; 481 set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); 482 set_bf(ppaace->impl_attr, PAACE_IA_ATM, 483 PAACE_ATM_NO_XLATE); 484 set_bf(ppaace->addr_bitfields, PAACE_AF_AP, 485 PAACE_AP_PERMS_ALL); 486 if (of_device_is_compatible(node, "fsl,qman-portal")) 487 setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); 488 if (of_device_is_compatible(node, "fsl,qman")) 489 setup_qbman_paace(ppaace, QMAN_PAACE); 490 if (of_device_is_compatible(node, "fsl,bman")) 491 setup_qbman_paace(ppaace, BMAN_PAACE); 492 mb(); 493 pamu_enable_liodn(liodn); 494 } 495 } 496 } 497 498 static irqreturn_t pamu_av_isr(int irq, void *arg) 499 { 500 struct pamu_isr_data *data = arg; 501 phys_addr_t phys; 502 unsigned int i, j, ret; 503 504 pr_emerg("access violation interrupt\n"); 505 506 for (i = 0; i < data->count; i++) { 507 void __iomem *p = data->pamu_reg_base + i * PAMU_OFFSET; 508 u32 pics = in_be32(p + PAMU_PICS); 509 510 if (pics & PAMU_ACCESS_VIOLATION_STAT) { 511 u32 avs1 = in_be32(p + PAMU_AVS1); 512 struct paace *paace; 513 514 pr_emerg("POES1=%08x\n", in_be32(p + PAMU_POES1)); 515 pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); 516 pr_emerg("AVS1=%08x\n", avs1); 517 pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); 518 pr_emerg("AVA=%016llx\n", 519 make64(in_be32(p + PAMU_AVAH), 520 in_be32(p + PAMU_AVAL))); 521 pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); 522 pr_emerg("POEA=%016llx\n", 523 make64(in_be32(p + PAMU_POEAH), 524 in_be32(p + PAMU_POEAL))); 525 526 phys = make64(in_be32(p + PAMU_POEAH), 527 in_be32(p + PAMU_POEAL)); 528 529 /* Assume that POEA points to a PAACE */ 530 if (phys) { 531 u32 *paace = phys_to_virt(phys); 532 533 /* Only the first four words are relevant */ 534 for (j = 0; j < 4; j++) 535 pr_emerg("PAACE[%u]=%08x\n", 536 j, in_be32(paace + j)); 537 } 538 539 /* clear access violation condition */ 540 out_be32(p + PAMU_AVS1, avs1 & PAMU_AV_MASK); 541 paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); 542 BUG_ON(!paace); 543 /* check if we got a violation for a disabled LIODN */ 544 if (!get_bf(paace->addr_bitfields, PAACE_AF_V)) { 545 /* 546 * As per hardware erratum A-003638, access 547 * violation can be reported for a disabled 548 * LIODN. If we hit that condition, disable 549 * access violation reporting. 550 */ 551 pics &= ~PAMU_ACCESS_VIOLATION_ENABLE; 552 } else { 553 /* Disable the LIODN */ 554 ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); 555 BUG_ON(ret); 556 pr_emerg("Disabling liodn %x\n", 557 avs1 >> PAMU_AVS1_LIODN_SHIFT); 558 } 559 out_be32((p + PAMU_PICS), pics); 560 } 561 } 562 563 return IRQ_HANDLED; 564 } 565 566 #define LAWAR_EN 0x80000000 567 #define LAWAR_TARGET_MASK 0x0FF00000 568 #define LAWAR_TARGET_SHIFT 20 569 #define LAWAR_SIZE_MASK 0x0000003F 570 #define LAWAR_CSDID_MASK 0x000FF000 571 #define LAWAR_CSDID_SHIFT 12 572 573 #define LAW_SIZE_4K 0xb 574 575 struct ccsr_law { 576 u32 lawbarh; /* LAWn base address high */ 577 u32 lawbarl; /* LAWn base address low */ 578 u32 lawar; /* LAWn attributes */ 579 u32 reserved; 580 }; 581 582 /* 583 * Create a coherence subdomain for a given memory block. 584 */ 585 static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) 586 { 587 struct device_node *np; 588 const __be32 *iprop; 589 void __iomem *lac = NULL; /* Local Access Control registers */ 590 struct ccsr_law __iomem *law; 591 void __iomem *ccm = NULL; 592 u32 __iomem *csdids; 593 unsigned int i, num_laws, num_csds; 594 u32 law_target = 0; 595 u32 csd_id = 0; 596 int ret = 0; 597 598 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-law"); 599 if (!np) 600 return -ENODEV; 601 602 iprop = of_get_property(np, "fsl,num-laws", NULL); 603 if (!iprop) { 604 ret = -ENODEV; 605 goto error; 606 } 607 608 num_laws = be32_to_cpup(iprop); 609 if (!num_laws) { 610 ret = -ENODEV; 611 goto error; 612 } 613 614 lac = of_iomap(np, 0); 615 if (!lac) { 616 ret = -ENODEV; 617 goto error; 618 } 619 620 /* LAW registers are at offset 0xC00 */ 621 law = lac + 0xC00; 622 623 of_node_put(np); 624 625 np = of_find_compatible_node(NULL, NULL, "fsl,corenet-cf"); 626 if (!np) { 627 ret = -ENODEV; 628 goto error; 629 } 630 631 iprop = of_get_property(np, "fsl,ccf-num-csdids", NULL); 632 if (!iprop) { 633 ret = -ENODEV; 634 goto error; 635 } 636 637 num_csds = be32_to_cpup(iprop); 638 if (!num_csds) { 639 ret = -ENODEV; 640 goto error; 641 } 642 643 ccm = of_iomap(np, 0); 644 if (!ccm) { 645 ret = -ENOMEM; 646 goto error; 647 } 648 649 /* The undocumented CSDID registers are at offset 0x600 */ 650 csdids = ccm + 0x600; 651 652 of_node_put(np); 653 np = NULL; 654 655 /* Find an unused coherence subdomain ID */ 656 for (csd_id = 0; csd_id < num_csds; csd_id++) { 657 if (!csdids[csd_id]) 658 break; 659 } 660 661 /* Store the Port ID in the (undocumented) proper CIDMRxx register */ 662 csdids[csd_id] = csd_port_id; 663 664 /* Find the DDR LAW that maps to our buffer. */ 665 for (i = 0; i < num_laws; i++) { 666 if (law[i].lawar & LAWAR_EN) { 667 phys_addr_t law_start, law_end; 668 669 law_start = make64(law[i].lawbarh, law[i].lawbarl); 670 law_end = law_start + 671 (2ULL << (law[i].lawar & LAWAR_SIZE_MASK)); 672 673 if (law_start <= phys && phys < law_end) { 674 law_target = law[i].lawar & LAWAR_TARGET_MASK; 675 break; 676 } 677 } 678 } 679 680 if (i == 0 || i == num_laws) { 681 /* This should never happen */ 682 ret = -ENOENT; 683 goto error; 684 } 685 686 /* Find a free LAW entry */ 687 while (law[--i].lawar & LAWAR_EN) { 688 if (i == 0) { 689 /* No higher priority LAW slots available */ 690 ret = -ENOENT; 691 goto error; 692 } 693 } 694 695 law[i].lawbarh = upper_32_bits(phys); 696 law[i].lawbarl = lower_32_bits(phys); 697 wmb(); 698 law[i].lawar = LAWAR_EN | law_target | (csd_id << LAWAR_CSDID_SHIFT) | 699 (LAW_SIZE_4K + get_order(size)); 700 wmb(); 701 702 error: 703 if (ccm) 704 iounmap(ccm); 705 706 if (lac) 707 iounmap(lac); 708 709 if (np) 710 of_node_put(np); 711 712 return ret; 713 } 714 715 /* 716 * Table of SVRs and the corresponding PORT_ID values. Port ID corresponds to a 717 * bit map of snoopers for a given range of memory mapped by a LAW. 718 * 719 * All future CoreNet-enabled SOCs will have this erratum(A-004510) fixed, so this 720 * table should never need to be updated. SVRs are guaranteed to be unique, so 721 * there is no worry that a future SOC will inadvertently have one of these 722 * values. 723 */ 724 static const struct { 725 u32 svr; 726 u32 port_id; 727 } port_id_map[] = { 728 {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ 729 {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ 730 {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ 731 {(SVR_P2041 << 8) | 0x11, 0xFF000000}, /* P2041 1.1 */ 732 {(SVR_P3041 << 8) | 0x10, 0xFF000000}, /* P3041 1.0 */ 733 {(SVR_P3041 << 8) | 0x11, 0xFF000000}, /* P3041 1.1 */ 734 {(SVR_P4040 << 8) | 0x20, 0xFFF80000}, /* P4040 2.0 */ 735 {(SVR_P4080 << 8) | 0x20, 0xFFF80000}, /* P4080 2.0 */ 736 {(SVR_P5010 << 8) | 0x10, 0xFC000000}, /* P5010 1.0 */ 737 {(SVR_P5010 << 8) | 0x20, 0xFC000000}, /* P5010 2.0 */ 738 {(SVR_P5020 << 8) | 0x10, 0xFC000000}, /* P5020 1.0 */ 739 {(SVR_P5021 << 8) | 0x10, 0xFF800000}, /* P5021 1.0 */ 740 {(SVR_P5040 << 8) | 0x10, 0xFF800000}, /* P5040 1.0 */ 741 }; 742 743 #define SVR_SECURITY 0x80000 /* The Security (E) bit */ 744 745 static int fsl_pamu_probe(struct platform_device *pdev) 746 { 747 struct device *dev = &pdev->dev; 748 void __iomem *pamu_regs = NULL; 749 struct ccsr_guts __iomem *guts_regs = NULL; 750 u32 pamubypenr, pamu_counter; 751 unsigned long pamu_reg_off; 752 unsigned long pamu_reg_base; 753 struct pamu_isr_data *data = NULL; 754 struct device_node *guts_node; 755 u64 size; 756 struct page *p; 757 int ret = 0; 758 int irq; 759 phys_addr_t ppaact_phys; 760 phys_addr_t spaact_phys; 761 struct ome *omt; 762 phys_addr_t omt_phys; 763 size_t mem_size = 0; 764 unsigned int order = 0; 765 u32 csd_port_id = 0; 766 unsigned i; 767 /* 768 * enumerate all PAMUs and allocate and setup PAMU tables 769 * for each of them, 770 * NOTE : All PAMUs share the same LIODN tables. 771 */ 772 773 if (WARN_ON(probed)) 774 return -EBUSY; 775 776 pamu_regs = of_iomap(dev->of_node, 0); 777 if (!pamu_regs) { 778 dev_err(dev, "ioremap of PAMU node failed\n"); 779 return -ENOMEM; 780 } 781 of_get_address(dev->of_node, 0, &size, NULL); 782 783 irq = irq_of_parse_and_map(dev->of_node, 0); 784 if (!irq) { 785 dev_warn(dev, "no interrupts listed in PAMU node\n"); 786 goto error; 787 } 788 789 data = kzalloc_obj(*data); 790 if (!data) { 791 ret = -ENOMEM; 792 goto error; 793 } 794 data->pamu_reg_base = pamu_regs; 795 data->count = size / PAMU_OFFSET; 796 797 /* The ISR needs access to the regs, so we won't iounmap them */ 798 ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); 799 if (ret < 0) { 800 dev_err(dev, "error %i installing ISR for irq %i\n", ret, irq); 801 goto error; 802 } 803 804 guts_node = of_find_matching_node(NULL, guts_device_ids); 805 if (!guts_node) { 806 dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node); 807 ret = -ENODEV; 808 goto error; 809 } 810 811 guts_regs = of_iomap(guts_node, 0); 812 of_node_put(guts_node); 813 if (!guts_regs) { 814 dev_err(dev, "ioremap of GUTS node failed\n"); 815 ret = -ENODEV; 816 goto error; 817 } 818 819 /* read in the PAMU capability registers */ 820 get_pamu_cap_values((unsigned long)pamu_regs); 821 /* 822 * To simplify the allocation of a coherency domain, we allocate the 823 * PAACT and the OMT in the same memory buffer. Unfortunately, this 824 * wastes more memory compared to allocating the buffers separately. 825 */ 826 /* Determine how much memory we need */ 827 mem_size = (PAGE_SIZE << get_order(PAACT_SIZE)) + 828 (PAGE_SIZE << get_order(SPAACT_SIZE)) + 829 (PAGE_SIZE << get_order(OMT_SIZE)); 830 order = get_order(mem_size); 831 832 p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); 833 if (!p) { 834 dev_err(dev, "unable to allocate PAACT/SPAACT/OMT block\n"); 835 ret = -ENOMEM; 836 goto error; 837 } 838 839 ppaact = page_address(p); 840 ppaact_phys = page_to_phys(p); 841 842 /* Make sure the memory is naturally aligned */ 843 if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { 844 dev_err(dev, "PAACT/OMT block is unaligned\n"); 845 ret = -ENOMEM; 846 goto error; 847 } 848 849 spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); 850 omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); 851 852 dev_dbg(dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); 853 854 /* Check to see if we need to implement the work-around on this SOC */ 855 856 /* Determine the Port ID for our coherence subdomain */ 857 for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { 858 if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { 859 csd_port_id = port_id_map[i].port_id; 860 dev_dbg(dev, "found matching SVR %08x\n", 861 port_id_map[i].svr); 862 break; 863 } 864 } 865 866 if (csd_port_id) { 867 dev_dbg(dev, "creating coherency subdomain at address %pa, size %zu, port id 0x%08x", 868 &ppaact_phys, mem_size, csd_port_id); 869 870 ret = create_csd(ppaact_phys, mem_size, csd_port_id); 871 if (ret) { 872 dev_err(dev, "could not create coherence subdomain\n"); 873 goto error; 874 } 875 } 876 877 spaact_phys = virt_to_phys(spaact); 878 omt_phys = virt_to_phys(omt); 879 880 pamubypenr = in_be32(&guts_regs->pamubypenr); 881 882 for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; 883 pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { 884 885 pamu_reg_base = (unsigned long)pamu_regs + pamu_reg_off; 886 setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, 887 spaact_phys, omt_phys); 888 /* Disable PAMU bypass for this PAMU */ 889 pamubypenr &= ~pamu_counter; 890 } 891 892 setup_omt(omt); 893 894 /* Enable all relevant PAMU(s) */ 895 out_be32(&guts_regs->pamubypenr, pamubypenr); 896 897 iounmap(guts_regs); 898 899 /* Enable DMA for the LIODNs in the device tree */ 900 901 setup_liodns(); 902 903 probed = true; 904 905 return 0; 906 907 error: 908 if (irq) 909 free_irq(irq, data); 910 911 kfree_sensitive(data); 912 913 if (pamu_regs) 914 iounmap(pamu_regs); 915 916 if (guts_regs) 917 iounmap(guts_regs); 918 919 if (ppaact) 920 free_pages((unsigned long)ppaact, order); 921 922 ppaact = NULL; 923 924 return ret; 925 } 926 927 static struct platform_driver fsl_of_pamu_driver = { 928 .driver = { 929 .name = "fsl-of-pamu", 930 }, 931 .probe = fsl_pamu_probe, 932 }; 933 934 static __init int fsl_pamu_init(void) 935 { 936 struct platform_device *pdev = NULL; 937 int ret; 938 939 /* 940 * The normal OF process calls the probe function at some 941 * indeterminate later time, after most drivers have loaded. This is 942 * too late for us, because PAMU clients (like the Qman driver) 943 * depend on PAMU being initialized early. 944 * 945 * So instead, we "manually" call our probe function by creating the 946 * platform devices ourselves. 947 */ 948 949 /* 950 * We assume that there is only one PAMU node in the device tree. A 951 * single PAMU node represents all of the PAMU devices in the SOC 952 * already. Everything else already makes that assumption, and the 953 * binding for the PAMU nodes doesn't allow for any parent-child 954 * relationships anyway. In other words, support for more than one 955 * PAMU node would require significant changes to a lot of code. 956 */ 957 958 struct device_node *np __free(device_node) = 959 of_find_compatible_node(NULL, NULL, "fsl,pamu"); 960 if (!np) { 961 pr_err("could not find a PAMU node\n"); 962 return -ENODEV; 963 } 964 965 ret = platform_driver_register(&fsl_of_pamu_driver); 966 if (ret) { 967 pr_err("could not register driver (err=%i)\n", ret); 968 return ret; 969 } 970 971 pdev = platform_device_alloc("fsl-of-pamu", 0); 972 if (!pdev) { 973 pr_err("could not allocate device %pOF\n", np); 974 ret = -ENOMEM; 975 goto error_device_alloc; 976 } 977 978 platform_device_set_of_node(pdev, np); 979 980 ret = pamu_domain_init(); 981 if (ret) 982 goto error_device_add; 983 984 ret = platform_device_add(pdev); 985 if (ret) { 986 pr_err("could not add device %pOF (err=%i)\n", np, ret); 987 goto error_device_add; 988 } 989 990 return 0; 991 992 error_device_add: 993 platform_device_put(pdev); 994 995 error_device_alloc: 996 platform_driver_unregister(&fsl_of_pamu_driver); 997 998 return ret; 999 } 1000 arch_initcall(fsl_pamu_init); 1001