1 /* 2 * Driver for Marvell NETA network controller Buffer Manager. 3 * 4 * Copyright (C) 2015 Marvell 5 * 6 * Marcin Wojtas <mw@semihalf.com> 7 * 8 * This file is licensed under the terms of the GNU General Public 9 * License version 2. This program is licensed "as is" without any 10 * warranty of any kind, whether express or implied. 11 */ 12 13 #include <linux/clk.h> 14 #include <linux/genalloc.h> 15 #include <linux/io.h> 16 #include <linux/kernel.h> 17 #include <linux/mbus.h> 18 #include <linux/module.h> 19 #include <linux/netdevice.h> 20 #include <linux/of.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/skbuff.h> 24 #include <net/hwbm.h> 25 #include "mvneta_bm.h" 26 27 #define MVNETA_BM_DRIVER_NAME "mvneta_bm" 28 #define MVNETA_BM_DRIVER_VERSION "1.0" 29 30 static void mvneta_bm_write(struct mvneta_bm *priv, u32 offset, u32 data) 31 { 32 writel(data, priv->reg_base + offset); 33 } 34 35 static u32 mvneta_bm_read(struct mvneta_bm *priv, u32 offset) 36 { 37 return readl(priv->reg_base + offset); 38 } 39 40 static void mvneta_bm_pool_enable(struct mvneta_bm *priv, int pool_id) 41 { 42 u32 val; 43 44 val = mvneta_bm_read(priv, MVNETA_BM_POOL_BASE_REG(pool_id)); 45 val |= MVNETA_BM_POOL_ENABLE_MASK; 46 mvneta_bm_write(priv, MVNETA_BM_POOL_BASE_REG(pool_id), val); 47 48 /* Clear BM cause register */ 49 mvneta_bm_write(priv, MVNETA_BM_INTR_CAUSE_REG, 0); 50 } 51 52 static void mvneta_bm_pool_disable(struct mvneta_bm *priv, int pool_id) 53 { 54 u32 val; 55 56 val = mvneta_bm_read(priv, MVNETA_BM_POOL_BASE_REG(pool_id)); 57 val &= ~MVNETA_BM_POOL_ENABLE_MASK; 58 mvneta_bm_write(priv, MVNETA_BM_POOL_BASE_REG(pool_id), val); 59 } 60 61 static inline void mvneta_bm_config_set(struct mvneta_bm *priv, u32 mask) 62 { 63 u32 val; 64 65 val = mvneta_bm_read(priv, MVNETA_BM_CONFIG_REG); 66 val |= mask; 67 mvneta_bm_write(priv, MVNETA_BM_CONFIG_REG, val); 68 } 69 70 static inline void mvneta_bm_config_clear(struct mvneta_bm *priv, u32 mask) 71 { 72 u32 val; 73 74 val = mvneta_bm_read(priv, MVNETA_BM_CONFIG_REG); 75 val &= ~mask; 76 mvneta_bm_write(priv, MVNETA_BM_CONFIG_REG, val); 77 } 78 79 static void mvneta_bm_pool_target_set(struct mvneta_bm *priv, int pool_id, 80 u8 target_id, u8 attr) 81 { 82 u32 val; 83 84 val = mvneta_bm_read(priv, MVNETA_BM_XBAR_POOL_REG(pool_id)); 85 val &= ~MVNETA_BM_TARGET_ID_MASK(pool_id); 86 val &= ~MVNETA_BM_XBAR_ATTR_MASK(pool_id); 87 val |= MVNETA_BM_TARGET_ID_VAL(pool_id, target_id); 88 val |= MVNETA_BM_XBAR_ATTR_VAL(pool_id, attr); 89 90 mvneta_bm_write(priv, MVNETA_BM_XBAR_POOL_REG(pool_id), val); 91 } 92 93 int mvneta_bm_construct(struct hwbm_pool *hwbm_pool, void *buf) 94 { 95 struct mvneta_bm_pool *bm_pool = 96 (struct mvneta_bm_pool *)hwbm_pool->priv; 97 struct mvneta_bm *priv = bm_pool->priv; 98 dma_addr_t phys_addr; 99 100 /* In order to update buf_cookie field of RX descriptor properly, 101 * BM hardware expects buf virtual address to be placed in the 102 * first four bytes of mapped buffer. 103 */ 104 *(u32 *)buf = (u32)buf; 105 phys_addr = dma_map_single(&priv->pdev->dev, buf, bm_pool->buf_size, 106 DMA_FROM_DEVICE); 107 if (unlikely(dma_mapping_error(&priv->pdev->dev, phys_addr))) 108 return -ENOMEM; 109 110 mvneta_bm_pool_put_bp(priv, bm_pool, phys_addr); 111 return 0; 112 } 113 EXPORT_SYMBOL_GPL(mvneta_bm_construct); 114 115 /* Create pool */ 116 static int mvneta_bm_pool_create(struct mvneta_bm *priv, 117 struct mvneta_bm_pool *bm_pool) 118 { 119 struct platform_device *pdev = priv->pdev; 120 u8 target_id, attr; 121 int size_bytes, err; 122 size_bytes = sizeof(u32) * bm_pool->hwbm_pool.size; 123 bm_pool->virt_addr = dma_alloc_coherent(&pdev->dev, size_bytes, 124 &bm_pool->phys_addr, 125 GFP_KERNEL); 126 if (!bm_pool->virt_addr) 127 return -ENOMEM; 128 129 if (!IS_ALIGNED((u32)bm_pool->virt_addr, MVNETA_BM_POOL_PTR_ALIGN)) { 130 dma_free_coherent(&pdev->dev, size_bytes, bm_pool->virt_addr, 131 bm_pool->phys_addr); 132 bm_pool->virt_addr = NULL; 133 dev_err(&pdev->dev, "BM pool %d is not %d bytes aligned\n", 134 bm_pool->id, MVNETA_BM_POOL_PTR_ALIGN); 135 return -ENOMEM; 136 } 137 138 err = mvebu_mbus_get_dram_win_info(bm_pool->phys_addr, &target_id, 139 &attr); 140 if (err < 0) { 141 dma_free_coherent(&pdev->dev, size_bytes, bm_pool->virt_addr, 142 bm_pool->phys_addr); 143 bm_pool->virt_addr = NULL; 144 return err; 145 } 146 147 /* Set pool address */ 148 mvneta_bm_write(priv, MVNETA_BM_POOL_BASE_REG(bm_pool->id), 149 bm_pool->phys_addr); 150 151 mvneta_bm_pool_target_set(priv, bm_pool->id, target_id, attr); 152 mvneta_bm_pool_enable(priv, bm_pool->id); 153 154 return 0; 155 } 156 157 /* Notify the driver that BM pool is being used as specific type and return the 158 * pool pointer on success 159 */ 160 struct mvneta_bm_pool *mvneta_bm_pool_use(struct mvneta_bm *priv, u8 pool_id, 161 enum mvneta_bm_type type, u8 port_id, 162 int pkt_size) 163 { 164 struct mvneta_bm_pool *new_pool = &priv->bm_pools[pool_id]; 165 int num, err; 166 167 if (new_pool->type == MVNETA_BM_LONG && 168 new_pool->port_map != 1 << port_id) { 169 dev_err(&priv->pdev->dev, 170 "long pool cannot be shared by the ports\n"); 171 return NULL; 172 } 173 174 if (new_pool->type == MVNETA_BM_SHORT && new_pool->type != type) { 175 dev_err(&priv->pdev->dev, 176 "mixing pools' types between the ports is forbidden\n"); 177 return NULL; 178 } 179 180 if (new_pool->pkt_size == 0 || type != MVNETA_BM_SHORT) 181 new_pool->pkt_size = pkt_size; 182 183 /* Allocate buffers in case BM pool hasn't been used yet */ 184 if (new_pool->type == MVNETA_BM_FREE) { 185 struct hwbm_pool *hwbm_pool = &new_pool->hwbm_pool; 186 187 new_pool->priv = priv; 188 new_pool->type = type; 189 new_pool->buf_size = MVNETA_RX_BUF_SIZE(new_pool->pkt_size); 190 hwbm_pool->frag_size = 191 SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(new_pool->pkt_size)) + 192 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 193 hwbm_pool->construct = mvneta_bm_construct; 194 hwbm_pool->priv = new_pool; 195 mutex_init(&hwbm_pool->buf_lock); 196 197 /* Create new pool */ 198 err = mvneta_bm_pool_create(priv, new_pool); 199 if (err) { 200 dev_err(&priv->pdev->dev, "fail to create pool %d\n", 201 new_pool->id); 202 return NULL; 203 } 204 205 /* Allocate buffers for this pool */ 206 num = hwbm_pool_add(hwbm_pool, hwbm_pool->size); 207 if (num != hwbm_pool->size) { 208 WARN(1, "pool %d: %d of %d allocated\n", 209 new_pool->id, num, hwbm_pool->size); 210 return NULL; 211 } 212 } 213 214 return new_pool; 215 } 216 EXPORT_SYMBOL_GPL(mvneta_bm_pool_use); 217 218 /* Free all buffers from the pool */ 219 void mvneta_bm_bufs_free(struct mvneta_bm *priv, struct mvneta_bm_pool *bm_pool, 220 u8 port_map) 221 { 222 int i; 223 224 bm_pool->port_map &= ~port_map; 225 if (bm_pool->port_map) 226 return; 227 228 mvneta_bm_config_set(priv, MVNETA_BM_EMPTY_LIMIT_MASK); 229 230 for (i = 0; i < bm_pool->hwbm_pool.buf_num; i++) { 231 dma_addr_t buf_phys_addr; 232 u32 *vaddr; 233 234 /* Get buffer physical address (indirect access) */ 235 buf_phys_addr = mvneta_bm_pool_get_bp(priv, bm_pool); 236 237 /* Work-around to the problems when destroying the pool, 238 * when it occurs that a read access to BPPI returns 0. 239 */ 240 if (buf_phys_addr == 0) 241 continue; 242 243 vaddr = phys_to_virt(buf_phys_addr); 244 if (!vaddr) 245 break; 246 247 dma_unmap_single(&priv->pdev->dev, buf_phys_addr, 248 bm_pool->buf_size, DMA_FROM_DEVICE); 249 hwbm_buf_free(&bm_pool->hwbm_pool, vaddr); 250 } 251 252 mvneta_bm_config_clear(priv, MVNETA_BM_EMPTY_LIMIT_MASK); 253 254 /* Update BM driver with number of buffers removed from pool */ 255 bm_pool->hwbm_pool.buf_num -= i; 256 } 257 EXPORT_SYMBOL_GPL(mvneta_bm_bufs_free); 258 259 /* Cleanup pool */ 260 void mvneta_bm_pool_destroy(struct mvneta_bm *priv, 261 struct mvneta_bm_pool *bm_pool, u8 port_map) 262 { 263 struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool; 264 bm_pool->port_map &= ~port_map; 265 if (bm_pool->port_map) 266 return; 267 268 bm_pool->type = MVNETA_BM_FREE; 269 270 mvneta_bm_bufs_free(priv, bm_pool, port_map); 271 if (hwbm_pool->buf_num) 272 WARN(1, "cannot free all buffers in pool %d\n", bm_pool->id); 273 274 if (bm_pool->virt_addr) { 275 dma_free_coherent(&priv->pdev->dev, 276 sizeof(u32) * hwbm_pool->size, 277 bm_pool->virt_addr, bm_pool->phys_addr); 278 bm_pool->virt_addr = NULL; 279 } 280 281 mvneta_bm_pool_disable(priv, bm_pool->id); 282 } 283 EXPORT_SYMBOL_GPL(mvneta_bm_pool_destroy); 284 285 static void mvneta_bm_pools_init(struct mvneta_bm *priv) 286 { 287 struct device_node *dn = priv->pdev->dev.of_node; 288 struct mvneta_bm_pool *bm_pool; 289 char prop[15]; 290 u32 size; 291 int i; 292 293 /* Activate BM unit */ 294 mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_START_MASK); 295 296 /* Create all pools with maximum size */ 297 for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) { 298 bm_pool = &priv->bm_pools[i]; 299 bm_pool->id = i; 300 bm_pool->type = MVNETA_BM_FREE; 301 302 /* Reset read pointer */ 303 mvneta_bm_write(priv, MVNETA_BM_POOL_READ_PTR_REG(i), 0); 304 305 /* Reset write pointer */ 306 mvneta_bm_write(priv, MVNETA_BM_POOL_WRITE_PTR_REG(i), 0); 307 308 /* Configure pool size according to DT or use default value */ 309 sprintf(prop, "pool%d,capacity", i); 310 if (of_property_read_u32(dn, prop, &size)) { 311 size = MVNETA_BM_POOL_CAP_DEF; 312 } else if (size > MVNETA_BM_POOL_CAP_MAX) { 313 dev_warn(&priv->pdev->dev, 314 "Illegal pool %d capacity %d, set to %d\n", 315 i, size, MVNETA_BM_POOL_CAP_MAX); 316 size = MVNETA_BM_POOL_CAP_MAX; 317 } else if (size < MVNETA_BM_POOL_CAP_MIN) { 318 dev_warn(&priv->pdev->dev, 319 "Illegal pool %d capacity %d, set to %d\n", 320 i, size, MVNETA_BM_POOL_CAP_MIN); 321 size = MVNETA_BM_POOL_CAP_MIN; 322 } else if (!IS_ALIGNED(size, MVNETA_BM_POOL_CAP_ALIGN)) { 323 dev_warn(&priv->pdev->dev, 324 "Illegal pool %d capacity %d, round to %d\n", 325 i, size, ALIGN(size, 326 MVNETA_BM_POOL_CAP_ALIGN)); 327 size = ALIGN(size, MVNETA_BM_POOL_CAP_ALIGN); 328 } 329 bm_pool->hwbm_pool.size = size; 330 331 mvneta_bm_write(priv, MVNETA_BM_POOL_SIZE_REG(i), 332 bm_pool->hwbm_pool.size); 333 334 /* Obtain custom pkt_size from DT */ 335 sprintf(prop, "pool%d,pkt-size", i); 336 if (of_property_read_u32(dn, prop, &bm_pool->pkt_size)) 337 bm_pool->pkt_size = 0; 338 } 339 } 340 341 static void mvneta_bm_default_set(struct mvneta_bm *priv) 342 { 343 u32 val; 344 345 /* Mask BM all interrupts */ 346 mvneta_bm_write(priv, MVNETA_BM_INTR_MASK_REG, 0); 347 348 /* Clear BM cause register */ 349 mvneta_bm_write(priv, MVNETA_BM_INTR_CAUSE_REG, 0); 350 351 /* Set BM configuration register */ 352 val = mvneta_bm_read(priv, MVNETA_BM_CONFIG_REG); 353 354 /* Reduce MaxInBurstSize from 32 BPs to 16 BPs */ 355 val &= ~MVNETA_BM_MAX_IN_BURST_SIZE_MASK; 356 val |= MVNETA_BM_MAX_IN_BURST_SIZE_16BP; 357 mvneta_bm_write(priv, MVNETA_BM_CONFIG_REG, val); 358 } 359 360 static int mvneta_bm_init(struct mvneta_bm *priv) 361 { 362 mvneta_bm_default_set(priv); 363 364 /* Allocate and initialize BM pools structures */ 365 priv->bm_pools = devm_kcalloc(&priv->pdev->dev, MVNETA_BM_POOLS_NUM, 366 sizeof(struct mvneta_bm_pool), 367 GFP_KERNEL); 368 if (!priv->bm_pools) 369 return -ENOMEM; 370 371 mvneta_bm_pools_init(priv); 372 373 return 0; 374 } 375 376 static int mvneta_bm_get_sram(struct device_node *dn, 377 struct mvneta_bm *priv) 378 { 379 priv->bppi_pool = of_gen_pool_get(dn, "internal-mem", 0); 380 if (!priv->bppi_pool) 381 return -ENOMEM; 382 383 priv->bppi_virt_addr = gen_pool_dma_alloc(priv->bppi_pool, 384 MVNETA_BM_BPPI_SIZE, 385 &priv->bppi_phys_addr); 386 if (!priv->bppi_virt_addr) 387 return -ENOMEM; 388 389 return 0; 390 } 391 392 static void mvneta_bm_put_sram(struct mvneta_bm *priv) 393 { 394 gen_pool_free(priv->bppi_pool, priv->bppi_phys_addr, 395 MVNETA_BM_BPPI_SIZE); 396 } 397 398 struct mvneta_bm *mvneta_bm_get(struct device_node *node) 399 { 400 struct platform_device *pdev; 401 struct mvneta_bm *priv; 402 403 pdev = of_find_device_by_node(node); 404 if (!pdev) 405 return NULL; 406 407 priv = platform_get_drvdata(pdev); 408 if (!priv) { 409 platform_device_put(pdev); 410 return NULL; 411 } 412 413 return priv; 414 } 415 EXPORT_SYMBOL_GPL(mvneta_bm_get); 416 417 void mvneta_bm_put(struct mvneta_bm *priv) 418 { 419 platform_device_put(priv->pdev); 420 } 421 EXPORT_SYMBOL_GPL(mvneta_bm_put); 422 423 static int mvneta_bm_probe(struct platform_device *pdev) 424 { 425 struct device_node *dn = pdev->dev.of_node; 426 struct mvneta_bm *priv; 427 int err; 428 429 priv = devm_kzalloc(&pdev->dev, sizeof(struct mvneta_bm), GFP_KERNEL); 430 if (!priv) 431 return -ENOMEM; 432 433 priv->reg_base = devm_platform_ioremap_resource(pdev, 0); 434 if (IS_ERR(priv->reg_base)) 435 return PTR_ERR(priv->reg_base); 436 437 priv->clk = devm_clk_get(&pdev->dev, NULL); 438 if (IS_ERR(priv->clk)) 439 return PTR_ERR(priv->clk); 440 err = clk_prepare_enable(priv->clk); 441 if (err < 0) 442 return err; 443 444 err = mvneta_bm_get_sram(dn, priv); 445 if (err < 0) { 446 dev_err(&pdev->dev, "failed to allocate internal memory\n"); 447 goto err_clk; 448 } 449 450 priv->pdev = pdev; 451 452 /* Initialize buffer manager internals */ 453 err = mvneta_bm_init(priv); 454 if (err < 0) { 455 dev_err(&pdev->dev, "failed to initialize controller\n"); 456 goto err_sram; 457 } 458 459 dn->data = priv; 460 platform_set_drvdata(pdev, priv); 461 462 dev_info(&pdev->dev, "Buffer Manager for network controller enabled\n"); 463 464 return 0; 465 466 err_sram: 467 mvneta_bm_put_sram(priv); 468 err_clk: 469 clk_disable_unprepare(priv->clk); 470 return err; 471 } 472 473 static void mvneta_bm_remove(struct platform_device *pdev) 474 { 475 struct mvneta_bm *priv = platform_get_drvdata(pdev); 476 u8 all_ports_map = 0xff; 477 int i = 0; 478 479 for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) { 480 struct mvneta_bm_pool *bm_pool = &priv->bm_pools[i]; 481 482 mvneta_bm_pool_destroy(priv, bm_pool, all_ports_map); 483 } 484 485 mvneta_bm_put_sram(priv); 486 487 /* Dectivate BM unit */ 488 mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_STOP_MASK); 489 490 clk_disable_unprepare(priv->clk); 491 } 492 493 static int mvneta_bm_suspend(struct device *dev) 494 { 495 struct mvneta_bm *priv = dev_get_drvdata(dev); 496 int i; 497 498 /* Drain buffers and free pool resources while BM is still clocked */ 499 for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) { 500 struct mvneta_bm_pool *bm_pool = &priv->bm_pools[i]; 501 int size_bytes; 502 503 if (bm_pool->type == MVNETA_BM_FREE) 504 continue; 505 506 mvneta_bm_bufs_free(priv, bm_pool, bm_pool->port_map); 507 if (bm_pool->hwbm_pool.buf_num) 508 dev_warn(&priv->pdev->dev, 509 "pool %d: %d buffers not freed\n", 510 bm_pool->id, bm_pool->hwbm_pool.buf_num); 511 512 mvneta_bm_pool_disable(priv, bm_pool->id); 513 514 if (bm_pool->virt_addr) { 515 size_bytes = sizeof(u32) * bm_pool->hwbm_pool.size; 516 dma_free_coherent(&priv->pdev->dev, size_bytes, 517 bm_pool->virt_addr, 518 bm_pool->phys_addr); 519 bm_pool->virt_addr = NULL; 520 } 521 /* 522 * Safe to destroy: device_link guarantees all mvneta ports 523 * have already suspended, so no hwbm_pool_add() can be in 524 * progress holding buf_lock. Pairs with mutex_init() in 525 * mvneta_bm_pool_use() on resume. 526 */ 527 mutex_destroy(&bm_pool->hwbm_pool.buf_lock); 528 bm_pool->type = MVNETA_BM_FREE; 529 } 530 531 mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_STOP_MASK); 532 clk_disable_unprepare(priv->clk); 533 return 0; 534 } 535 536 static int mvneta_bm_resume(struct device *dev) 537 { 538 struct mvneta_bm *priv = dev_get_drvdata(dev); 539 int i, err; 540 541 err = clk_prepare_enable(priv->clk); 542 if (err) 543 return err; 544 545 /* Reinitialize BM hardware; pools are refilled by mvneta_resume() */ 546 mvneta_bm_default_set(priv); 547 548 /* Restore pool registers lost during clock gating */ 549 for (i = 0; i < MVNETA_BM_POOLS_NUM; i++) { 550 mvneta_bm_write(priv, MVNETA_BM_POOL_READ_PTR_REG(i), 0); 551 mvneta_bm_write(priv, MVNETA_BM_POOL_WRITE_PTR_REG(i), 0); 552 mvneta_bm_write(priv, MVNETA_BM_POOL_SIZE_REG(i), 553 priv->bm_pools[i].hwbm_pool.size); 554 } 555 556 mvneta_bm_write(priv, MVNETA_BM_COMMAND_REG, MVNETA_BM_START_MASK); 557 return 0; 558 } 559 560 static DEFINE_SIMPLE_DEV_PM_OPS(mvneta_bm_pm_ops, mvneta_bm_suspend, mvneta_bm_resume); 561 562 static const struct of_device_id mvneta_bm_match[] = { 563 { .compatible = "marvell,armada-380-neta-bm" }, 564 { } 565 }; 566 MODULE_DEVICE_TABLE(of, mvneta_bm_match); 567 568 static struct platform_driver mvneta_bm_driver = { 569 .probe = mvneta_bm_probe, 570 .remove = mvneta_bm_remove, 571 .driver = { 572 .name = MVNETA_BM_DRIVER_NAME, 573 .of_match_table = mvneta_bm_match, 574 .pm = pm_sleep_ptr(&mvneta_bm_pm_ops), 575 }, 576 }; 577 578 module_platform_driver(mvneta_bm_driver); 579 580 MODULE_DESCRIPTION("Marvell NETA Buffer Manager Driver - www.marvell.com"); 581 MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>"); 582 MODULE_LICENSE("GPL v2"); 583