1 /* 2 * Copyright (c) 2014-2015 Hisilicon Limited. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 */ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/interrupt.h> 12 #include <linux/skbuff.h> 13 #include <linux/slab.h> 14 15 #include "hnae.h" 16 17 #define cls_to_ae_dev(dev) container_of(dev, struct hnae_ae_dev, cls_dev) 18 19 static struct class *hnae_class; 20 21 static void 22 hnae_list_add(spinlock_t *lock, struct list_head *node, struct list_head *head) 23 { 24 unsigned long flags; 25 26 spin_lock_irqsave(lock, flags); 27 list_add_tail_rcu(node, head); 28 spin_unlock_irqrestore(lock, flags); 29 } 30 31 static void hnae_list_del(spinlock_t *lock, struct list_head *node) 32 { 33 unsigned long flags; 34 35 spin_lock_irqsave(lock, flags); 36 list_del_rcu(node); 37 spin_unlock_irqrestore(lock, flags); 38 } 39 40 static int hnae_alloc_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb) 41 { 42 unsigned int order = hnae_page_order(ring); 43 struct page *p = dev_alloc_pages(order); 44 45 if (!p) 46 return -ENOMEM; 47 48 cb->priv = p; 49 cb->page_offset = 0; 50 cb->reuse_flag = 0; 51 cb->buf = page_address(p); 52 cb->length = hnae_page_size(ring); 53 cb->type = DESC_TYPE_PAGE; 54 55 return 0; 56 } 57 58 static void hnae_free_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb) 59 { 60 if (cb->type == DESC_TYPE_SKB) 61 dev_kfree_skb_any((struct sk_buff *)cb->priv); 62 else if (unlikely(is_rx_ring(ring))) 63 put_page((struct page *)cb->priv); 64 memset(cb, 0, sizeof(*cb)); 65 } 66 67 static int hnae_map_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb) 68 { 69 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0, 70 cb->length, ring_to_dma_dir(ring)); 71 72 if (dma_mapping_error(ring_to_dev(ring), cb->dma)) 73 return -EIO; 74 75 return 0; 76 } 77 78 static void hnae_unmap_buffer(struct hnae_ring *ring, struct hnae_desc_cb *cb) 79 { 80 if (cb->type == DESC_TYPE_SKB) 81 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length, 82 ring_to_dma_dir(ring)); 83 else 84 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length, 85 ring_to_dma_dir(ring)); 86 } 87 88 static struct hnae_buf_ops hnae_bops = { 89 .alloc_buffer = hnae_alloc_buffer, 90 .free_buffer = hnae_free_buffer, 91 .map_buffer = hnae_map_buffer, 92 .unmap_buffer = hnae_unmap_buffer, 93 }; 94 95 static int __ae_match(struct device *dev, const void *data) 96 { 97 struct hnae_ae_dev *hdev = cls_to_ae_dev(dev); 98 99 return hdev->dev->of_node == data; 100 } 101 102 static struct hnae_ae_dev *find_ae(const struct device_node *ae_node) 103 { 104 struct device *dev; 105 106 WARN_ON(!ae_node); 107 108 dev = class_find_device(hnae_class, NULL, ae_node, __ae_match); 109 110 return dev ? cls_to_ae_dev(dev) : NULL; 111 } 112 113 static void hnae_free_buffers(struct hnae_ring *ring) 114 { 115 int i; 116 117 for (i = 0; i < ring->desc_num; i++) 118 hnae_free_buffer_detach(ring, i); 119 } 120 121 /* Allocate memory for raw pkg, and map with dma */ 122 static int hnae_alloc_buffers(struct hnae_ring *ring) 123 { 124 int i, j, ret; 125 126 for (i = 0; i < ring->desc_num; i++) { 127 ret = hnae_alloc_buffer_attach(ring, i); 128 if (ret) 129 goto out_buffer_fail; 130 } 131 132 return 0; 133 134 out_buffer_fail: 135 for (j = i - 1; j >= 0; j--) 136 hnae_free_buffer_detach(ring, j); 137 return ret; 138 } 139 140 /* free desc along with its attached buffer */ 141 static void hnae_free_desc(struct hnae_ring *ring) 142 { 143 hnae_free_buffers(ring); 144 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr, 145 ring->desc_num * sizeof(ring->desc[0]), 146 ring_to_dma_dir(ring)); 147 ring->desc_dma_addr = 0; 148 kfree(ring->desc); 149 ring->desc = NULL; 150 } 151 152 /* alloc desc, without buffer attached */ 153 static int hnae_alloc_desc(struct hnae_ring *ring) 154 { 155 int size = ring->desc_num * sizeof(ring->desc[0]); 156 157 ring->desc = kzalloc(size, GFP_KERNEL); 158 if (!ring->desc) 159 return -ENOMEM; 160 161 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), 162 ring->desc, size, ring_to_dma_dir(ring)); 163 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) { 164 ring->desc_dma_addr = 0; 165 kfree(ring->desc); 166 ring->desc = NULL; 167 return -ENOMEM; 168 } 169 170 return 0; 171 } 172 173 /* fini ring, also free the buffer for the ring */ 174 static void hnae_fini_ring(struct hnae_ring *ring) 175 { 176 hnae_free_desc(ring); 177 kfree(ring->desc_cb); 178 ring->desc_cb = NULL; 179 ring->next_to_clean = 0; 180 ring->next_to_use = 0; 181 } 182 183 /* init ring, and with buffer for rx ring */ 184 static int 185 hnae_init_ring(struct hnae_queue *q, struct hnae_ring *ring, int flags) 186 { 187 int ret; 188 189 if (ring->desc_num <= 0 || ring->buf_size <= 0) 190 return -EINVAL; 191 192 ring->q = q; 193 ring->flags = flags; 194 assert(!ring->desc && !ring->desc_cb && !ring->desc_dma_addr); 195 196 /* not matter for tx or rx ring, the ntc and ntc start from 0 */ 197 assert(ring->next_to_use == 0); 198 assert(ring->next_to_clean == 0); 199 200 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]), 201 GFP_KERNEL); 202 if (!ring->desc_cb) { 203 ret = -ENOMEM; 204 goto out; 205 } 206 207 ret = hnae_alloc_desc(ring); 208 if (ret) 209 goto out_with_desc_cb; 210 211 if (is_rx_ring(ring)) { 212 ret = hnae_alloc_buffers(ring); 213 if (ret) 214 goto out_with_desc; 215 } 216 217 return 0; 218 219 out_with_desc: 220 hnae_free_desc(ring); 221 out_with_desc_cb: 222 kfree(ring->desc_cb); 223 ring->desc_cb = NULL; 224 out: 225 return ret; 226 } 227 228 static int hnae_init_queue(struct hnae_handle *h, struct hnae_queue *q, 229 struct hnae_ae_dev *dev) 230 { 231 int ret; 232 233 q->dev = dev; 234 q->handle = h; 235 236 ret = hnae_init_ring(q, &q->tx_ring, q->tx_ring.flags | RINGF_DIR); 237 if (ret) 238 goto out; 239 240 ret = hnae_init_ring(q, &q->rx_ring, q->rx_ring.flags & ~RINGF_DIR); 241 if (ret) 242 goto out_with_tx_ring; 243 244 if (dev->ops->init_queue) 245 dev->ops->init_queue(q); 246 247 return 0; 248 249 out_with_tx_ring: 250 hnae_fini_ring(&q->tx_ring); 251 out: 252 return ret; 253 } 254 255 static void hnae_fini_queue(struct hnae_queue *q) 256 { 257 if (q->dev->ops->fini_queue) 258 q->dev->ops->fini_queue(q); 259 260 hnae_fini_ring(&q->tx_ring); 261 hnae_fini_ring(&q->rx_ring); 262 } 263 264 /** 265 * ae_chain - define ae chain head 266 */ 267 static RAW_NOTIFIER_HEAD(ae_chain); 268 269 int hnae_register_notifier(struct notifier_block *nb) 270 { 271 return raw_notifier_chain_register(&ae_chain, nb); 272 } 273 EXPORT_SYMBOL(hnae_register_notifier); 274 275 void hnae_unregister_notifier(struct notifier_block *nb) 276 { 277 if (raw_notifier_chain_unregister(&ae_chain, nb)) 278 dev_err(NULL, "notifier chain unregister fail\n"); 279 } 280 EXPORT_SYMBOL(hnae_unregister_notifier); 281 282 int hnae_reinit_handle(struct hnae_handle *handle) 283 { 284 int i, j; 285 int ret; 286 287 for (i = 0; i < handle->q_num; i++) /* free ring*/ 288 hnae_fini_queue(handle->qs[i]); 289 290 if (handle->dev->ops->reset) 291 handle->dev->ops->reset(handle); 292 293 for (i = 0; i < handle->q_num; i++) {/* reinit ring*/ 294 ret = hnae_init_queue(handle, handle->qs[i], handle->dev); 295 if (ret) 296 goto out_when_init_queue; 297 } 298 return 0; 299 out_when_init_queue: 300 for (j = i - 1; j >= 0; j--) 301 hnae_fini_queue(handle->qs[j]); 302 return ret; 303 } 304 EXPORT_SYMBOL(hnae_reinit_handle); 305 306 /* hnae_get_handle - get a handle from the AE 307 * @owner_dev: the dev use this handle 308 * @ae_id: the id of the ae to be used 309 * @ae_opts: the options set for the handle 310 * @bops: the callbacks for buffer management 311 * 312 * return handle ptr or ERR_PTR 313 */ 314 struct hnae_handle *hnae_get_handle(struct device *owner_dev, 315 const struct device_node *ae_node, 316 u32 port_id, 317 struct hnae_buf_ops *bops) 318 { 319 struct hnae_ae_dev *dev; 320 struct hnae_handle *handle; 321 int i, j; 322 int ret; 323 324 dev = find_ae(ae_node); 325 if (!dev) 326 return ERR_PTR(-ENODEV); 327 328 handle = dev->ops->get_handle(dev, port_id); 329 if (IS_ERR(handle)) 330 return handle; 331 332 handle->dev = dev; 333 handle->owner_dev = owner_dev; 334 handle->bops = bops ? bops : &hnae_bops; 335 handle->eport_id = port_id; 336 337 for (i = 0; i < handle->q_num; i++) { 338 ret = hnae_init_queue(handle, handle->qs[i], dev); 339 if (ret) 340 goto out_when_init_queue; 341 } 342 343 __module_get(dev->owner); 344 345 hnae_list_add(&dev->lock, &handle->node, &dev->handle_list); 346 347 return handle; 348 349 out_when_init_queue: 350 for (j = i - 1; j >= 0; j--) 351 hnae_fini_queue(handle->qs[j]); 352 353 return ERR_PTR(-ENOMEM); 354 } 355 EXPORT_SYMBOL(hnae_get_handle); 356 357 void hnae_put_handle(struct hnae_handle *h) 358 { 359 struct hnae_ae_dev *dev = h->dev; 360 int i; 361 362 for (i = 0; i < h->q_num; i++) 363 hnae_fini_queue(h->qs[i]); 364 365 if (h->dev->ops->reset) 366 h->dev->ops->reset(h); 367 368 hnae_list_del(&dev->lock, &h->node); 369 370 if (dev->ops->put_handle) 371 dev->ops->put_handle(h); 372 373 module_put(dev->owner); 374 } 375 EXPORT_SYMBOL(hnae_put_handle); 376 377 static void hnae_release(struct device *dev) 378 { 379 } 380 381 /** 382 * hnae_ae_register - register a AE engine to hnae framework 383 * @hdev: the hnae ae engine device 384 * @owner: the module who provides this dev 385 * NOTE: the duplicated name will not be checked 386 */ 387 int hnae_ae_register(struct hnae_ae_dev *hdev, struct module *owner) 388 { 389 static atomic_t id = ATOMIC_INIT(-1); 390 int ret; 391 392 if (!hdev->dev) 393 return -ENODEV; 394 395 if (!hdev->ops || !hdev->ops->get_handle || 396 !hdev->ops->toggle_ring_irq || 397 !hdev->ops->toggle_queue_status || 398 !hdev->ops->get_status || !hdev->ops->adjust_link) 399 return -EINVAL; 400 401 hdev->owner = owner; 402 hdev->id = (int)atomic_inc_return(&id); 403 hdev->cls_dev.parent = hdev->dev; 404 hdev->cls_dev.class = hnae_class; 405 hdev->cls_dev.release = hnae_release; 406 (void)dev_set_name(&hdev->cls_dev, "hnae%d", hdev->id); 407 ret = device_register(&hdev->cls_dev); 408 if (ret) 409 return ret; 410 411 __module_get(THIS_MODULE); 412 413 INIT_LIST_HEAD(&hdev->handle_list); 414 spin_lock_init(&hdev->lock); 415 416 ret = raw_notifier_call_chain(&ae_chain, HNAE_AE_REGISTER, NULL); 417 if (ret) 418 dev_dbg(hdev->dev, 419 "has not notifier for AE: %s\n", hdev->name); 420 421 return 0; 422 } 423 EXPORT_SYMBOL(hnae_ae_register); 424 425 /** 426 * hnae_ae_unregister - unregisters a HNAE AE engine 427 * @cdev: the device to unregister 428 */ 429 void hnae_ae_unregister(struct hnae_ae_dev *hdev) 430 { 431 device_unregister(&hdev->cls_dev); 432 module_put(THIS_MODULE); 433 } 434 EXPORT_SYMBOL(hnae_ae_unregister); 435 436 static int __init hnae_init(void) 437 { 438 hnae_class = class_create(THIS_MODULE, "hnae"); 439 return PTR_ERR_OR_ZERO(hnae_class); 440 } 441 442 static void __exit hnae_exit(void) 443 { 444 class_destroy(hnae_class); 445 } 446 447 subsys_initcall(hnae_init); 448 module_exit(hnae_exit); 449 450 MODULE_AUTHOR("Hisilicon, Inc."); 451 MODULE_LICENSE("GPL"); 452 MODULE_DESCRIPTION("Hisilicon Network Acceleration Engine Framework"); 453 454 /* vi: set tw=78 noet: */ 455