dma.c (cd60cd96137f6cb3ea82cace9225626619e7a52d) dma.c (dd4645ebb7d100bb04ba38ec58b499cbe95322fa)
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 17 unchanged lines hidden (view full) ---

26#include <linux/slab.h>
27#include <linux/pci.h>
28#include <linux/interrupt.h>
29#include <linux/dmaengine.h>
30#include <linux/delay.h>
31#include <linux/dma-mapping.h>
32#include <linux/workqueue.h>
33#include <linux/prefetch.h>
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 17 unchanged lines hidden (view full) ---

26#include <linux/slab.h>
27#include <linux/pci.h>
28#include <linux/interrupt.h>
29#include <linux/dmaengine.h>
30#include <linux/delay.h>
31#include <linux/dma-mapping.h>
32#include <linux/workqueue.h>
33#include <linux/prefetch.h>
34#include <linux/sizes.h>
34#include "dma.h"
35#include "registers.h"
36#include "hw.h"
37
38#include "../dmaengine.h"
39
40static void ioat_eh(struct ioatdma_chan *ioat_chan);
41

--- 243 unchanged lines hidden (view full) ---

285
286 ioat_update_pending(ioat_chan);
287 spin_unlock_bh(&ioat_chan->prep_lock);
288
289 return cookie;
290}
291
292static struct ioat_ring_ent *
35#include "dma.h"
36#include "registers.h"
37#include "hw.h"
38
39#include "../dmaengine.h"
40
41static void ioat_eh(struct ioatdma_chan *ioat_chan);
42

--- 243 unchanged lines hidden (view full) ---

286
287 ioat_update_pending(ioat_chan);
288 spin_unlock_bh(&ioat_chan->prep_lock);
289
290 return cookie;
291}
292
293static struct ioat_ring_ent *
293ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
294ioat_alloc_ring_ent(struct dma_chan *chan, int idx, gfp_t flags)
294{
295 struct ioat_dma_descriptor *hw;
296 struct ioat_ring_ent *desc;
297 struct ioatdma_device *ioat_dma;
295{
296 struct ioat_dma_descriptor *hw;
297 struct ioat_ring_ent *desc;
298 struct ioatdma_device *ioat_dma;
299 struct ioatdma_chan *ioat_chan = to_ioat_chan(chan);
300 int chunk;
298 dma_addr_t phys;
301 dma_addr_t phys;
302 u8 *pos;
303 off_t offs;
299
300 ioat_dma = to_ioatdma_device(chan->device);
304
305 ioat_dma = to_ioatdma_device(chan->device);
301 hw = dma_pool_alloc(ioat_dma->dma_pool, flags, &phys);
302 if (!hw)
303 return NULL;
306
307 chunk = idx / IOAT_DESCS_PER_2M;
308 idx &= (IOAT_DESCS_PER_2M - 1);
309 offs = idx * IOAT_DESC_SZ;
310 pos = (u8 *)ioat_chan->descs[chunk].virt + offs;
311 phys = ioat_chan->descs[chunk].hw + offs;
312 hw = (struct ioat_dma_descriptor *)pos;
304 memset(hw, 0, sizeof(*hw));
305
306 desc = kmem_cache_zalloc(ioat_cache, flags);
313 memset(hw, 0, sizeof(*hw));
314
315 desc = kmem_cache_zalloc(ioat_cache, flags);
307 if (!desc) {
308 dma_pool_free(ioat_dma->dma_pool, hw, phys);
316 if (!desc)
309 return NULL;
317 return NULL;
310 }
311
312 dma_async_tx_descriptor_init(&desc->txd, chan);
313 desc->txd.tx_submit = ioat_tx_submit_unlock;
314 desc->hw = hw;
315 desc->txd.phys = phys;
316 return desc;
317}
318
319void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
320{
318
319 dma_async_tx_descriptor_init(&desc->txd, chan);
320 desc->txd.tx_submit = ioat_tx_submit_unlock;
321 desc->hw = hw;
322 desc->txd.phys = phys;
323 return desc;
324}
325
326void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
327{
321 struct ioatdma_device *ioat_dma;
322
323 ioat_dma = to_ioatdma_device(chan->device);
324 dma_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
325 kmem_cache_free(ioat_cache, desc);
326}
327
328struct ioat_ring_ent **
329ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
330{
328 kmem_cache_free(ioat_cache, desc);
329}
330
331struct ioat_ring_ent **
332ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
333{
334 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
331 struct ioat_ring_ent **ring;
335 struct ioat_ring_ent **ring;
332 int descs = 1 << order;
333 int i;
336 int total_descs = 1 << order;
337 int i, chunks;
334
335 /* allocate the array to hold the software ring */
338
339 /* allocate the array to hold the software ring */
336 ring = kcalloc(descs, sizeof(*ring), flags);
340 ring = kcalloc(total_descs, sizeof(*ring), flags);
337 if (!ring)
338 return NULL;
341 if (!ring)
342 return NULL;
339 for (i = 0; i < descs; i++) {
340 ring[i] = ioat_alloc_ring_ent(c, flags);
343
344 ioat_chan->desc_chunks = chunks = (total_descs * IOAT_DESC_SZ) / SZ_2M;
345
346 for (i = 0; i < chunks; i++) {
347 struct ioat_descs *descs = &ioat_chan->descs[i];
348
349 descs->virt = dma_alloc_coherent(to_dev(ioat_chan),
350 SZ_2M, &descs->hw, flags);
351 if (!descs->virt && (i > 0)) {
352 int idx;
353
354 for (idx = 0; idx < i; idx++) {
355 dma_free_coherent(to_dev(ioat_chan), SZ_2M,
356 descs->virt, descs->hw);
357 descs->virt = NULL;
358 descs->hw = 0;
359 }
360
361 ioat_chan->desc_chunks = 0;
362 kfree(ring);
363 return NULL;
364 }
365 }
366
367 for (i = 0; i < total_descs; i++) {
368 ring[i] = ioat_alloc_ring_ent(c, i, flags);
341 if (!ring[i]) {
369 if (!ring[i]) {
370 int idx;
371
342 while (i--)
343 ioat_free_ring_ent(ring[i], c);
372 while (i--)
373 ioat_free_ring_ent(ring[i], c);
374
375 for (idx = 0; idx < ioat_chan->desc_chunks; idx++) {
376 dma_free_coherent(to_dev(ioat_chan),
377 SZ_2M,
378 ioat_chan->descs[idx].virt,
379 ioat_chan->descs[idx].hw);
380 ioat_chan->descs[idx].virt = NULL;
381 ioat_chan->descs[idx].hw = 0;
382 }
383
384 ioat_chan->desc_chunks = 0;
344 kfree(ring);
345 return NULL;
346 }
347 set_desc_id(ring[i], i);
348 }
349
350 /* link descs */
385 kfree(ring);
386 return NULL;
387 }
388 set_desc_id(ring[i], i);
389 }
390
391 /* link descs */
351 for (i = 0; i < descs-1; i++) {
392 for (i = 0; i < total_descs-1; i++) {
352 struct ioat_ring_ent *next = ring[i+1];
353 struct ioat_dma_descriptor *hw = ring[i]->hw;
354
355 hw->next = next->txd.phys;
356 }
357 ring[i]->hw->next = ring[0]->txd.phys;
358
359 return ring;

--- 488 unchanged lines hidden --- 		393 struct ioat_ring_ent *next = ring[i+1];
394 struct ioat_dma_descriptor *hw = ring[i]->hw;
395
396 hw->next = next->txd.phys;
397 }
398 ring[i]->hw->next = ring[0]->txd.phys;
399
400 return ring;

--- 488 unchanged lines hidden ---