dma.c - OpenGrok cross reference for /linux/drivers/dma/ioat/dma.c

Deleted Added

sdiffudifftextold (679cfbf7..)new (cd60cd96..)

dma.c (679cfbf79b4eb7d7d81195e6b9ab98106fd78a54)	dma.c (cd60cd96137f6cb3ea82cace9225626619e7a52d)
1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2015 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * --- 318 unchanged lines hidden (view full) --- 327 328struct ioat_ring_ent ** 329ioat_alloc_ring(struct dma_chan c, int order, gfp_t flags) 330{ 331 struct ioat_ring_ent *ring; 332 int descs = 1 << order; 333 int i; 334	1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2015 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * --- 318 unchanged lines hidden (view full) --- 327 328struct ioat_ring_ent ** 329ioat_alloc_ring(struct dma_chan c, int order, gfp_t flags) 330{ 331 struct ioat_ring_ent *ring; 332 int descs = 1 << order; 333 int i; 334
335 if (order > ioat_get_max_alloc_order()) 336 return NULL; 337
338 /* allocate the array to hold the software ring / 339 ring = kcalloc(descs, sizeof(ring), flags); 340 if (!ring) 341 return NULL; 342 for (i = 0; i < descs; i++) { 343 ring[i] = ioat_alloc_ring_ent(c, flags); 344 if (!ring[i]) { 345 while (i--) --- 11 unchanged lines hidden (view full) --- 357 358 hw->next = next->txd.phys; 359 } 360 ring[i]->hw->next = ring[0]->txd.phys; 361 362 return ring; 363} 364	335 /* allocate the array to hold the software ring / 336 ring = kcalloc(descs, sizeof(ring), flags); 337 if (!ring) 338 return NULL; 339 for (i = 0; i < descs; i++) { 340 ring[i] = ioat_alloc_ring_ent(c, flags); 341 if (!ring[i]) { 342 while (i--) --- 11 unchanged lines hidden (view full) --- 354 355 hw->next = next->txd.phys; 356 } 357 ring[i]->hw->next = ring[0]->txd.phys; 358 359 return ring; 360} 361
365static bool reshape_ring(struct ioatdma_chan ioat_chan, int order) 366{ 367 / reshape differs from normal ring allocation in that we want 368 * to allocate a new software ring while only 369 * extending/truncating the hardware ring 370 / 371 struct dma_chan c = &ioat_chan->dma_chan; 372 const u32 curr_size = ioat_ring_size(ioat_chan); 373 const u16 active = ioat_ring_active(ioat_chan); 374 const u32 new_size = 1 << order; 375 struct ioat_ring_ent *ring; 376 u32 i; 377 378 if (order > ioat_get_max_alloc_order()) 379 return false; 380 381 / double check that we have at least 1 free descriptor / 382 if (active == curr_size) 383 return false; 384 385 / when shrinking, verify that we can hold the current active 386 * set in the new ring 387 / 388 if (active >= new_size) 389 return false; 390 391 / allocate the array to hold the software ring / 392 ring = kcalloc(new_size, sizeof(ring), GFP_NOWAIT); 393 if (!ring) 394 return false; 395 396 /* allocate/trim descriptors as needed / 397 if (new_size > curr_size) { 398 / copy current descriptors to the new ring / 399 for (i = 0; i < curr_size; i++) { 400 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); 401 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 402 403 ring[new_idx] = ioat_chan->ring[curr_idx]; 404 set_desc_id(ring[new_idx], new_idx); 405 } 406 407 / add new descriptors to the ring / 408 for (i = curr_size; i < new_size; i++) { 409 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 410 411 ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT); 412 if (!ring[new_idx]) { 413 while (i--) { 414 u16 new_idx = (ioat_chan->tail+i) & 415 (new_size-1); 416 417 ioat_free_ring_ent(ring[new_idx], c); 418 } 419 kfree(ring); 420 return false; 421 } 422 set_desc_id(ring[new_idx], new_idx); 423 } 424 425 / hw link new descriptors / 426 for (i = curr_size-1; i < new_size; i++) { 427 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 428 struct ioat_ring_ent next = 429 ring[(new_idx+1) & (new_size-1)]; 430 struct ioat_dma_descriptor hw = ring[new_idx]->hw; 431 432 hw->next = next->txd.phys; 433 } 434 } else { 435 struct ioat_dma_descriptor hw; 436 struct ioat_ring_ent next; 437 438 / copy current descriptors to the new ring, dropping the 439 * removed descriptors 440 / 441 for (i = 0; i < new_size; i++) { 442 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); 443 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 444 445 ring[new_idx] = ioat_chan->ring[curr_idx]; 446 set_desc_id(ring[new_idx], new_idx); 447 } 448 449 / free deleted descriptors / 450 for (i = new_size; i < curr_size; i++) { 451 struct ioat_ring_ent ent; 452 453 ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i); 454 ioat_free_ring_ent(ent, c); 455 } 456 457 /* fix up hardware ring */ 458 hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw; 459 next = ring[(ioat_chan->tail+new_size) & (new_size-1)]; 460 hw->next = next->txd.phys; 461 } 462 463 dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n", 464 __func__, new_size); 465 466 kfree(ioat_chan->ring); 467 ioat_chan->ring = ring; 468 ioat_chan->alloc_order = order; 469 470 return true; 471} 472
473/** 474 * ioat_check_space_lock - verify space and grab ring producer lock 475 * @ioat: ioat,3 channel (ring) to operate on 476 * @num_descs: allocation length 477 / 478int ioat_check_space_lock(struct ioatdma_chan ioat_chan, int num_descs) 479 __acquires(&ioat_chan->prep_lock) 480{	362/** 363 * ioat_check_space_lock - verify space and grab ring producer lock 364 * @ioat: ioat,3 channel (ring) to operate on 365 * @num_descs: allocation length 366 / 367int ioat_check_space_lock(struct ioatdma_chan ioat_chan, int num_descs) 368 __acquires(&ioat_chan->prep_lock) 369{
481 bool retry; 482 483 retry:
484 spin_lock_bh(&ioat_chan->prep_lock); 485 /* never allow the last descriptor to be consumed, we need at 486 * least one free at all times to allow for on-the-fly ring 487 * resizing. 488 / 489 if (likely(ioat_ring_space(ioat_chan) > num_descs)) { 490 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", 491 __func__, num_descs, ioat_chan->head, 492 ioat_chan->tail, ioat_chan->issued); 493 ioat_chan->produce = num_descs; 494 return 0; / with ioat->prep_lock held */ 495 }	370 spin_lock_bh(&ioat_chan->prep_lock); 371 /* never allow the last descriptor to be consumed, we need at 372 * least one free at all times to allow for on-the-fly ring 373 * resizing. 374 / 375 if (likely(ioat_ring_space(ioat_chan) > num_descs)) { 376 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", 377 __func__, num_descs, ioat_chan->head, 378 ioat_chan->tail, ioat_chan->issued); 379 ioat_chan->produce = num_descs; 380 return 0; / with ioat->prep_lock held */ 381 }
496 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
497 spin_unlock_bh(&ioat_chan->prep_lock); 498	382 spin_unlock_bh(&ioat_chan->prep_lock); 383
499 /* is another cpu already trying to expand the ring? / 500 if (retry) 501 goto retry; 502 503 spin_lock_bh(&ioat_chan->cleanup_lock); 504 spin_lock_bh(&ioat_chan->prep_lock); 505 retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1); 506 clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); 507 spin_unlock_bh(&ioat_chan->prep_lock); 508 spin_unlock_bh(&ioat_chan->cleanup_lock); 509 510 / if we were able to expand the ring retry the allocation */ 511 if (retry) 512 goto retry; 513
514 dev_dbg_ratelimited(to_dev(ioat_chan), 515 "%s: ring full! num_descs: %d (%x:%x:%x)\n", 516 __func__, num_descs, ioat_chan->head, 517 ioat_chan->tail, ioat_chan->issued); 518 519 /* progress reclaim in the allocation failure case we may be 520 * called under bh_disabled so we need to trigger the timer 521 * event directly --- 296 unchanged lines hidden (view full) --- 818{ 819 if (ioat_ring_active(ioat_chan)) { 820 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 821 return; 822 } 823 824 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 825 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);	384 dev_dbg_ratelimited(to_dev(ioat_chan), 385 "%s: ring full! num_descs: %d (%x:%x:%x)\n", 386 __func__, num_descs, ioat_chan->head, 387 ioat_chan->tail, ioat_chan->issued); 388 389 /* progress reclaim in the allocation failure case we may be 390 * called under bh_disabled so we need to trigger the timer 391 * event directly --- 296 unchanged lines hidden (view full) --- 688{ 689 if (ioat_ring_active(ioat_chan)) { 690 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 691 return; 692 } 693 694 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 695 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
826 else if (ioat_chan->alloc_order > ioat_get_alloc_order()) { 827 /* if the ring is idle, empty, and oversized try to step 828 * down the size 829 / 830 reshape_ring(ioat_chan, ioat_chan->alloc_order - 1); 831 832 / keep shrinking until we get back to our minimum 833 * default size 834 */ 835 if (ioat_chan->alloc_order > ioat_get_alloc_order()) 836 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); 837 } 838
839} 840 841void ioat_timer_event(unsigned long data) 842{ 843 struct ioatdma_chan ioat_chan = to_ioat_chan((void )data); 844 dma_addr_t phys_complete; 845 u64 status; 846 --- 144 unchanged lines hidden ---	696} 697 698void ioat_timer_event(unsigned long data) 699{ 700 struct ioatdma_chan ioat_chan = to_ioat_chan((void )data); 701 dma_addr_t phys_complete; 702 u64 status; 703 --- 144 unchanged lines hidden ---

dma.c (679cfbf79b4eb7d7d81195e6b9ab98106fd78a54)	dma.c (cd60cd96137f6cb3ea82cace9225626619e7a52d)
1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2015 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * --- 318 unchanged lines hidden (view full) --- 327 328struct ioat_ring_ent ** 329ioat_alloc_ring(struct dma_chan c, int order, gfp_t flags) 330{ 331 struct ioat_ring_ent *ring; 332 int descs = 1 << order; 333 int i; 334	1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2015 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * --- 318 unchanged lines hidden (view full) --- 327 328struct ioat_ring_ent ** 329ioat_alloc_ring(struct dma_chan c, int order, gfp_t flags) 330{ 331 struct ioat_ring_ent *ring; 332 int descs = 1 << order; 333 int i; 334
335 if (order > ioat_get_max_alloc_order()) 336 return NULL; 337
338 /* allocate the array to hold the software ring / 339 ring = kcalloc(descs, sizeof(ring), flags); 340 if (!ring) 341 return NULL; 342 for (i = 0; i < descs; i++) { 343 ring[i] = ioat_alloc_ring_ent(c, flags); 344 if (!ring[i]) { 345 while (i--) --- 11 unchanged lines hidden (view full) --- 357 358 hw->next = next->txd.phys; 359 } 360 ring[i]->hw->next = ring[0]->txd.phys; 361 362 return ring; 363} 364	335 /* allocate the array to hold the software ring / 336 ring = kcalloc(descs, sizeof(ring), flags); 337 if (!ring) 338 return NULL; 339 for (i = 0; i < descs; i++) { 340 ring[i] = ioat_alloc_ring_ent(c, flags); 341 if (!ring[i]) { 342 while (i--) --- 11 unchanged lines hidden (view full) --- 354 355 hw->next = next->txd.phys; 356 } 357 ring[i]->hw->next = ring[0]->txd.phys; 358 359 return ring; 360} 361
365static bool reshape_ring(struct ioatdma_chan ioat_chan, int order) 366{ 367 / reshape differs from normal ring allocation in that we want 368 * to allocate a new software ring while only 369 * extending/truncating the hardware ring 370 / 371 struct dma_chan c = &ioat_chan->dma_chan; 372 const u32 curr_size = ioat_ring_size(ioat_chan); 373 const u16 active = ioat_ring_active(ioat_chan); 374 const u32 new_size = 1 << order; 375 struct ioat_ring_ent *ring; 376 u32 i; 377 378 if (order > ioat_get_max_alloc_order()) 379 return false; 380 381 / double check that we have at least 1 free descriptor / 382 if (active == curr_size) 383 return false; 384 385 / when shrinking, verify that we can hold the current active 386 * set in the new ring 387 / 388 if (active >= new_size) 389 return false; 390 391 / allocate the array to hold the software ring / 392 ring = kcalloc(new_size, sizeof(ring), GFP_NOWAIT); 393 if (!ring) 394 return false; 395 396 /* allocate/trim descriptors as needed / 397 if (new_size > curr_size) { 398 / copy current descriptors to the new ring / 399 for (i = 0; i < curr_size; i++) { 400 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); 401 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 402 403 ring[new_idx] = ioat_chan->ring[curr_idx]; 404 set_desc_id(ring[new_idx], new_idx); 405 } 406 407 / add new descriptors to the ring / 408 for (i = curr_size; i < new_size; i++) { 409 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 410 411 ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT); 412 if (!ring[new_idx]) { 413 while (i--) { 414 u16 new_idx = (ioat_chan->tail+i) & 415 (new_size-1); 416 417 ioat_free_ring_ent(ring[new_idx], c); 418 } 419 kfree(ring); 420 return false; 421 } 422 set_desc_id(ring[new_idx], new_idx); 423 } 424 425 / hw link new descriptors / 426 for (i = curr_size-1; i < new_size; i++) { 427 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 428 struct ioat_ring_ent next = 429 ring[(new_idx+1) & (new_size-1)]; 430 struct ioat_dma_descriptor hw = ring[new_idx]->hw; 431 432 hw->next = next->txd.phys; 433 } 434 } else { 435 struct ioat_dma_descriptor hw; 436 struct ioat_ring_ent next; 437 438 / copy current descriptors to the new ring, dropping the 439 * removed descriptors 440 / 441 for (i = 0; i < new_size; i++) { 442 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); 443 u16 new_idx = (ioat_chan->tail+i) & (new_size-1); 444 445 ring[new_idx] = ioat_chan->ring[curr_idx]; 446 set_desc_id(ring[new_idx], new_idx); 447 } 448 449 / free deleted descriptors / 450 for (i = new_size; i < curr_size; i++) { 451 struct ioat_ring_ent ent; 452 453 ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i); 454 ioat_free_ring_ent(ent, c); 455 } 456 457 /* fix up hardware ring */ 458 hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw; 459 next = ring[(ioat_chan->tail+new_size) & (new_size-1)]; 460 hw->next = next->txd.phys; 461 } 462 463 dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n", 464 __func__, new_size); 465 466 kfree(ioat_chan->ring); 467 ioat_chan->ring = ring; 468 ioat_chan->alloc_order = order; 469 470 return true; 471} 472
473/** 474 * ioat_check_space_lock - verify space and grab ring producer lock 475 * @ioat: ioat,3 channel (ring) to operate on 476 * @num_descs: allocation length 477 / 478int ioat_check_space_lock(struct ioatdma_chan ioat_chan, int num_descs) 479 __acquires(&ioat_chan->prep_lock) 480{	362/** 363 * ioat_check_space_lock - verify space and grab ring producer lock 364 * @ioat: ioat,3 channel (ring) to operate on 365 * @num_descs: allocation length 366 / 367int ioat_check_space_lock(struct ioatdma_chan ioat_chan, int num_descs) 368 __acquires(&ioat_chan->prep_lock) 369{
481 bool retry; 482 483 retry:
484 spin_lock_bh(&ioat_chan->prep_lock); 485 /* never allow the last descriptor to be consumed, we need at 486 * least one free at all times to allow for on-the-fly ring 487 * resizing. 488 / 489 if (likely(ioat_ring_space(ioat_chan) > num_descs)) { 490 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", 491 __func__, num_descs, ioat_chan->head, 492 ioat_chan->tail, ioat_chan->issued); 493 ioat_chan->produce = num_descs; 494 return 0; / with ioat->prep_lock held */ 495 }	370 spin_lock_bh(&ioat_chan->prep_lock); 371 /* never allow the last descriptor to be consumed, we need at 372 * least one free at all times to allow for on-the-fly ring 373 * resizing. 374 / 375 if (likely(ioat_ring_space(ioat_chan) > num_descs)) { 376 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", 377 __func__, num_descs, ioat_chan->head, 378 ioat_chan->tail, ioat_chan->issued); 379 ioat_chan->produce = num_descs; 380 return 0; / with ioat->prep_lock held */ 381 }
496 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
497 spin_unlock_bh(&ioat_chan->prep_lock); 498	382 spin_unlock_bh(&ioat_chan->prep_lock); 383
499 /* is another cpu already trying to expand the ring? / 500 if (retry) 501 goto retry; 502 503 spin_lock_bh(&ioat_chan->cleanup_lock); 504 spin_lock_bh(&ioat_chan->prep_lock); 505 retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1); 506 clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); 507 spin_unlock_bh(&ioat_chan->prep_lock); 508 spin_unlock_bh(&ioat_chan->cleanup_lock); 509 510 / if we were able to expand the ring retry the allocation */ 511 if (retry) 512 goto retry; 513
514 dev_dbg_ratelimited(to_dev(ioat_chan), 515 "%s: ring full! num_descs: %d (%x:%x:%x)\n", 516 __func__, num_descs, ioat_chan->head, 517 ioat_chan->tail, ioat_chan->issued); 518 519 /* progress reclaim in the allocation failure case we may be 520 * called under bh_disabled so we need to trigger the timer 521 * event directly --- 296 unchanged lines hidden (view full) --- 818{ 819 if (ioat_ring_active(ioat_chan)) { 820 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 821 return; 822 } 823 824 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 825 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);	384 dev_dbg_ratelimited(to_dev(ioat_chan), 385 "%s: ring full! num_descs: %d (%x:%x:%x)\n", 386 __func__, num_descs, ioat_chan->head, 387 ioat_chan->tail, ioat_chan->issued); 388 389 /* progress reclaim in the allocation failure case we may be 390 * called under bh_disabled so we need to trigger the timer 391 * event directly --- 296 unchanged lines hidden (view full) --- 688{ 689 if (ioat_ring_active(ioat_chan)) { 690 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); 691 return; 692 } 693 694 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) 695 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
826 else if (ioat_chan->alloc_order > ioat_get_alloc_order()) { 827 /* if the ring is idle, empty, and oversized try to step 828 * down the size 829 / 830 reshape_ring(ioat_chan, ioat_chan->alloc_order - 1); 831 832 / keep shrinking until we get back to our minimum 833 * default size 834 */ 835 if (ioat_chan->alloc_order > ioat_get_alloc_order()) 836 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); 837 } 838
839} 840 841void ioat_timer_event(unsigned long data) 842{ 843 struct ioatdma_chan ioat_chan = to_ioat_chan((void )data); 844 dma_addr_t phys_complete; 845 u64 status; 846 --- 144 unchanged lines hidden ---	696} 697 698void ioat_timer_event(unsigned long data) 699{ 700 struct ioatdma_chan ioat_chan = to_ioat_chan((void )data); 701 dma_addr_t phys_complete; 702 u64 status; 703 --- 144 unchanged lines hidden ---