1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * A generic kernel FIFO implementation 4 * 5 * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net> 6 */ 7 8 #ifndef _LINUX_KFIFO_H 9 #define _LINUX_KFIFO_H 10 11 /* 12 * How to porting drivers to the new generic FIFO API: 13 * 14 * - Modify the declaration of the "struct kfifo *" object into a 15 * in-place "struct kfifo" object 16 * - Init the in-place object with kfifo_alloc() or kfifo_init() 17 * Note: The address of the in-place "struct kfifo" object must be 18 * passed as the first argument to this functions 19 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get 20 * into kfifo_out 21 * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get 22 * into kfifo_out_spinlocked 23 * Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc 24 * must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked 25 * as the last parameter 26 * - The formerly __kfifo_* functions are renamed into kfifo_* 27 */ 28 29 /* 30 * Note about locking: There is no locking required until only one reader 31 * and one writer is using the fifo and no kfifo_reset() will be called. 32 * kfifo_reset_out() can be safely used, until it will be only called 33 * in the reader thread. 34 * For multiple writer and one reader there is only a need to lock the writer. 35 * And vice versa for only one writer and multiple reader there is only a need 36 * to lock the reader. 37 */ 38 39 #include <linux/array_size.h> 40 #include <linux/spinlock.h> 41 #include <linux/stddef.h> 42 #include <linux/types.h> 43 44 #include <asm/barrier.h> 45 #include <asm/errno.h> 46 47 struct scatterlist; 48 49 struct __kfifo { 50 unsigned int in; 51 unsigned int out; 52 unsigned int mask; 53 unsigned int esize; 54 void *data; 55 }; 56 57 #define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \ 58 union { \ 59 struct __kfifo kfifo; \ 60 datatype *type; \ 61 const datatype *const_type; \ 62 char (*rectype)[recsize]; \ 63 ptrtype *ptr; \ 64 ptrtype const *ptr_const; \ 65 } 66 67 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \ 68 { \ 69 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \ 70 type buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \ 71 } 72 73 #define STRUCT_KFIFO(type, size) \ 74 struct __STRUCT_KFIFO(type, size, 0, type) 75 76 #define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \ 77 { \ 78 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \ 79 type buf[0]; \ 80 } 81 82 #define STRUCT_KFIFO_PTR(type) \ 83 struct __STRUCT_KFIFO_PTR(type, 0, type) 84 85 /* 86 * define compatibility "struct kfifo" for dynamic allocated fifos 87 */ 88 struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void); 89 90 #define STRUCT_KFIFO_REC_1(size) \ 91 struct __STRUCT_KFIFO(unsigned char, size, 1, void) 92 93 #define STRUCT_KFIFO_REC_2(size) \ 94 struct __STRUCT_KFIFO(unsigned char, size, 2, void) 95 96 /* 97 * define kfifo_rec types 98 */ 99 struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void); 100 struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void); 101 102 /* 103 * helper macro to distinguish between real in place fifo where the fifo 104 * array is a part of the structure and the fifo type where the array is 105 * outside of the fifo structure. 106 */ 107 #define __is_kfifo_ptr(fifo) \ 108 (sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type)))) 109 110 /** 111 * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object 112 * @fifo: name of the declared fifo 113 * @type: type of the fifo elements 114 */ 115 #define DECLARE_KFIFO_PTR(fifo, type) STRUCT_KFIFO_PTR(type) fifo 116 117 /** 118 * DECLARE_KFIFO - macro to declare a fifo object 119 * @fifo: name of the declared fifo 120 * @type: type of the fifo elements 121 * @size: the number of elements in the fifo, this must be a power of 2 122 */ 123 #define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo 124 125 /** 126 * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO 127 * @fifo: name of the declared fifo datatype 128 */ 129 #define INIT_KFIFO(fifo) \ 130 (void)({ \ 131 typeof(&(fifo)) __tmp = &(fifo); \ 132 struct __kfifo *__kfifo = &__tmp->kfifo; \ 133 __kfifo->in = 0; \ 134 __kfifo->out = 0; \ 135 __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\ 136 __kfifo->esize = sizeof(*__tmp->buf); \ 137 __kfifo->data = __is_kfifo_ptr(__tmp) ? NULL : __tmp->buf; \ 138 }) 139 140 /** 141 * DEFINE_KFIFO - macro to define and initialize a fifo 142 * @fifo: name of the declared fifo datatype 143 * @type: type of the fifo elements 144 * @size: the number of elements in the fifo, this must be a power of 2 145 * 146 * Note: the macro can be used for global and local fifo data type variables. 147 */ 148 #define DEFINE_KFIFO(fifo, type, size) \ 149 DECLARE_KFIFO(fifo, type, size) = \ 150 (typeof(fifo)) { \ 151 { \ 152 { \ 153 .in = 0, \ 154 .out = 0, \ 155 .mask = __is_kfifo_ptr(&(fifo)) ? \ 156 0 : \ 157 ARRAY_SIZE((fifo).buf) - 1, \ 158 .esize = sizeof(*(fifo).buf), \ 159 .data = __is_kfifo_ptr(&(fifo)) ? \ 160 NULL : \ 161 (fifo).buf, \ 162 } \ 163 } \ 164 } 165 166 167 static inline unsigned int __must_check 168 __kfifo_uint_must_check_helper(unsigned int val) 169 { 170 return val; 171 } 172 173 static inline int __must_check 174 __kfifo_int_must_check_helper(int val) 175 { 176 return val; 177 } 178 179 /** 180 * kfifo_initialized - Check if the fifo is initialized 181 * @fifo: address of the fifo to check 182 * 183 * Return %true if fifo is initialized, otherwise %false. 184 * Assumes the fifo was 0 before. 185 */ 186 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask) 187 188 /** 189 * kfifo_esize - returns the size of the element managed by the fifo 190 * @fifo: address of the fifo to be used 191 */ 192 #define kfifo_esize(fifo) ((fifo)->kfifo.esize) 193 194 /** 195 * kfifo_recsize - returns the size of the record length field 196 * @fifo: address of the fifo to be used 197 */ 198 #define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype)) 199 200 /** 201 * kfifo_size - returns the size of the fifo in elements 202 * @fifo: address of the fifo to be used 203 */ 204 #define kfifo_size(fifo) ((fifo)->kfifo.mask + 1) 205 206 /** 207 * kfifo_reset - removes the entire fifo content 208 * @fifo: address of the fifo to be used 209 * 210 * Note: usage of kfifo_reset() is dangerous. It should be only called when the 211 * fifo is exclusived locked or when it is secured that no other thread is 212 * accessing the fifo. 213 */ 214 #define kfifo_reset(fifo) \ 215 (void)({ \ 216 typeof((fifo) + 1) __tmp = (fifo); \ 217 __tmp->kfifo.in = __tmp->kfifo.out = 0; \ 218 }) 219 220 /** 221 * kfifo_reset_out - skip fifo content 222 * @fifo: address of the fifo to be used 223 * 224 * Note: The usage of kfifo_reset_out() is safe until it will be only called 225 * from the reader thread and there is only one concurrent reader. Otherwise 226 * it is dangerous and must be handled in the same way as kfifo_reset(). 227 */ 228 #define kfifo_reset_out(fifo) \ 229 (void)({ \ 230 typeof((fifo) + 1) __tmp = (fifo); \ 231 __tmp->kfifo.out = __tmp->kfifo.in; \ 232 }) 233 234 /** 235 * kfifo_len - returns the number of used elements in the fifo 236 * @fifo: address of the fifo to be used 237 */ 238 #define kfifo_len(fifo) \ 239 ({ \ 240 typeof((fifo) + 1) __tmpl = (fifo); \ 241 __tmpl->kfifo.in - __tmpl->kfifo.out; \ 242 }) 243 244 /** 245 * kfifo_is_empty - returns true if the fifo is empty 246 * @fifo: address of the fifo to be used 247 */ 248 #define kfifo_is_empty(fifo) \ 249 ({ \ 250 typeof((fifo) + 1) __tmpq = (fifo); \ 251 __tmpq->kfifo.in == __tmpq->kfifo.out; \ 252 }) 253 254 /** 255 * kfifo_is_empty_spinlocked - returns true if the fifo is empty using 256 * a spinlock for locking 257 * @fifo: address of the fifo to be used 258 * @lock: spinlock to be used for locking 259 */ 260 #define kfifo_is_empty_spinlocked(fifo, lock) \ 261 ({ \ 262 unsigned long __flags; \ 263 bool __ret; \ 264 spin_lock_irqsave(lock, __flags); \ 265 __ret = kfifo_is_empty(fifo); \ 266 spin_unlock_irqrestore(lock, __flags); \ 267 __ret; \ 268 }) 269 270 /** 271 * kfifo_is_empty_spinlocked_noirqsave - returns true if the fifo is empty 272 * using a spinlock for locking, doesn't disable interrupts 273 * @fifo: address of the fifo to be used 274 * @lock: spinlock to be used for locking 275 */ 276 #define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \ 277 ({ \ 278 bool __ret; \ 279 spin_lock(lock); \ 280 __ret = kfifo_is_empty(fifo); \ 281 spin_unlock(lock); \ 282 __ret; \ 283 }) 284 285 /** 286 * kfifo_is_full - returns true if the fifo is full 287 * @fifo: address of the fifo to be used 288 */ 289 #define kfifo_is_full(fifo) \ 290 ({ \ 291 typeof((fifo) + 1) __tmpq = (fifo); \ 292 kfifo_len(__tmpq) > __tmpq->kfifo.mask; \ 293 }) 294 295 /** 296 * kfifo_avail - returns the number of unused elements in the fifo 297 * @fifo: address of the fifo to be used 298 */ 299 #define kfifo_avail(fifo) \ 300 __kfifo_uint_must_check_helper( \ 301 ({ \ 302 typeof((fifo) + 1) __tmpq = (fifo); \ 303 const size_t __recsize = sizeof(*__tmpq->rectype); \ 304 unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \ 305 (__recsize) ? ((__avail <= __recsize) ? 0 : \ 306 __kfifo_max_r(__avail - __recsize, __recsize)) : \ 307 __avail; \ 308 }) \ 309 ) 310 311 /** 312 * kfifo_skip_count - skip output data 313 * @fifo: address of the fifo to be used 314 * @count: count of data to skip 315 */ 316 #define kfifo_skip_count(fifo, count) do { \ 317 typeof((fifo) + 1) __tmp = (fifo); \ 318 const size_t __recsize = sizeof(*__tmp->rectype); \ 319 struct __kfifo *__kfifo = &__tmp->kfifo; \ 320 if (__recsize) \ 321 __kfifo_skip_r(__kfifo, __recsize); \ 322 else \ 323 __kfifo->out += (count); \ 324 } while(0) 325 326 /** 327 * kfifo_skip - skip output data 328 * @fifo: address of the fifo to be used 329 */ 330 #define kfifo_skip(fifo) kfifo_skip_count(fifo, 1) 331 332 /** 333 * kfifo_peek_len - gets the size of the next fifo record 334 * @fifo: address of the fifo to be used 335 * 336 * This function returns the size of the next fifo record in number of bytes. 337 */ 338 #define kfifo_peek_len(fifo) \ 339 __kfifo_uint_must_check_helper( \ 340 ({ \ 341 typeof((fifo) + 1) __tmp = (fifo); \ 342 const size_t __recsize = sizeof(*__tmp->rectype); \ 343 struct __kfifo *__kfifo = &__tmp->kfifo; \ 344 (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \ 345 __kfifo_len_r(__kfifo, __recsize); \ 346 }) \ 347 ) 348 349 /** 350 * kfifo_alloc - dynamically allocates a new fifo buffer 351 * @fifo: pointer to the fifo 352 * @size: the number of elements in the fifo, this must be a power of 2 353 * @gfp_mask: get_free_pages mask, passed to kmalloc() 354 * 355 * This macro dynamically allocates a new fifo buffer. 356 * 357 * The number of elements will be rounded-up to a power of 2. 358 * The fifo will be release with kfifo_free(). 359 * Return 0 if no error, otherwise an error code. 360 */ 361 #define kfifo_alloc(fifo, size, gfp_mask) \ 362 __kfifo_int_must_check_helper( \ 363 ({ \ 364 typeof((fifo) + 1) __tmp = (fifo); \ 365 struct __kfifo *__kfifo = &__tmp->kfifo; \ 366 __is_kfifo_ptr(__tmp) ? \ 367 __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \ 368 -EINVAL; \ 369 }) \ 370 ) 371 372 /** 373 * kfifo_alloc_node - dynamically allocates a new fifo buffer on a NUMA node 374 * @fifo: pointer to the fifo 375 * @size: the number of elements in the fifo, this must be a power of 2 376 * @gfp_mask: get_free_pages mask, passed to kmalloc() 377 * @node: NUMA node to allocate memory on 378 * 379 * This macro dynamically allocates a new fifo buffer with NUMA node awareness. 380 * 381 * The number of elements will be rounded-up to a power of 2. 382 * The fifo will be release with kfifo_free(). 383 * Return 0 if no error, otherwise an error code. 384 */ 385 #define kfifo_alloc_node(fifo, size, gfp_mask, node) \ 386 __kfifo_int_must_check_helper( \ 387 ({ \ 388 typeof((fifo) + 1) __tmp = (fifo); \ 389 struct __kfifo *__kfifo = &__tmp->kfifo; \ 390 __is_kfifo_ptr(__tmp) ? \ 391 __kfifo_alloc_node(__kfifo, size, sizeof(*__tmp->type), gfp_mask, node) : \ 392 -EINVAL; \ 393 }) \ 394 ) 395 396 /** 397 * kfifo_free - frees the fifo 398 * @fifo: the fifo to be freed 399 */ 400 #define kfifo_free(fifo) \ 401 ({ \ 402 typeof((fifo) + 1) __tmp = (fifo); \ 403 struct __kfifo *__kfifo = &__tmp->kfifo; \ 404 if (__is_kfifo_ptr(__tmp)) \ 405 __kfifo_free(__kfifo); \ 406 }) 407 408 /** 409 * kfifo_init - initialize a fifo using a preallocated buffer 410 * @fifo: the fifo to assign the buffer 411 * @buffer: the preallocated buffer to be used 412 * @size: the size of the internal buffer, this have to be a power of 2 413 * 414 * This macro initializes a fifo using a preallocated buffer. 415 * 416 * The number of elements will be rounded-up to a power of 2. 417 * Return 0 if no error, otherwise an error code. 418 */ 419 #define kfifo_init(fifo, buffer, size) \ 420 ({ \ 421 typeof((fifo) + 1) __tmp = (fifo); \ 422 struct __kfifo *__kfifo = &__tmp->kfifo; \ 423 __is_kfifo_ptr(__tmp) ? \ 424 __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \ 425 -EINVAL; \ 426 }) 427 428 /** 429 * kfifo_put - put data into the fifo 430 * @fifo: address of the fifo to be used 431 * @val: the data to be added 432 * 433 * This macro copies the given value into the fifo. 434 * It returns 0 if the fifo was full. Otherwise it returns the number 435 * processed elements. 436 * 437 * Note that with only one concurrent reader and one concurrent 438 * writer, you don't need extra locking to use these macro. 439 */ 440 #define kfifo_put(fifo, val) \ 441 ({ \ 442 typeof((fifo) + 1) __tmp = (fifo); \ 443 typeof(*__tmp->const_type) __val = (val); \ 444 unsigned int __ret; \ 445 size_t __recsize = sizeof(*__tmp->rectype); \ 446 struct __kfifo *__kfifo = &__tmp->kfifo; \ 447 if (__recsize) \ 448 __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \ 449 __recsize); \ 450 else { \ 451 __ret = !kfifo_is_full(__tmp); \ 452 if (__ret) { \ 453 (__is_kfifo_ptr(__tmp) ? \ 454 ((typeof(__tmp->type))__kfifo->data) : \ 455 (__tmp->buf) \ 456 )[__kfifo->in & __tmp->kfifo.mask] = \ 457 *(typeof(__tmp->type))&__val; \ 458 smp_wmb(); \ 459 __kfifo->in++; \ 460 } \ 461 } \ 462 __ret; \ 463 }) 464 465 /** 466 * kfifo_get - get data from the fifo 467 * @fifo: address of the fifo to be used 468 * @val: address where to store the data 469 * 470 * This macro reads the data from the fifo. 471 * It returns 0 if the fifo was empty. Otherwise it returns the number 472 * processed elements. 473 * 474 * Note that with only one concurrent reader and one concurrent 475 * writer, you don't need extra locking to use these macro. 476 */ 477 #define kfifo_get(fifo, val) \ 478 __kfifo_uint_must_check_helper( \ 479 ({ \ 480 typeof((fifo) + 1) __tmp = (fifo); \ 481 typeof(__tmp->ptr) __val = (val); \ 482 unsigned int __ret; \ 483 const size_t __recsize = sizeof(*__tmp->rectype); \ 484 struct __kfifo *__kfifo = &__tmp->kfifo; \ 485 if (__recsize) \ 486 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \ 487 __recsize); \ 488 else { \ 489 __ret = !kfifo_is_empty(__tmp); \ 490 if (__ret) { \ 491 *(typeof(__tmp->type))__val = \ 492 (__is_kfifo_ptr(__tmp) ? \ 493 ((typeof(__tmp->type))__kfifo->data) : \ 494 (__tmp->buf) \ 495 )[__kfifo->out & __tmp->kfifo.mask]; \ 496 smp_wmb(); \ 497 __kfifo->out++; \ 498 } \ 499 } \ 500 __ret; \ 501 }) \ 502 ) 503 504 /** 505 * kfifo_peek - get data from the fifo without removing 506 * @fifo: address of the fifo to be used 507 * @val: address where to store the data 508 * 509 * This reads the data from the fifo without removing it from the fifo. 510 * It returns 0 if the fifo was empty. Otherwise it returns the number 511 * processed elements. 512 * 513 * Note that with only one concurrent reader and one concurrent 514 * writer, you don't need extra locking to use these macro. 515 */ 516 #define kfifo_peek(fifo, val) \ 517 __kfifo_uint_must_check_helper( \ 518 ({ \ 519 typeof((fifo) + 1) __tmp = (fifo); \ 520 typeof(__tmp->ptr) __val = (val); \ 521 unsigned int __ret; \ 522 const size_t __recsize = sizeof(*__tmp->rectype); \ 523 struct __kfifo *__kfifo = &__tmp->kfifo; \ 524 if (__recsize) \ 525 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \ 526 __recsize); \ 527 else { \ 528 __ret = !kfifo_is_empty(__tmp); \ 529 if (__ret) { \ 530 *(typeof(__tmp->type))__val = \ 531 (__is_kfifo_ptr(__tmp) ? \ 532 ((typeof(__tmp->type))__kfifo->data) : \ 533 (__tmp->buf) \ 534 )[__kfifo->out & __tmp->kfifo.mask]; \ 535 smp_wmb(); \ 536 } \ 537 } \ 538 __ret; \ 539 }) \ 540 ) 541 542 /** 543 * kfifo_in - put data into the fifo 544 * @fifo: address of the fifo to be used 545 * @buf: the data to be added 546 * @n: number of elements to be added 547 * 548 * This macro copies the given buffer into the fifo and returns the 549 * number of copied elements. 550 * 551 * Note that with only one concurrent reader and one concurrent 552 * writer, you don't need extra locking to use these macro. 553 */ 554 #define kfifo_in(fifo, buf, n) \ 555 ({ \ 556 typeof((fifo) + 1) __tmp = (fifo); \ 557 typeof(__tmp->ptr_const) __buf = (buf); \ 558 unsigned long __n = (n); \ 559 const size_t __recsize = sizeof(*__tmp->rectype); \ 560 struct __kfifo *__kfifo = &__tmp->kfifo; \ 561 (__recsize) ?\ 562 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \ 563 __kfifo_in(__kfifo, __buf, __n); \ 564 }) 565 566 /** 567 * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking 568 * @fifo: address of the fifo to be used 569 * @buf: the data to be added 570 * @n: number of elements to be added 571 * @lock: pointer to the spinlock to use for locking 572 * 573 * This macro copies the given values buffer into the fifo and returns the 574 * number of copied elements. 575 */ 576 #define kfifo_in_spinlocked(fifo, buf, n, lock) \ 577 ({ \ 578 unsigned long __flags; \ 579 unsigned int __ret; \ 580 spin_lock_irqsave(lock, __flags); \ 581 __ret = kfifo_in(fifo, buf, n); \ 582 spin_unlock_irqrestore(lock, __flags); \ 583 __ret; \ 584 }) 585 586 /** 587 * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for 588 * locking, don't disable interrupts 589 * @fifo: address of the fifo to be used 590 * @buf: the data to be added 591 * @n: number of elements to be added 592 * @lock: pointer to the spinlock to use for locking 593 * 594 * This is a variant of kfifo_in_spinlocked() but uses spin_lock/unlock() 595 * for locking and doesn't disable interrupts. 596 */ 597 #define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \ 598 ({ \ 599 unsigned int __ret; \ 600 spin_lock(lock); \ 601 __ret = kfifo_in(fifo, buf, n); \ 602 spin_unlock(lock); \ 603 __ret; \ 604 }) 605 606 /* alias for kfifo_in_spinlocked, will be removed in a future release */ 607 #define kfifo_in_locked(fifo, buf, n, lock) \ 608 kfifo_in_spinlocked(fifo, buf, n, lock) 609 610 /** 611 * kfifo_out - get data from the fifo 612 * @fifo: address of the fifo to be used 613 * @buf: pointer to the storage buffer 614 * @n: max. number of elements to get 615 * 616 * This macro gets some data from the fifo and returns the numbers of elements 617 * copied. 618 * 619 * Note that with only one concurrent reader and one concurrent 620 * writer, you don't need extra locking to use these macro. 621 */ 622 #define kfifo_out(fifo, buf, n) \ 623 __kfifo_uint_must_check_helper( \ 624 ({ \ 625 typeof((fifo) + 1) __tmp = (fifo); \ 626 typeof(__tmp->ptr) __buf = (buf); \ 627 unsigned long __n = (n); \ 628 const size_t __recsize = sizeof(*__tmp->rectype); \ 629 struct __kfifo *__kfifo = &__tmp->kfifo; \ 630 (__recsize) ?\ 631 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \ 632 __kfifo_out(__kfifo, __buf, __n); \ 633 }) \ 634 ) 635 636 /** 637 * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking 638 * @fifo: address of the fifo to be used 639 * @buf: pointer to the storage buffer 640 * @n: max. number of elements to get 641 * @lock: pointer to the spinlock to use for locking 642 * 643 * This macro gets the data from the fifo and returns the numbers of elements 644 * copied. 645 */ 646 #define kfifo_out_spinlocked(fifo, buf, n, lock) \ 647 __kfifo_uint_must_check_helper( \ 648 ({ \ 649 unsigned long __flags; \ 650 unsigned int __ret; \ 651 spin_lock_irqsave(lock, __flags); \ 652 __ret = kfifo_out(fifo, buf, n); \ 653 spin_unlock_irqrestore(lock, __flags); \ 654 __ret; \ 655 }) \ 656 ) 657 658 /** 659 * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock 660 * for locking, don't disable interrupts 661 * @fifo: address of the fifo to be used 662 * @buf: pointer to the storage buffer 663 * @n: max. number of elements to get 664 * @lock: pointer to the spinlock to use for locking 665 * 666 * This is a variant of kfifo_out_spinlocked() which uses spin_lock/unlock() 667 * for locking and doesn't disable interrupts. 668 */ 669 #define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \ 670 __kfifo_uint_must_check_helper( \ 671 ({ \ 672 unsigned int __ret; \ 673 spin_lock(lock); \ 674 __ret = kfifo_out(fifo, buf, n); \ 675 spin_unlock(lock); \ 676 __ret; \ 677 }) \ 678 ) 679 680 /* alias for kfifo_out_spinlocked, will be removed in a future release */ 681 #define kfifo_out_locked(fifo, buf, n, lock) \ 682 kfifo_out_spinlocked(fifo, buf, n, lock) 683 684 /** 685 * kfifo_from_user - puts some data from user space into the fifo 686 * @fifo: address of the fifo to be used 687 * @from: pointer to the data to be added 688 * @len: the length of the data to be added 689 * @copied: pointer to output variable to store the number of copied bytes 690 * 691 * This macro copies at most @len bytes from the @from into the 692 * fifo, depending of the available space and returns -EFAULT/0. 693 * 694 * Note that with only one concurrent reader and one concurrent 695 * writer, you don't need extra locking to use these macro. 696 */ 697 #define kfifo_from_user(fifo, from, len, copied) \ 698 __kfifo_uint_must_check_helper( \ 699 ({ \ 700 typeof((fifo) + 1) __tmp = (fifo); \ 701 const void __user *__from = (from); \ 702 unsigned int __len = (len); \ 703 unsigned int *__copied = (copied); \ 704 const size_t __recsize = sizeof(*__tmp->rectype); \ 705 struct __kfifo *__kfifo = &__tmp->kfifo; \ 706 (__recsize) ? \ 707 __kfifo_from_user_r(__kfifo, __from, __len, __copied, __recsize) : \ 708 __kfifo_from_user(__kfifo, __from, __len, __copied); \ 709 }) \ 710 ) 711 712 /** 713 * kfifo_to_user - copies data from the fifo into user space 714 * @fifo: address of the fifo to be used 715 * @to: where the data must be copied 716 * @len: the size of the destination buffer 717 * @copied: pointer to output variable to store the number of copied bytes 718 * 719 * This macro copies at most @len bytes from the fifo into the 720 * @to buffer and returns -EFAULT/0. 721 * 722 * Note that with only one concurrent reader and one concurrent 723 * writer, you don't need extra locking to use these macro. 724 */ 725 #define kfifo_to_user(fifo, to, len, copied) \ 726 __kfifo_int_must_check_helper( \ 727 ({ \ 728 typeof((fifo) + 1) __tmp = (fifo); \ 729 void __user *__to = (to); \ 730 unsigned int __len = (len); \ 731 unsigned int *__copied = (copied); \ 732 const size_t __recsize = sizeof(*__tmp->rectype); \ 733 struct __kfifo *__kfifo = &__tmp->kfifo; \ 734 (__recsize) ? \ 735 __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \ 736 __kfifo_to_user(__kfifo, __to, __len, __copied); \ 737 }) \ 738 ) 739 740 /** 741 * kfifo_dma_in_prepare_mapped - setup a scatterlist for DMA input 742 * @fifo: address of the fifo to be used 743 * @sgl: pointer to the scatterlist array 744 * @nents: number of entries in the scatterlist array 745 * @len: number of elements to transfer 746 * @dma: mapped dma address to fill into @sgl 747 * 748 * This macro fills a scatterlist for DMA input. 749 * It returns the number entries in the scatterlist array. 750 * 751 * Note that with only one concurrent reader and one concurrent 752 * writer, you don't need extra locking to use these macros. 753 */ 754 #define kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, dma) \ 755 ({ \ 756 typeof((fifo) + 1) __tmp = (fifo); \ 757 struct scatterlist *__sgl = (sgl); \ 758 int __nents = (nents); \ 759 unsigned int __len = (len); \ 760 const size_t __recsize = sizeof(*__tmp->rectype); \ 761 struct __kfifo *__kfifo = &__tmp->kfifo; \ 762 (__recsize) ? \ 763 __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize, \ 764 dma) : \ 765 __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len, dma); \ 766 }) 767 768 #define kfifo_dma_in_prepare(fifo, sgl, nents, len) \ 769 kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR) 770 771 /** 772 * kfifo_dma_in_finish - finish a DMA IN operation 773 * @fifo: address of the fifo to be used 774 * @len: number of bytes to received 775 * 776 * This macro finishes a DMA IN operation. The in counter will be updated by 777 * the len parameter. No error checking will be done. 778 * 779 * Note that with only one concurrent reader and one concurrent 780 * writer, you don't need extra locking to use these macros. 781 */ 782 #define kfifo_dma_in_finish(fifo, len) \ 783 (void)({ \ 784 typeof((fifo) + 1) __tmp = (fifo); \ 785 unsigned int __len = (len); \ 786 const size_t __recsize = sizeof(*__tmp->rectype); \ 787 struct __kfifo *__kfifo = &__tmp->kfifo; \ 788 if (__recsize) \ 789 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \ 790 else \ 791 __kfifo->in += __len / sizeof(*__tmp->type); \ 792 }) 793 794 /** 795 * kfifo_dma_out_prepare_mapped - setup a scatterlist for DMA output 796 * @fifo: address of the fifo to be used 797 * @sgl: pointer to the scatterlist array 798 * @nents: number of entries in the scatterlist array 799 * @len: number of elements to transfer 800 * @dma: mapped dma address to fill into @sgl 801 * 802 * This macro fills a scatterlist for DMA output which at most @len bytes 803 * to transfer. 804 * It returns the number entries in the scatterlist array. 805 * A zero means there is no space available and the scatterlist is not filled. 806 * 807 * Note that with only one concurrent reader and one concurrent 808 * writer, you don't need extra locking to use these macros. 809 */ 810 #define kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, dma) \ 811 ({ \ 812 typeof((fifo) + 1) __tmp = (fifo); \ 813 struct scatterlist *__sgl = (sgl); \ 814 int __nents = (nents); \ 815 unsigned int __len = (len); \ 816 const size_t __recsize = sizeof(*__tmp->rectype); \ 817 struct __kfifo *__kfifo = &__tmp->kfifo; \ 818 (__recsize) ? \ 819 __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize, \ 820 dma) : \ 821 __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len, dma); \ 822 }) 823 824 #define kfifo_dma_out_prepare(fifo, sgl, nents, len) \ 825 kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR) 826 827 /** 828 * kfifo_dma_out_finish - finish a DMA OUT operation 829 * @fifo: address of the fifo to be used 830 * @len: number of bytes transferred 831 * 832 * This macro finishes a DMA OUT operation. The out counter will be updated by 833 * the len parameter. No error checking will be done. 834 * 835 * Note that with only one concurrent reader and one concurrent 836 * writer, you don't need extra locking to use these macros. 837 */ 838 #define kfifo_dma_out_finish(fifo, len) do { \ 839 typeof((fifo) + 1) ___tmp = (fifo); \ 840 kfifo_skip_count(___tmp, (len) / sizeof(*___tmp->type)); \ 841 } while (0) 842 843 /** 844 * kfifo_out_peek - gets some data from the fifo 845 * @fifo: address of the fifo to be used 846 * @buf: pointer to the storage buffer 847 * @n: max. number of elements to get 848 * 849 * This macro gets the data from the fifo and returns the numbers of elements 850 * copied. The data is not removed from the fifo. 851 * 852 * Note that with only one concurrent reader and one concurrent 853 * writer, you don't need extra locking to use these macro. 854 */ 855 #define kfifo_out_peek(fifo, buf, n) \ 856 __kfifo_uint_must_check_helper( \ 857 ({ \ 858 typeof((fifo) + 1) __tmp = (fifo); \ 859 typeof(__tmp->ptr) __buf = (buf); \ 860 unsigned long __n = (n); \ 861 const size_t __recsize = sizeof(*__tmp->rectype); \ 862 struct __kfifo *__kfifo = &__tmp->kfifo; \ 863 (__recsize) ? \ 864 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \ 865 __kfifo_out_peek(__kfifo, __buf, __n); \ 866 }) \ 867 ) 868 869 /** 870 * kfifo_out_linear - gets a tail of/offset to available data 871 * @fifo: address of the fifo to be used 872 * @tail: pointer to an unsigned int to store the value of tail 873 * @n: max. number of elements to point at 874 * 875 * This macro obtains the offset (tail) to the available data in the fifo 876 * buffer and returns the 877 * numbers of elements available. It returns the available count till the end 878 * of data or till the end of the buffer. So that it can be used for linear 879 * data processing (like memcpy() of (@fifo->data + @tail) with count 880 * returned). 881 * 882 * Note that with only one concurrent reader and one concurrent 883 * writer, you don't need extra locking to use these macro. 884 */ 885 #define kfifo_out_linear(fifo, tail, n) \ 886 __kfifo_uint_must_check_helper( \ 887 ({ \ 888 typeof((fifo) + 1) __tmp = (fifo); \ 889 unsigned int *__tail = (tail); \ 890 unsigned long __n = (n); \ 891 const size_t __recsize = sizeof(*__tmp->rectype); \ 892 struct __kfifo *__kfifo = &__tmp->kfifo; \ 893 (__recsize) ? \ 894 __kfifo_out_linear_r(__kfifo, __tail, __n, __recsize) : \ 895 __kfifo_out_linear(__kfifo, __tail, __n); \ 896 }) \ 897 ) 898 899 /** 900 * kfifo_out_linear_ptr - gets a pointer to the available data 901 * @fifo: address of the fifo to be used 902 * @ptr: pointer to data to store the pointer to tail 903 * @n: max. number of elements to point at 904 * 905 * Similarly to kfifo_out_linear(), this macro obtains the pointer to the 906 * available data in the fifo buffer and returns the numbers of elements 907 * available. It returns the available count till the end of available data or 908 * till the end of the buffer. So that it can be used for linear data 909 * processing (like memcpy() of @ptr with count returned). 910 * 911 * Note that with only one concurrent reader and one concurrent 912 * writer, you don't need extra locking to use these macro. 913 */ 914 #define kfifo_out_linear_ptr(fifo, ptr, n) \ 915 __kfifo_uint_must_check_helper( \ 916 ({ \ 917 typeof((fifo) + 1) ___tmp = (fifo); \ 918 unsigned int ___tail; \ 919 unsigned int ___n = kfifo_out_linear(___tmp, &___tail, (n)); \ 920 *(ptr) = ___tmp->kfifo.data + ___tail * kfifo_esize(___tmp); \ 921 ___n; \ 922 }) \ 923 ) 924 925 926 extern int __kfifo_alloc_node(struct __kfifo *fifo, unsigned int size, 927 size_t esize, gfp_t gfp_mask, int node); 928 929 static inline int __kfifo_alloc(struct __kfifo *fifo, unsigned int size, 930 size_t esize, gfp_t gfp_mask) 931 { 932 return __kfifo_alloc_node(fifo, size, esize, gfp_mask, NUMA_NO_NODE); 933 } 934 935 extern void __kfifo_free(struct __kfifo *fifo); 936 937 extern int __kfifo_init(struct __kfifo *fifo, void *buffer, 938 unsigned int size, size_t esize); 939 940 extern unsigned int __kfifo_in(struct __kfifo *fifo, 941 const void *buf, unsigned int len); 942 943 extern unsigned int __kfifo_out(struct __kfifo *fifo, 944 void *buf, unsigned int len); 945 946 extern int __kfifo_from_user(struct __kfifo *fifo, 947 const void __user *from, unsigned long len, unsigned int *copied); 948 949 extern int __kfifo_to_user(struct __kfifo *fifo, 950 void __user *to, unsigned long len, unsigned int *copied); 951 952 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo, 953 struct scatterlist *sgl, int nents, unsigned int len, dma_addr_t dma); 954 955 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo, 956 struct scatterlist *sgl, int nents, unsigned int len, dma_addr_t dma); 957 958 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo, 959 void *buf, unsigned int len); 960 961 extern unsigned int __kfifo_out_linear(struct __kfifo *fifo, 962 unsigned int *tail, unsigned int n); 963 964 extern unsigned int __kfifo_in_r(struct __kfifo *fifo, 965 const void *buf, unsigned int len, size_t recsize); 966 967 extern unsigned int __kfifo_out_r(struct __kfifo *fifo, 968 void *buf, unsigned int len, size_t recsize); 969 970 extern int __kfifo_from_user_r(struct __kfifo *fifo, 971 const void __user *from, unsigned long len, unsigned int *copied, 972 size_t recsize); 973 974 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to, 975 unsigned long len, unsigned int *copied, size_t recsize); 976 977 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo, 978 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize, 979 dma_addr_t dma); 980 981 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo, 982 unsigned int len, size_t recsize); 983 984 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo, 985 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize, 986 dma_addr_t dma); 987 988 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize); 989 990 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize); 991 992 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo, 993 void *buf, unsigned int len, size_t recsize); 994 995 extern unsigned int __kfifo_out_linear_r(struct __kfifo *fifo, 996 unsigned int *tail, unsigned int n, size_t recsize); 997 998 extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize); 999 1000 #endif 1001