1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* Queue of folios definitions
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #ifndef _LINUX_FOLIO_QUEUE_H
9 #define _LINUX_FOLIO_QUEUE_H
10
11 #include <linux/pagevec.h>
12
13 /*
14 * Segment in a queue of running buffers. Each segment can hold a number of
15 * folios and a portion of the queue can be referenced with the ITER_FOLIOQ
16 * iterator. The possibility exists of inserting non-folio elements into the
17 * queue (such as gaps).
18 *
19 * Explicit prev and next pointers are used instead of a list_head to make it
20 * easier to add segments to tail and remove them from the head without the
21 * need for a lock.
22 */
23 struct folio_queue {
24 struct folio_batch vec; /* Folios in the queue segment */
25 u8 orders[PAGEVEC_SIZE]; /* Order of each folio */
26 struct folio_queue *next; /* Next queue segment or NULL */
27 struct folio_queue *prev; /* Previous queue segment of NULL */
28 unsigned long marks; /* 1-bit mark per folio */
29 unsigned long marks2; /* Second 1-bit mark per folio */
30 unsigned long marks3; /* Third 1-bit mark per folio */
31 #if PAGEVEC_SIZE > BITS_PER_LONG
32 #error marks is not big enough
33 #endif
34 };
35
folioq_init(struct folio_queue * folioq)36 static inline void folioq_init(struct folio_queue *folioq)
37 {
38 folio_batch_init(&folioq->vec);
39 folioq->next = NULL;
40 folioq->prev = NULL;
41 folioq->marks = 0;
42 folioq->marks2 = 0;
43 folioq->marks3 = 0;
44 }
45
folioq_nr_slots(const struct folio_queue * folioq)46 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
47 {
48 return PAGEVEC_SIZE;
49 }
50
folioq_count(struct folio_queue * folioq)51 static inline unsigned int folioq_count(struct folio_queue *folioq)
52 {
53 return folio_batch_count(&folioq->vec);
54 }
55
folioq_full(struct folio_queue * folioq)56 static inline bool folioq_full(struct folio_queue *folioq)
57 {
58 //return !folio_batch_space(&folioq->vec);
59 return folioq_count(folioq) >= folioq_nr_slots(folioq);
60 }
61
folioq_is_marked(const struct folio_queue * folioq,unsigned int slot)62 static inline bool folioq_is_marked(const struct folio_queue *folioq, unsigned int slot)
63 {
64 return test_bit(slot, &folioq->marks);
65 }
66
folioq_mark(struct folio_queue * folioq,unsigned int slot)67 static inline void folioq_mark(struct folio_queue *folioq, unsigned int slot)
68 {
69 set_bit(slot, &folioq->marks);
70 }
71
folioq_unmark(struct folio_queue * folioq,unsigned int slot)72 static inline void folioq_unmark(struct folio_queue *folioq, unsigned int slot)
73 {
74 clear_bit(slot, &folioq->marks);
75 }
76
folioq_is_marked2(const struct folio_queue * folioq,unsigned int slot)77 static inline bool folioq_is_marked2(const struct folio_queue *folioq, unsigned int slot)
78 {
79 return test_bit(slot, &folioq->marks2);
80 }
81
folioq_mark2(struct folio_queue * folioq,unsigned int slot)82 static inline void folioq_mark2(struct folio_queue *folioq, unsigned int slot)
83 {
84 set_bit(slot, &folioq->marks2);
85 }
86
folioq_unmark2(struct folio_queue * folioq,unsigned int slot)87 static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot)
88 {
89 clear_bit(slot, &folioq->marks2);
90 }
91
folioq_is_marked3(const struct folio_queue * folioq,unsigned int slot)92 static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot)
93 {
94 return test_bit(slot, &folioq->marks3);
95 }
96
folioq_mark3(struct folio_queue * folioq,unsigned int slot)97 static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot)
98 {
99 set_bit(slot, &folioq->marks3);
100 }
101
folioq_unmark3(struct folio_queue * folioq,unsigned int slot)102 static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot)
103 {
104 clear_bit(slot, &folioq->marks3);
105 }
106
__folio_order(struct folio * folio)107 static inline unsigned int __folio_order(struct folio *folio)
108 {
109 if (!folio_test_large(folio))
110 return 0;
111 return folio->_flags_1 & 0xff;
112 }
113
folioq_append(struct folio_queue * folioq,struct folio * folio)114 static inline unsigned int folioq_append(struct folio_queue *folioq, struct folio *folio)
115 {
116 unsigned int slot = folioq->vec.nr++;
117
118 folioq->vec.folios[slot] = folio;
119 folioq->orders[slot] = __folio_order(folio);
120 return slot;
121 }
122
folioq_append_mark(struct folio_queue * folioq,struct folio * folio)123 static inline unsigned int folioq_append_mark(struct folio_queue *folioq, struct folio *folio)
124 {
125 unsigned int slot = folioq->vec.nr++;
126
127 folioq->vec.folios[slot] = folio;
128 folioq->orders[slot] = __folio_order(folio);
129 folioq_mark(folioq, slot);
130 return slot;
131 }
132
folioq_folio(const struct folio_queue * folioq,unsigned int slot)133 static inline struct folio *folioq_folio(const struct folio_queue *folioq, unsigned int slot)
134 {
135 return folioq->vec.folios[slot];
136 }
137
folioq_folio_order(const struct folio_queue * folioq,unsigned int slot)138 static inline unsigned int folioq_folio_order(const struct folio_queue *folioq, unsigned int slot)
139 {
140 return folioq->orders[slot];
141 }
142
folioq_folio_size(const struct folio_queue * folioq,unsigned int slot)143 static inline size_t folioq_folio_size(const struct folio_queue *folioq, unsigned int slot)
144 {
145 return PAGE_SIZE << folioq_folio_order(folioq, slot);
146 }
147
folioq_clear(struct folio_queue * folioq,unsigned int slot)148 static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot)
149 {
150 folioq->vec.folios[slot] = NULL;
151 folioq_unmark(folioq, slot);
152 folioq_unmark2(folioq, slot);
153 folioq_unmark3(folioq, slot);
154 }
155
156 #endif /* _LINUX_FOLIO_QUEUE_H */
157