1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Basic tests for PR_GET/SET_THP_DISABLE prctl calls
4 *
5 * Author(s): Usama Arif <usamaarif642@gmail.com>
6 */
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <unistd.h>
11 #include <sys/mman.h>
12 #include <linux/mman.h>
13 #include <sys/prctl.h>
14 #include <sys/wait.h>
15
16 #include "../kselftest_harness.h"
17 #include "thp_settings.h"
18 #include "vm_util.h"
19
20 #ifndef PR_THP_DISABLE_EXCEPT_ADVISED
21 #define PR_THP_DISABLE_EXCEPT_ADVISED (1 << 1)
22 #endif
23
24 enum thp_collapse_type {
25 THP_COLLAPSE_NONE,
26 THP_COLLAPSE_MADV_NOHUGEPAGE,
27 THP_COLLAPSE_MADV_HUGEPAGE, /* MADV_HUGEPAGE before access */
28 THP_COLLAPSE_MADV_COLLAPSE, /* MADV_COLLAPSE after access */
29 };
30
31 /*
32 * Function to mmap a buffer, fault it in, madvise it appropriately (before
33 * page fault for MADV_HUGE, and after for MADV_COLLAPSE), and check if the
34 * mmap region is huge.
35 * Returns:
36 * 0 if test doesn't give hugepage
37 * 1 if test gives a hugepage
38 * -errno if mmap fails
39 */
test_mmap_thp(enum thp_collapse_type madvise_buf,size_t pmdsize)40 static int test_mmap_thp(enum thp_collapse_type madvise_buf, size_t pmdsize)
41 {
42 char *mem, *mmap_mem;
43 size_t mmap_size;
44 int ret;
45
46 /* For alignment purposes, we need twice the THP size. */
47 mmap_size = 2 * pmdsize;
48 mmap_mem = (char *)mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
49 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
50 if (mmap_mem == MAP_FAILED)
51 return -errno;
52
53 /* We need a THP-aligned memory area. */
54 mem = (char *)(((uintptr_t)mmap_mem + pmdsize) & ~(pmdsize - 1));
55
56 if (madvise_buf == THP_COLLAPSE_MADV_HUGEPAGE)
57 madvise(mem, pmdsize, MADV_HUGEPAGE);
58 else if (madvise_buf == THP_COLLAPSE_MADV_NOHUGEPAGE)
59 madvise(mem, pmdsize, MADV_NOHUGEPAGE);
60
61 /* Ensure memory is allocated */
62 memset(mem, 1, pmdsize);
63
64 if (madvise_buf == THP_COLLAPSE_MADV_COLLAPSE)
65 madvise(mem, pmdsize, MADV_COLLAPSE);
66
67 /* HACK: make sure we have a separate VMA that we can check reliably. */
68 mprotect(mem, pmdsize, PROT_READ);
69
70 ret = check_huge_anon(mem, 1, pmdsize);
71 munmap(mmap_mem, mmap_size);
72 return ret;
73 }
74
prctl_thp_disable_completely_test(struct __test_metadata * const _metadata,size_t pmdsize,enum thp_enabled thp_policy)75 static void prctl_thp_disable_completely_test(struct __test_metadata *const _metadata,
76 size_t pmdsize,
77 enum thp_enabled thp_policy)
78 {
79 ASSERT_EQ(prctl(PR_GET_THP_DISABLE, NULL, NULL, NULL, NULL), 1);
80
81 /* tests after prctl overrides global policy */
82 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_NONE, pmdsize), 0);
83
84 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_NOHUGEPAGE, pmdsize), 0);
85
86 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_HUGEPAGE, pmdsize), 0);
87
88 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_COLLAPSE, pmdsize), 0);
89
90 /* Reset to global policy */
91 ASSERT_EQ(prctl(PR_SET_THP_DISABLE, 0, NULL, NULL, NULL), 0);
92
93 /* tests after prctl is cleared, and only global policy is effective */
94 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_NONE, pmdsize),
95 thp_policy == THP_ALWAYS ? 1 : 0);
96
97 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_NOHUGEPAGE, pmdsize), 0);
98
99 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_HUGEPAGE, pmdsize),
100 thp_policy == THP_NEVER ? 0 : 1);
101
102 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_COLLAPSE, pmdsize), 1);
103 }
104
FIXTURE(prctl_thp_disable_completely)105 FIXTURE(prctl_thp_disable_completely)
106 {
107 struct thp_settings settings;
108 size_t pmdsize;
109 };
110
FIXTURE_VARIANT(prctl_thp_disable_completely)111 FIXTURE_VARIANT(prctl_thp_disable_completely)
112 {
113 enum thp_enabled thp_policy;
114 };
115
FIXTURE_VARIANT_ADD(prctl_thp_disable_completely,never)116 FIXTURE_VARIANT_ADD(prctl_thp_disable_completely, never)
117 {
118 .thp_policy = THP_NEVER,
119 };
120
FIXTURE_VARIANT_ADD(prctl_thp_disable_completely,madvise)121 FIXTURE_VARIANT_ADD(prctl_thp_disable_completely, madvise)
122 {
123 .thp_policy = THP_MADVISE,
124 };
125
FIXTURE_VARIANT_ADD(prctl_thp_disable_completely,always)126 FIXTURE_VARIANT_ADD(prctl_thp_disable_completely, always)
127 {
128 .thp_policy = THP_ALWAYS,
129 };
130
FIXTURE_SETUP(prctl_thp_disable_completely)131 FIXTURE_SETUP(prctl_thp_disable_completely)
132 {
133 if (!thp_available())
134 SKIP(return, "Transparent Hugepages not available\n");
135
136 self->pmdsize = read_pmd_pagesize();
137 if (!self->pmdsize)
138 SKIP(return, "Unable to read PMD size\n");
139
140 if (prctl(PR_SET_THP_DISABLE, 1, NULL, NULL, NULL))
141 SKIP(return, "Unable to disable THPs completely for the process\n");
142
143 thp_save_settings();
144 thp_read_settings(&self->settings);
145 self->settings.thp_enabled = variant->thp_policy;
146 self->settings.hugepages[sz2ord(self->pmdsize, getpagesize())].enabled = THP_INHERIT;
147 thp_write_settings(&self->settings);
148 }
149
FIXTURE_TEARDOWN(prctl_thp_disable_completely)150 FIXTURE_TEARDOWN(prctl_thp_disable_completely)
151 {
152 thp_restore_settings();
153 }
154
TEST_F(prctl_thp_disable_completely,nofork)155 TEST_F(prctl_thp_disable_completely, nofork)
156 {
157 prctl_thp_disable_completely_test(_metadata, self->pmdsize, variant->thp_policy);
158 }
159
TEST_F(prctl_thp_disable_completely,fork)160 TEST_F(prctl_thp_disable_completely, fork)
161 {
162 int ret = 0;
163 pid_t pid;
164
165 /* Make sure prctl changes are carried across fork */
166 pid = fork();
167 ASSERT_GE(pid, 0);
168
169 if (!pid) {
170 prctl_thp_disable_completely_test(_metadata, self->pmdsize, variant->thp_policy);
171 return;
172 }
173
174 wait(&ret);
175 if (WIFEXITED(ret))
176 ret = WEXITSTATUS(ret);
177 else
178 ret = -EINVAL;
179 ASSERT_EQ(ret, 0);
180 }
181
prctl_thp_disable_except_madvise_test(struct __test_metadata * const _metadata,size_t pmdsize,enum thp_enabled thp_policy)182 static void prctl_thp_disable_except_madvise_test(struct __test_metadata *const _metadata,
183 size_t pmdsize,
184 enum thp_enabled thp_policy)
185 {
186 ASSERT_EQ(prctl(PR_GET_THP_DISABLE, NULL, NULL, NULL, NULL), 3);
187
188 /* tests after prctl overrides global policy */
189 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_NONE, pmdsize), 0);
190
191 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_NOHUGEPAGE, pmdsize), 0);
192
193 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_HUGEPAGE, pmdsize),
194 thp_policy == THP_NEVER ? 0 : 1);
195
196 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_COLLAPSE, pmdsize), 1);
197
198 /* Reset to global policy */
199 ASSERT_EQ(prctl(PR_SET_THP_DISABLE, 0, NULL, NULL, NULL), 0);
200
201 /* tests after prctl is cleared, and only global policy is effective */
202 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_NONE, pmdsize),
203 thp_policy == THP_ALWAYS ? 1 : 0);
204
205 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_NOHUGEPAGE, pmdsize), 0);
206
207 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_HUGEPAGE, pmdsize),
208 thp_policy == THP_NEVER ? 0 : 1);
209
210 ASSERT_EQ(test_mmap_thp(THP_COLLAPSE_MADV_COLLAPSE, pmdsize), 1);
211 }
212
FIXTURE(prctl_thp_disable_except_madvise)213 FIXTURE(prctl_thp_disable_except_madvise)
214 {
215 struct thp_settings settings;
216 size_t pmdsize;
217 };
218
FIXTURE_VARIANT(prctl_thp_disable_except_madvise)219 FIXTURE_VARIANT(prctl_thp_disable_except_madvise)
220 {
221 enum thp_enabled thp_policy;
222 };
223
FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise,never)224 FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise, never)
225 {
226 .thp_policy = THP_NEVER,
227 };
228
FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise,madvise)229 FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise, madvise)
230 {
231 .thp_policy = THP_MADVISE,
232 };
233
FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise,always)234 FIXTURE_VARIANT_ADD(prctl_thp_disable_except_madvise, always)
235 {
236 .thp_policy = THP_ALWAYS,
237 };
238
FIXTURE_SETUP(prctl_thp_disable_except_madvise)239 FIXTURE_SETUP(prctl_thp_disable_except_madvise)
240 {
241 if (!thp_available())
242 SKIP(return, "Transparent Hugepages not available\n");
243
244 self->pmdsize = read_pmd_pagesize();
245 if (!self->pmdsize)
246 SKIP(return, "Unable to read PMD size\n");
247
248 if (prctl(PR_SET_THP_DISABLE, 1, PR_THP_DISABLE_EXCEPT_ADVISED, NULL, NULL))
249 SKIP(return, "Unable to set PR_THP_DISABLE_EXCEPT_ADVISED\n");
250
251 thp_save_settings();
252 thp_read_settings(&self->settings);
253 self->settings.thp_enabled = variant->thp_policy;
254 self->settings.hugepages[sz2ord(self->pmdsize, getpagesize())].enabled = THP_INHERIT;
255 thp_write_settings(&self->settings);
256 }
257
FIXTURE_TEARDOWN(prctl_thp_disable_except_madvise)258 FIXTURE_TEARDOWN(prctl_thp_disable_except_madvise)
259 {
260 thp_restore_settings();
261 }
262
TEST_F(prctl_thp_disable_except_madvise,nofork)263 TEST_F(prctl_thp_disable_except_madvise, nofork)
264 {
265 prctl_thp_disable_except_madvise_test(_metadata, self->pmdsize, variant->thp_policy);
266 }
267
TEST_F(prctl_thp_disable_except_madvise,fork)268 TEST_F(prctl_thp_disable_except_madvise, fork)
269 {
270 int ret = 0;
271 pid_t pid;
272
273 /* Make sure prctl changes are carried across fork */
274 pid = fork();
275 ASSERT_GE(pid, 0);
276
277 if (!pid) {
278 prctl_thp_disable_except_madvise_test(_metadata, self->pmdsize,
279 variant->thp_policy);
280 return;
281 }
282
283 wait(&ret);
284 if (WIFEXITED(ret))
285 ret = WEXITSTATUS(ret);
286 else
287 ret = -EINVAL;
288 ASSERT_EQ(ret, 0);
289 }
290
291 TEST_HARNESS_MAIN
292