xref: /linux/fs/proc/task_mmu.c (revision 905e46acd3272d04566fec49afbd7ad9e2ed9ae3)
1 #include <linux/mm.h>
2 #include <linux/vmacache.h>
3 #include <linux/hugetlb.h>
4 #include <linux/huge_mm.h>
5 #include <linux/mount.h>
6 #include <linux/seq_file.h>
7 #include <linux/highmem.h>
8 #include <linux/ptrace.h>
9 #include <linux/slab.h>
10 #include <linux/pagemap.h>
11 #include <linux/mempolicy.h>
12 #include <linux/rmap.h>
13 #include <linux/swap.h>
14 #include <linux/sched/mm.h>
15 #include <linux/swapops.h>
16 #include <linux/mmu_notifier.h>
17 #include <linux/page_idle.h>
18 #include <linux/shmem_fs.h>
19 
20 #include <asm/elf.h>
21 #include <linux/uaccess.h>
22 #include <asm/tlbflush.h>
23 #include "internal.h"
24 
25 void task_mem(struct seq_file *m, struct mm_struct *mm)
26 {
27 	unsigned long text, lib, swap, ptes, pmds, anon, file, shmem;
28 	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
29 
30 	anon = get_mm_counter(mm, MM_ANONPAGES);
31 	file = get_mm_counter(mm, MM_FILEPAGES);
32 	shmem = get_mm_counter(mm, MM_SHMEMPAGES);
33 
34 	/*
35 	 * Note: to minimize their overhead, mm maintains hiwater_vm and
36 	 * hiwater_rss only when about to *lower* total_vm or rss.  Any
37 	 * collector of these hiwater stats must therefore get total_vm
38 	 * and rss too, which will usually be the higher.  Barriers? not
39 	 * worth the effort, such snapshots can always be inconsistent.
40 	 */
41 	hiwater_vm = total_vm = mm->total_vm;
42 	if (hiwater_vm < mm->hiwater_vm)
43 		hiwater_vm = mm->hiwater_vm;
44 	hiwater_rss = total_rss = anon + file + shmem;
45 	if (hiwater_rss < mm->hiwater_rss)
46 		hiwater_rss = mm->hiwater_rss;
47 
48 	text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
49 	lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
50 	swap = get_mm_counter(mm, MM_SWAPENTS);
51 	ptes = PTRS_PER_PTE * sizeof(pte_t) * atomic_long_read(&mm->nr_ptes);
52 	pmds = PTRS_PER_PMD * sizeof(pmd_t) * mm_nr_pmds(mm);
53 	seq_printf(m,
54 		"VmPeak:\t%8lu kB\n"
55 		"VmSize:\t%8lu kB\n"
56 		"VmLck:\t%8lu kB\n"
57 		"VmPin:\t%8lu kB\n"
58 		"VmHWM:\t%8lu kB\n"
59 		"VmRSS:\t%8lu kB\n"
60 		"RssAnon:\t%8lu kB\n"
61 		"RssFile:\t%8lu kB\n"
62 		"RssShmem:\t%8lu kB\n"
63 		"VmData:\t%8lu kB\n"
64 		"VmStk:\t%8lu kB\n"
65 		"VmExe:\t%8lu kB\n"
66 		"VmLib:\t%8lu kB\n"
67 		"VmPTE:\t%8lu kB\n"
68 		"VmPMD:\t%8lu kB\n"
69 		"VmSwap:\t%8lu kB\n",
70 		hiwater_vm << (PAGE_SHIFT-10),
71 		total_vm << (PAGE_SHIFT-10),
72 		mm->locked_vm << (PAGE_SHIFT-10),
73 		mm->pinned_vm << (PAGE_SHIFT-10),
74 		hiwater_rss << (PAGE_SHIFT-10),
75 		total_rss << (PAGE_SHIFT-10),
76 		anon << (PAGE_SHIFT-10),
77 		file << (PAGE_SHIFT-10),
78 		shmem << (PAGE_SHIFT-10),
79 		mm->data_vm << (PAGE_SHIFT-10),
80 		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
81 		ptes >> 10,
82 		pmds >> 10,
83 		swap << (PAGE_SHIFT-10));
84 	hugetlb_report_usage(m, mm);
85 }
86 
87 unsigned long task_vsize(struct mm_struct *mm)
88 {
89 	return PAGE_SIZE * mm->total_vm;
90 }
91 
92 unsigned long task_statm(struct mm_struct *mm,
93 			 unsigned long *shared, unsigned long *text,
94 			 unsigned long *data, unsigned long *resident)
95 {
96 	*shared = get_mm_counter(mm, MM_FILEPAGES) +
97 			get_mm_counter(mm, MM_SHMEMPAGES);
98 	*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
99 								>> PAGE_SHIFT;
100 	*data = mm->data_vm + mm->stack_vm;
101 	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
102 	return mm->total_vm;
103 }
104 
105 #ifdef CONFIG_NUMA
106 /*
107  * Save get_task_policy() for show_numa_map().
108  */
109 static void hold_task_mempolicy(struct proc_maps_private *priv)
110 {
111 	struct task_struct *task = priv->task;
112 
113 	task_lock(task);
114 	priv->task_mempolicy = get_task_policy(task);
115 	mpol_get(priv->task_mempolicy);
116 	task_unlock(task);
117 }
118 static void release_task_mempolicy(struct proc_maps_private *priv)
119 {
120 	mpol_put(priv->task_mempolicy);
121 }
122 #else
123 static void hold_task_mempolicy(struct proc_maps_private *priv)
124 {
125 }
126 static void release_task_mempolicy(struct proc_maps_private *priv)
127 {
128 }
129 #endif
130 
131 static void vma_stop(struct proc_maps_private *priv)
132 {
133 	struct mm_struct *mm = priv->mm;
134 
135 	release_task_mempolicy(priv);
136 	up_read(&mm->mmap_sem);
137 	mmput(mm);
138 }
139 
140 static struct vm_area_struct *
141 m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
142 {
143 	if (vma == priv->tail_vma)
144 		return NULL;
145 	return vma->vm_next ?: priv->tail_vma;
146 }
147 
148 static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
149 {
150 	if (m->count < m->size)	/* vma is copied successfully */
151 		m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL;
152 }
153 
154 static void *m_start(struct seq_file *m, loff_t *ppos)
155 {
156 	struct proc_maps_private *priv = m->private;
157 	unsigned long last_addr = m->version;
158 	struct mm_struct *mm;
159 	struct vm_area_struct *vma;
160 	unsigned int pos = *ppos;
161 
162 	/* See m_cache_vma(). Zero at the start or after lseek. */
163 	if (last_addr == -1UL)
164 		return NULL;
165 
166 	priv->task = get_proc_task(priv->inode);
167 	if (!priv->task)
168 		return ERR_PTR(-ESRCH);
169 
170 	mm = priv->mm;
171 	if (!mm || !mmget_not_zero(mm))
172 		return NULL;
173 
174 	down_read(&mm->mmap_sem);
175 	hold_task_mempolicy(priv);
176 	priv->tail_vma = get_gate_vma(mm);
177 
178 	if (last_addr) {
179 		vma = find_vma(mm, last_addr - 1);
180 		if (vma && vma->vm_start <= last_addr)
181 			vma = m_next_vma(priv, vma);
182 		if (vma)
183 			return vma;
184 	}
185 
186 	m->version = 0;
187 	if (pos < mm->map_count) {
188 		for (vma = mm->mmap; pos; pos--) {
189 			m->version = vma->vm_start;
190 			vma = vma->vm_next;
191 		}
192 		return vma;
193 	}
194 
195 	/* we do not bother to update m->version in this case */
196 	if (pos == mm->map_count && priv->tail_vma)
197 		return priv->tail_vma;
198 
199 	vma_stop(priv);
200 	return NULL;
201 }
202 
203 static void *m_next(struct seq_file *m, void *v, loff_t *pos)
204 {
205 	struct proc_maps_private *priv = m->private;
206 	struct vm_area_struct *next;
207 
208 	(*pos)++;
209 	next = m_next_vma(priv, v);
210 	if (!next)
211 		vma_stop(priv);
212 	return next;
213 }
214 
215 static void m_stop(struct seq_file *m, void *v)
216 {
217 	struct proc_maps_private *priv = m->private;
218 
219 	if (!IS_ERR_OR_NULL(v))
220 		vma_stop(priv);
221 	if (priv->task) {
222 		put_task_struct(priv->task);
223 		priv->task = NULL;
224 	}
225 }
226 
227 static int proc_maps_open(struct inode *inode, struct file *file,
228 			const struct seq_operations *ops, int psize)
229 {
230 	struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
231 
232 	if (!priv)
233 		return -ENOMEM;
234 
235 	priv->inode = inode;
236 	priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
237 	if (IS_ERR(priv->mm)) {
238 		int err = PTR_ERR(priv->mm);
239 
240 		seq_release_private(inode, file);
241 		return err;
242 	}
243 
244 	return 0;
245 }
246 
247 static int proc_map_release(struct inode *inode, struct file *file)
248 {
249 	struct seq_file *seq = file->private_data;
250 	struct proc_maps_private *priv = seq->private;
251 
252 	if (priv->mm)
253 		mmdrop(priv->mm);
254 
255 	return seq_release_private(inode, file);
256 }
257 
258 static int do_maps_open(struct inode *inode, struct file *file,
259 			const struct seq_operations *ops)
260 {
261 	return proc_maps_open(inode, file, ops,
262 				sizeof(struct proc_maps_private));
263 }
264 
265 /*
266  * Indicate if the VMA is a stack for the given task; for
267  * /proc/PID/maps that is the stack of the main task.
268  */
269 static int is_stack(struct proc_maps_private *priv,
270 		    struct vm_area_struct *vma)
271 {
272 	/*
273 	 * We make no effort to guess what a given thread considers to be
274 	 * its "stack".  It's not even well-defined for programs written
275 	 * languages like Go.
276 	 */
277 	return vma->vm_start <= vma->vm_mm->start_stack &&
278 		vma->vm_end >= vma->vm_mm->start_stack;
279 }
280 
281 static void
282 show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid)
283 {
284 	struct mm_struct *mm = vma->vm_mm;
285 	struct file *file = vma->vm_file;
286 	struct proc_maps_private *priv = m->private;
287 	vm_flags_t flags = vma->vm_flags;
288 	unsigned long ino = 0;
289 	unsigned long long pgoff = 0;
290 	unsigned long start, end;
291 	dev_t dev = 0;
292 	const char *name = NULL;
293 
294 	if (file) {
295 		struct inode *inode = file_inode(vma->vm_file);
296 		dev = inode->i_sb->s_dev;
297 		ino = inode->i_ino;
298 		pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
299 	}
300 
301 	/* We don't show the stack guard page in /proc/maps */
302 	start = vma->vm_start;
303 	if (stack_guard_page_start(vma, start))
304 		start += PAGE_SIZE;
305 	end = vma->vm_end;
306 	if (stack_guard_page_end(vma, end))
307 		end -= PAGE_SIZE;
308 
309 	seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
310 	seq_printf(m, "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ",
311 			start,
312 			end,
313 			flags & VM_READ ? 'r' : '-',
314 			flags & VM_WRITE ? 'w' : '-',
315 			flags & VM_EXEC ? 'x' : '-',
316 			flags & VM_MAYSHARE ? 's' : 'p',
317 			pgoff,
318 			MAJOR(dev), MINOR(dev), ino);
319 
320 	/*
321 	 * Print the dentry name for named mappings, and a
322 	 * special [heap] marker for the heap:
323 	 */
324 	if (file) {
325 		seq_pad(m, ' ');
326 		seq_file_path(m, file, "\n");
327 		goto done;
328 	}
329 
330 	if (vma->vm_ops && vma->vm_ops->name) {
331 		name = vma->vm_ops->name(vma);
332 		if (name)
333 			goto done;
334 	}
335 
336 	name = arch_vma_name(vma);
337 	if (!name) {
338 		if (!mm) {
339 			name = "[vdso]";
340 			goto done;
341 		}
342 
343 		if (vma->vm_start <= mm->brk &&
344 		    vma->vm_end >= mm->start_brk) {
345 			name = "[heap]";
346 			goto done;
347 		}
348 
349 		if (is_stack(priv, vma))
350 			name = "[stack]";
351 	}
352 
353 done:
354 	if (name) {
355 		seq_pad(m, ' ');
356 		seq_puts(m, name);
357 	}
358 	seq_putc(m, '\n');
359 }
360 
361 static int show_map(struct seq_file *m, void *v, int is_pid)
362 {
363 	show_map_vma(m, v, is_pid);
364 	m_cache_vma(m, v);
365 	return 0;
366 }
367 
368 static int show_pid_map(struct seq_file *m, void *v)
369 {
370 	return show_map(m, v, 1);
371 }
372 
373 static int show_tid_map(struct seq_file *m, void *v)
374 {
375 	return show_map(m, v, 0);
376 }
377 
378 static const struct seq_operations proc_pid_maps_op = {
379 	.start	= m_start,
380 	.next	= m_next,
381 	.stop	= m_stop,
382 	.show	= show_pid_map
383 };
384 
385 static const struct seq_operations proc_tid_maps_op = {
386 	.start	= m_start,
387 	.next	= m_next,
388 	.stop	= m_stop,
389 	.show	= show_tid_map
390 };
391 
392 static int pid_maps_open(struct inode *inode, struct file *file)
393 {
394 	return do_maps_open(inode, file, &proc_pid_maps_op);
395 }
396 
397 static int tid_maps_open(struct inode *inode, struct file *file)
398 {
399 	return do_maps_open(inode, file, &proc_tid_maps_op);
400 }
401 
402 const struct file_operations proc_pid_maps_operations = {
403 	.open		= pid_maps_open,
404 	.read		= seq_read,
405 	.llseek		= seq_lseek,
406 	.release	= proc_map_release,
407 };
408 
409 const struct file_operations proc_tid_maps_operations = {
410 	.open		= tid_maps_open,
411 	.read		= seq_read,
412 	.llseek		= seq_lseek,
413 	.release	= proc_map_release,
414 };
415 
416 /*
417  * Proportional Set Size(PSS): my share of RSS.
418  *
419  * PSS of a process is the count of pages it has in memory, where each
420  * page is divided by the number of processes sharing it.  So if a
421  * process has 1000 pages all to itself, and 1000 shared with one other
422  * process, its PSS will be 1500.
423  *
424  * To keep (accumulated) division errors low, we adopt a 64bit
425  * fixed-point pss counter to minimize division errors. So (pss >>
426  * PSS_SHIFT) would be the real byte count.
427  *
428  * A shift of 12 before division means (assuming 4K page size):
429  * 	- 1M 3-user-pages add up to 8KB errors;
430  * 	- supports mapcount up to 2^24, or 16M;
431  * 	- supports PSS up to 2^52 bytes, or 4PB.
432  */
433 #define PSS_SHIFT 12
434 
435 #ifdef CONFIG_PROC_PAGE_MONITOR
436 struct mem_size_stats {
437 	unsigned long resident;
438 	unsigned long shared_clean;
439 	unsigned long shared_dirty;
440 	unsigned long private_clean;
441 	unsigned long private_dirty;
442 	unsigned long referenced;
443 	unsigned long anonymous;
444 	unsigned long lazyfree;
445 	unsigned long anonymous_thp;
446 	unsigned long shmem_thp;
447 	unsigned long swap;
448 	unsigned long shared_hugetlb;
449 	unsigned long private_hugetlb;
450 	u64 pss;
451 	u64 swap_pss;
452 	bool check_shmem_swap;
453 };
454 
455 static void smaps_account(struct mem_size_stats *mss, struct page *page,
456 		bool compound, bool young, bool dirty)
457 {
458 	int i, nr = compound ? 1 << compound_order(page) : 1;
459 	unsigned long size = nr * PAGE_SIZE;
460 
461 	if (PageAnon(page)) {
462 		mss->anonymous += size;
463 		if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
464 			mss->lazyfree += size;
465 	}
466 
467 	mss->resident += size;
468 	/* Accumulate the size in pages that have been accessed. */
469 	if (young || page_is_young(page) || PageReferenced(page))
470 		mss->referenced += size;
471 
472 	/*
473 	 * page_count(page) == 1 guarantees the page is mapped exactly once.
474 	 * If any subpage of the compound page mapped with PTE it would elevate
475 	 * page_count().
476 	 */
477 	if (page_count(page) == 1) {
478 		if (dirty || PageDirty(page))
479 			mss->private_dirty += size;
480 		else
481 			mss->private_clean += size;
482 		mss->pss += (u64)size << PSS_SHIFT;
483 		return;
484 	}
485 
486 	for (i = 0; i < nr; i++, page++) {
487 		int mapcount = page_mapcount(page);
488 
489 		if (mapcount >= 2) {
490 			if (dirty || PageDirty(page))
491 				mss->shared_dirty += PAGE_SIZE;
492 			else
493 				mss->shared_clean += PAGE_SIZE;
494 			mss->pss += (PAGE_SIZE << PSS_SHIFT) / mapcount;
495 		} else {
496 			if (dirty || PageDirty(page))
497 				mss->private_dirty += PAGE_SIZE;
498 			else
499 				mss->private_clean += PAGE_SIZE;
500 			mss->pss += PAGE_SIZE << PSS_SHIFT;
501 		}
502 	}
503 }
504 
505 #ifdef CONFIG_SHMEM
506 static int smaps_pte_hole(unsigned long addr, unsigned long end,
507 		struct mm_walk *walk)
508 {
509 	struct mem_size_stats *mss = walk->private;
510 
511 	mss->swap += shmem_partial_swap_usage(
512 			walk->vma->vm_file->f_mapping, addr, end);
513 
514 	return 0;
515 }
516 #endif
517 
518 static void smaps_pte_entry(pte_t *pte, unsigned long addr,
519 		struct mm_walk *walk)
520 {
521 	struct mem_size_stats *mss = walk->private;
522 	struct vm_area_struct *vma = walk->vma;
523 	struct page *page = NULL;
524 
525 	if (pte_present(*pte)) {
526 		page = vm_normal_page(vma, addr, *pte);
527 	} else if (is_swap_pte(*pte)) {
528 		swp_entry_t swpent = pte_to_swp_entry(*pte);
529 
530 		if (!non_swap_entry(swpent)) {
531 			int mapcount;
532 
533 			mss->swap += PAGE_SIZE;
534 			mapcount = swp_swapcount(swpent);
535 			if (mapcount >= 2) {
536 				u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
537 
538 				do_div(pss_delta, mapcount);
539 				mss->swap_pss += pss_delta;
540 			} else {
541 				mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
542 			}
543 		} else if (is_migration_entry(swpent))
544 			page = migration_entry_to_page(swpent);
545 	} else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
546 							&& pte_none(*pte))) {
547 		page = find_get_entry(vma->vm_file->f_mapping,
548 						linear_page_index(vma, addr));
549 		if (!page)
550 			return;
551 
552 		if (radix_tree_exceptional_entry(page))
553 			mss->swap += PAGE_SIZE;
554 		else
555 			put_page(page);
556 
557 		return;
558 	}
559 
560 	if (!page)
561 		return;
562 
563 	smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte));
564 }
565 
566 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
567 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
568 		struct mm_walk *walk)
569 {
570 	struct mem_size_stats *mss = walk->private;
571 	struct vm_area_struct *vma = walk->vma;
572 	struct page *page;
573 
574 	/* FOLL_DUMP will return -EFAULT on huge zero page */
575 	page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
576 	if (IS_ERR_OR_NULL(page))
577 		return;
578 	if (PageAnon(page))
579 		mss->anonymous_thp += HPAGE_PMD_SIZE;
580 	else if (PageSwapBacked(page))
581 		mss->shmem_thp += HPAGE_PMD_SIZE;
582 	else if (is_zone_device_page(page))
583 		/* pass */;
584 	else
585 		VM_BUG_ON_PAGE(1, page);
586 	smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd));
587 }
588 #else
589 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
590 		struct mm_walk *walk)
591 {
592 }
593 #endif
594 
595 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
596 			   struct mm_walk *walk)
597 {
598 	struct vm_area_struct *vma = walk->vma;
599 	pte_t *pte;
600 	spinlock_t *ptl;
601 
602 	ptl = pmd_trans_huge_lock(pmd, vma);
603 	if (ptl) {
604 		smaps_pmd_entry(pmd, addr, walk);
605 		spin_unlock(ptl);
606 		return 0;
607 	}
608 
609 	if (pmd_trans_unstable(pmd))
610 		return 0;
611 	/*
612 	 * The mmap_sem held all the way back in m_start() is what
613 	 * keeps khugepaged out of here and from collapsing things
614 	 * in here.
615 	 */
616 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
617 	for (; addr != end; pte++, addr += PAGE_SIZE)
618 		smaps_pte_entry(pte, addr, walk);
619 	pte_unmap_unlock(pte - 1, ptl);
620 	cond_resched();
621 	return 0;
622 }
623 
624 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
625 {
626 	/*
627 	 * Don't forget to update Documentation/ on changes.
628 	 */
629 	static const char mnemonics[BITS_PER_LONG][2] = {
630 		/*
631 		 * In case if we meet a flag we don't know about.
632 		 */
633 		[0 ... (BITS_PER_LONG-1)] = "??",
634 
635 		[ilog2(VM_READ)]	= "rd",
636 		[ilog2(VM_WRITE)]	= "wr",
637 		[ilog2(VM_EXEC)]	= "ex",
638 		[ilog2(VM_SHARED)]	= "sh",
639 		[ilog2(VM_MAYREAD)]	= "mr",
640 		[ilog2(VM_MAYWRITE)]	= "mw",
641 		[ilog2(VM_MAYEXEC)]	= "me",
642 		[ilog2(VM_MAYSHARE)]	= "ms",
643 		[ilog2(VM_GROWSDOWN)]	= "gd",
644 		[ilog2(VM_PFNMAP)]	= "pf",
645 		[ilog2(VM_DENYWRITE)]	= "dw",
646 #ifdef CONFIG_X86_INTEL_MPX
647 		[ilog2(VM_MPX)]		= "mp",
648 #endif
649 		[ilog2(VM_LOCKED)]	= "lo",
650 		[ilog2(VM_IO)]		= "io",
651 		[ilog2(VM_SEQ_READ)]	= "sr",
652 		[ilog2(VM_RAND_READ)]	= "rr",
653 		[ilog2(VM_DONTCOPY)]	= "dc",
654 		[ilog2(VM_DONTEXPAND)]	= "de",
655 		[ilog2(VM_ACCOUNT)]	= "ac",
656 		[ilog2(VM_NORESERVE)]	= "nr",
657 		[ilog2(VM_HUGETLB)]	= "ht",
658 		[ilog2(VM_ARCH_1)]	= "ar",
659 		[ilog2(VM_DONTDUMP)]	= "dd",
660 #ifdef CONFIG_MEM_SOFT_DIRTY
661 		[ilog2(VM_SOFTDIRTY)]	= "sd",
662 #endif
663 		[ilog2(VM_MIXEDMAP)]	= "mm",
664 		[ilog2(VM_HUGEPAGE)]	= "hg",
665 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
666 		[ilog2(VM_MERGEABLE)]	= "mg",
667 		[ilog2(VM_UFFD_MISSING)]= "um",
668 		[ilog2(VM_UFFD_WP)]	= "uw",
669 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
670 		/* These come out via ProtectionKey: */
671 		[ilog2(VM_PKEY_BIT0)]	= "",
672 		[ilog2(VM_PKEY_BIT1)]	= "",
673 		[ilog2(VM_PKEY_BIT2)]	= "",
674 		[ilog2(VM_PKEY_BIT3)]	= "",
675 #endif
676 	};
677 	size_t i;
678 
679 	seq_puts(m, "VmFlags: ");
680 	for (i = 0; i < BITS_PER_LONG; i++) {
681 		if (!mnemonics[i][0])
682 			continue;
683 		if (vma->vm_flags & (1UL << i)) {
684 			seq_printf(m, "%c%c ",
685 				   mnemonics[i][0], mnemonics[i][1]);
686 		}
687 	}
688 	seq_putc(m, '\n');
689 }
690 
691 #ifdef CONFIG_HUGETLB_PAGE
692 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
693 				 unsigned long addr, unsigned long end,
694 				 struct mm_walk *walk)
695 {
696 	struct mem_size_stats *mss = walk->private;
697 	struct vm_area_struct *vma = walk->vma;
698 	struct page *page = NULL;
699 
700 	if (pte_present(*pte)) {
701 		page = vm_normal_page(vma, addr, *pte);
702 	} else if (is_swap_pte(*pte)) {
703 		swp_entry_t swpent = pte_to_swp_entry(*pte);
704 
705 		if (is_migration_entry(swpent))
706 			page = migration_entry_to_page(swpent);
707 	}
708 	if (page) {
709 		int mapcount = page_mapcount(page);
710 
711 		if (mapcount >= 2)
712 			mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
713 		else
714 			mss->private_hugetlb += huge_page_size(hstate_vma(vma));
715 	}
716 	return 0;
717 }
718 #endif /* HUGETLB_PAGE */
719 
720 void __weak arch_show_smap(struct seq_file *m, struct vm_area_struct *vma)
721 {
722 }
723 
724 static int show_smap(struct seq_file *m, void *v, int is_pid)
725 {
726 	struct vm_area_struct *vma = v;
727 	struct mem_size_stats mss;
728 	struct mm_walk smaps_walk = {
729 		.pmd_entry = smaps_pte_range,
730 #ifdef CONFIG_HUGETLB_PAGE
731 		.hugetlb_entry = smaps_hugetlb_range,
732 #endif
733 		.mm = vma->vm_mm,
734 		.private = &mss,
735 	};
736 
737 	memset(&mss, 0, sizeof mss);
738 
739 #ifdef CONFIG_SHMEM
740 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
741 		/*
742 		 * For shared or readonly shmem mappings we know that all
743 		 * swapped out pages belong to the shmem object, and we can
744 		 * obtain the swap value much more efficiently. For private
745 		 * writable mappings, we might have COW pages that are
746 		 * not affected by the parent swapped out pages of the shmem
747 		 * object, so we have to distinguish them during the page walk.
748 		 * Unless we know that the shmem object (or the part mapped by
749 		 * our VMA) has no swapped out pages at all.
750 		 */
751 		unsigned long shmem_swapped = shmem_swap_usage(vma);
752 
753 		if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
754 					!(vma->vm_flags & VM_WRITE)) {
755 			mss.swap = shmem_swapped;
756 		} else {
757 			mss.check_shmem_swap = true;
758 			smaps_walk.pte_hole = smaps_pte_hole;
759 		}
760 	}
761 #endif
762 
763 	/* mmap_sem is held in m_start */
764 	walk_page_vma(vma, &smaps_walk);
765 
766 	show_map_vma(m, vma, is_pid);
767 
768 	seq_printf(m,
769 		   "Size:           %8lu kB\n"
770 		   "Rss:            %8lu kB\n"
771 		   "Pss:            %8lu kB\n"
772 		   "Shared_Clean:   %8lu kB\n"
773 		   "Shared_Dirty:   %8lu kB\n"
774 		   "Private_Clean:  %8lu kB\n"
775 		   "Private_Dirty:  %8lu kB\n"
776 		   "Referenced:     %8lu kB\n"
777 		   "Anonymous:      %8lu kB\n"
778 		   "LazyFree:       %8lu kB\n"
779 		   "AnonHugePages:  %8lu kB\n"
780 		   "ShmemPmdMapped: %8lu kB\n"
781 		   "Shared_Hugetlb: %8lu kB\n"
782 		   "Private_Hugetlb: %7lu kB\n"
783 		   "Swap:           %8lu kB\n"
784 		   "SwapPss:        %8lu kB\n"
785 		   "KernelPageSize: %8lu kB\n"
786 		   "MMUPageSize:    %8lu kB\n"
787 		   "Locked:         %8lu kB\n",
788 		   (vma->vm_end - vma->vm_start) >> 10,
789 		   mss.resident >> 10,
790 		   (unsigned long)(mss.pss >> (10 + PSS_SHIFT)),
791 		   mss.shared_clean  >> 10,
792 		   mss.shared_dirty  >> 10,
793 		   mss.private_clean >> 10,
794 		   mss.private_dirty >> 10,
795 		   mss.referenced >> 10,
796 		   mss.anonymous >> 10,
797 		   mss.lazyfree >> 10,
798 		   mss.anonymous_thp >> 10,
799 		   mss.shmem_thp >> 10,
800 		   mss.shared_hugetlb >> 10,
801 		   mss.private_hugetlb >> 10,
802 		   mss.swap >> 10,
803 		   (unsigned long)(mss.swap_pss >> (10 + PSS_SHIFT)),
804 		   vma_kernel_pagesize(vma) >> 10,
805 		   vma_mmu_pagesize(vma) >> 10,
806 		   (vma->vm_flags & VM_LOCKED) ?
807 			(unsigned long)(mss.pss >> (10 + PSS_SHIFT)) : 0);
808 
809 	arch_show_smap(m, vma);
810 	show_smap_vma_flags(m, vma);
811 	m_cache_vma(m, vma);
812 	return 0;
813 }
814 
815 static int show_pid_smap(struct seq_file *m, void *v)
816 {
817 	return show_smap(m, v, 1);
818 }
819 
820 static int show_tid_smap(struct seq_file *m, void *v)
821 {
822 	return show_smap(m, v, 0);
823 }
824 
825 static const struct seq_operations proc_pid_smaps_op = {
826 	.start	= m_start,
827 	.next	= m_next,
828 	.stop	= m_stop,
829 	.show	= show_pid_smap
830 };
831 
832 static const struct seq_operations proc_tid_smaps_op = {
833 	.start	= m_start,
834 	.next	= m_next,
835 	.stop	= m_stop,
836 	.show	= show_tid_smap
837 };
838 
839 static int pid_smaps_open(struct inode *inode, struct file *file)
840 {
841 	return do_maps_open(inode, file, &proc_pid_smaps_op);
842 }
843 
844 static int tid_smaps_open(struct inode *inode, struct file *file)
845 {
846 	return do_maps_open(inode, file, &proc_tid_smaps_op);
847 }
848 
849 const struct file_operations proc_pid_smaps_operations = {
850 	.open		= pid_smaps_open,
851 	.read		= seq_read,
852 	.llseek		= seq_lseek,
853 	.release	= proc_map_release,
854 };
855 
856 const struct file_operations proc_tid_smaps_operations = {
857 	.open		= tid_smaps_open,
858 	.read		= seq_read,
859 	.llseek		= seq_lseek,
860 	.release	= proc_map_release,
861 };
862 
863 enum clear_refs_types {
864 	CLEAR_REFS_ALL = 1,
865 	CLEAR_REFS_ANON,
866 	CLEAR_REFS_MAPPED,
867 	CLEAR_REFS_SOFT_DIRTY,
868 	CLEAR_REFS_MM_HIWATER_RSS,
869 	CLEAR_REFS_LAST,
870 };
871 
872 struct clear_refs_private {
873 	enum clear_refs_types type;
874 };
875 
876 #ifdef CONFIG_MEM_SOFT_DIRTY
877 static inline void clear_soft_dirty(struct vm_area_struct *vma,
878 		unsigned long addr, pte_t *pte)
879 {
880 	/*
881 	 * The soft-dirty tracker uses #PF-s to catch writes
882 	 * to pages, so write-protect the pte as well. See the
883 	 * Documentation/vm/soft-dirty.txt for full description
884 	 * of how soft-dirty works.
885 	 */
886 	pte_t ptent = *pte;
887 
888 	if (pte_present(ptent)) {
889 		ptent = ptep_modify_prot_start(vma->vm_mm, addr, pte);
890 		ptent = pte_wrprotect(ptent);
891 		ptent = pte_clear_soft_dirty(ptent);
892 		ptep_modify_prot_commit(vma->vm_mm, addr, pte, ptent);
893 	} else if (is_swap_pte(ptent)) {
894 		ptent = pte_swp_clear_soft_dirty(ptent);
895 		set_pte_at(vma->vm_mm, addr, pte, ptent);
896 	}
897 }
898 #else
899 static inline void clear_soft_dirty(struct vm_area_struct *vma,
900 		unsigned long addr, pte_t *pte)
901 {
902 }
903 #endif
904 
905 #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
906 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
907 		unsigned long addr, pmd_t *pmdp)
908 {
909 	pmd_t pmd = *pmdp;
910 
911 	/* See comment in change_huge_pmd() */
912 	pmdp_invalidate(vma, addr, pmdp);
913 	if (pmd_dirty(*pmdp))
914 		pmd = pmd_mkdirty(pmd);
915 	if (pmd_young(*pmdp))
916 		pmd = pmd_mkyoung(pmd);
917 
918 	pmd = pmd_wrprotect(pmd);
919 	pmd = pmd_clear_soft_dirty(pmd);
920 
921 	set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
922 }
923 #else
924 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
925 		unsigned long addr, pmd_t *pmdp)
926 {
927 }
928 #endif
929 
930 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
931 				unsigned long end, struct mm_walk *walk)
932 {
933 	struct clear_refs_private *cp = walk->private;
934 	struct vm_area_struct *vma = walk->vma;
935 	pte_t *pte, ptent;
936 	spinlock_t *ptl;
937 	struct page *page;
938 
939 	ptl = pmd_trans_huge_lock(pmd, vma);
940 	if (ptl) {
941 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
942 			clear_soft_dirty_pmd(vma, addr, pmd);
943 			goto out;
944 		}
945 
946 		page = pmd_page(*pmd);
947 
948 		/* Clear accessed and referenced bits. */
949 		pmdp_test_and_clear_young(vma, addr, pmd);
950 		test_and_clear_page_young(page);
951 		ClearPageReferenced(page);
952 out:
953 		spin_unlock(ptl);
954 		return 0;
955 	}
956 
957 	if (pmd_trans_unstable(pmd))
958 		return 0;
959 
960 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
961 	for (; addr != end; pte++, addr += PAGE_SIZE) {
962 		ptent = *pte;
963 
964 		if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
965 			clear_soft_dirty(vma, addr, pte);
966 			continue;
967 		}
968 
969 		if (!pte_present(ptent))
970 			continue;
971 
972 		page = vm_normal_page(vma, addr, ptent);
973 		if (!page)
974 			continue;
975 
976 		/* Clear accessed and referenced bits. */
977 		ptep_test_and_clear_young(vma, addr, pte);
978 		test_and_clear_page_young(page);
979 		ClearPageReferenced(page);
980 	}
981 	pte_unmap_unlock(pte - 1, ptl);
982 	cond_resched();
983 	return 0;
984 }
985 
986 static int clear_refs_test_walk(unsigned long start, unsigned long end,
987 				struct mm_walk *walk)
988 {
989 	struct clear_refs_private *cp = walk->private;
990 	struct vm_area_struct *vma = walk->vma;
991 
992 	if (vma->vm_flags & VM_PFNMAP)
993 		return 1;
994 
995 	/*
996 	 * Writing 1 to /proc/pid/clear_refs affects all pages.
997 	 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
998 	 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
999 	 * Writing 4 to /proc/pid/clear_refs affects all pages.
1000 	 */
1001 	if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1002 		return 1;
1003 	if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1004 		return 1;
1005 	return 0;
1006 }
1007 
1008 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1009 				size_t count, loff_t *ppos)
1010 {
1011 	struct task_struct *task;
1012 	char buffer[PROC_NUMBUF];
1013 	struct mm_struct *mm;
1014 	struct vm_area_struct *vma;
1015 	enum clear_refs_types type;
1016 	int itype;
1017 	int rv;
1018 
1019 	memset(buffer, 0, sizeof(buffer));
1020 	if (count > sizeof(buffer) - 1)
1021 		count = sizeof(buffer) - 1;
1022 	if (copy_from_user(buffer, buf, count))
1023 		return -EFAULT;
1024 	rv = kstrtoint(strstrip(buffer), 10, &itype);
1025 	if (rv < 0)
1026 		return rv;
1027 	type = (enum clear_refs_types)itype;
1028 	if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1029 		return -EINVAL;
1030 
1031 	task = get_proc_task(file_inode(file));
1032 	if (!task)
1033 		return -ESRCH;
1034 	mm = get_task_mm(task);
1035 	if (mm) {
1036 		struct clear_refs_private cp = {
1037 			.type = type,
1038 		};
1039 		struct mm_walk clear_refs_walk = {
1040 			.pmd_entry = clear_refs_pte_range,
1041 			.test_walk = clear_refs_test_walk,
1042 			.mm = mm,
1043 			.private = &cp,
1044 		};
1045 
1046 		if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1047 			if (down_write_killable(&mm->mmap_sem)) {
1048 				count = -EINTR;
1049 				goto out_mm;
1050 			}
1051 
1052 			/*
1053 			 * Writing 5 to /proc/pid/clear_refs resets the peak
1054 			 * resident set size to this mm's current rss value.
1055 			 */
1056 			reset_mm_hiwater_rss(mm);
1057 			up_write(&mm->mmap_sem);
1058 			goto out_mm;
1059 		}
1060 
1061 		down_read(&mm->mmap_sem);
1062 		if (type == CLEAR_REFS_SOFT_DIRTY) {
1063 			for (vma = mm->mmap; vma; vma = vma->vm_next) {
1064 				if (!(vma->vm_flags & VM_SOFTDIRTY))
1065 					continue;
1066 				up_read(&mm->mmap_sem);
1067 				if (down_write_killable(&mm->mmap_sem)) {
1068 					count = -EINTR;
1069 					goto out_mm;
1070 				}
1071 				for (vma = mm->mmap; vma; vma = vma->vm_next) {
1072 					vma->vm_flags &= ~VM_SOFTDIRTY;
1073 					vma_set_page_prot(vma);
1074 				}
1075 				downgrade_write(&mm->mmap_sem);
1076 				break;
1077 			}
1078 			mmu_notifier_invalidate_range_start(mm, 0, -1);
1079 		}
1080 		walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
1081 		if (type == CLEAR_REFS_SOFT_DIRTY)
1082 			mmu_notifier_invalidate_range_end(mm, 0, -1);
1083 		flush_tlb_mm(mm);
1084 		up_read(&mm->mmap_sem);
1085 out_mm:
1086 		mmput(mm);
1087 	}
1088 	put_task_struct(task);
1089 
1090 	return count;
1091 }
1092 
1093 const struct file_operations proc_clear_refs_operations = {
1094 	.write		= clear_refs_write,
1095 	.llseek		= noop_llseek,
1096 };
1097 
1098 typedef struct {
1099 	u64 pme;
1100 } pagemap_entry_t;
1101 
1102 struct pagemapread {
1103 	int pos, len;		/* units: PM_ENTRY_BYTES, not bytes */
1104 	pagemap_entry_t *buffer;
1105 	bool show_pfn;
1106 };
1107 
1108 #define PAGEMAP_WALK_SIZE	(PMD_SIZE)
1109 #define PAGEMAP_WALK_MASK	(PMD_MASK)
1110 
1111 #define PM_ENTRY_BYTES		sizeof(pagemap_entry_t)
1112 #define PM_PFRAME_BITS		55
1113 #define PM_PFRAME_MASK		GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1114 #define PM_SOFT_DIRTY		BIT_ULL(55)
1115 #define PM_MMAP_EXCLUSIVE	BIT_ULL(56)
1116 #define PM_FILE			BIT_ULL(61)
1117 #define PM_SWAP			BIT_ULL(62)
1118 #define PM_PRESENT		BIT_ULL(63)
1119 
1120 #define PM_END_OF_BUFFER    1
1121 
1122 static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1123 {
1124 	return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1125 }
1126 
1127 static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1128 			  struct pagemapread *pm)
1129 {
1130 	pm->buffer[pm->pos++] = *pme;
1131 	if (pm->pos >= pm->len)
1132 		return PM_END_OF_BUFFER;
1133 	return 0;
1134 }
1135 
1136 static int pagemap_pte_hole(unsigned long start, unsigned long end,
1137 				struct mm_walk *walk)
1138 {
1139 	struct pagemapread *pm = walk->private;
1140 	unsigned long addr = start;
1141 	int err = 0;
1142 
1143 	while (addr < end) {
1144 		struct vm_area_struct *vma = find_vma(walk->mm, addr);
1145 		pagemap_entry_t pme = make_pme(0, 0);
1146 		/* End of address space hole, which we mark as non-present. */
1147 		unsigned long hole_end;
1148 
1149 		if (vma)
1150 			hole_end = min(end, vma->vm_start);
1151 		else
1152 			hole_end = end;
1153 
1154 		for (; addr < hole_end; addr += PAGE_SIZE) {
1155 			err = add_to_pagemap(addr, &pme, pm);
1156 			if (err)
1157 				goto out;
1158 		}
1159 
1160 		if (!vma)
1161 			break;
1162 
1163 		/* Addresses in the VMA. */
1164 		if (vma->vm_flags & VM_SOFTDIRTY)
1165 			pme = make_pme(0, PM_SOFT_DIRTY);
1166 		for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1167 			err = add_to_pagemap(addr, &pme, pm);
1168 			if (err)
1169 				goto out;
1170 		}
1171 	}
1172 out:
1173 	return err;
1174 }
1175 
1176 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1177 		struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1178 {
1179 	u64 frame = 0, flags = 0;
1180 	struct page *page = NULL;
1181 
1182 	if (pte_present(pte)) {
1183 		if (pm->show_pfn)
1184 			frame = pte_pfn(pte);
1185 		flags |= PM_PRESENT;
1186 		page = vm_normal_page(vma, addr, pte);
1187 		if (pte_soft_dirty(pte))
1188 			flags |= PM_SOFT_DIRTY;
1189 	} else if (is_swap_pte(pte)) {
1190 		swp_entry_t entry;
1191 		if (pte_swp_soft_dirty(pte))
1192 			flags |= PM_SOFT_DIRTY;
1193 		entry = pte_to_swp_entry(pte);
1194 		frame = swp_type(entry) |
1195 			(swp_offset(entry) << MAX_SWAPFILES_SHIFT);
1196 		flags |= PM_SWAP;
1197 		if (is_migration_entry(entry))
1198 			page = migration_entry_to_page(entry);
1199 	}
1200 
1201 	if (page && !PageAnon(page))
1202 		flags |= PM_FILE;
1203 	if (page && page_mapcount(page) == 1)
1204 		flags |= PM_MMAP_EXCLUSIVE;
1205 	if (vma->vm_flags & VM_SOFTDIRTY)
1206 		flags |= PM_SOFT_DIRTY;
1207 
1208 	return make_pme(frame, flags);
1209 }
1210 
1211 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1212 			     struct mm_walk *walk)
1213 {
1214 	struct vm_area_struct *vma = walk->vma;
1215 	struct pagemapread *pm = walk->private;
1216 	spinlock_t *ptl;
1217 	pte_t *pte, *orig_pte;
1218 	int err = 0;
1219 
1220 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1221 	ptl = pmd_trans_huge_lock(pmdp, vma);
1222 	if (ptl) {
1223 		u64 flags = 0, frame = 0;
1224 		pmd_t pmd = *pmdp;
1225 
1226 		if ((vma->vm_flags & VM_SOFTDIRTY) || pmd_soft_dirty(pmd))
1227 			flags |= PM_SOFT_DIRTY;
1228 
1229 		/*
1230 		 * Currently pmd for thp is always present because thp
1231 		 * can not be swapped-out, migrated, or HWPOISONed
1232 		 * (split in such cases instead.)
1233 		 * This if-check is just to prepare for future implementation.
1234 		 */
1235 		if (pmd_present(pmd)) {
1236 			struct page *page = pmd_page(pmd);
1237 
1238 			if (page_mapcount(page) == 1)
1239 				flags |= PM_MMAP_EXCLUSIVE;
1240 
1241 			flags |= PM_PRESENT;
1242 			if (pm->show_pfn)
1243 				frame = pmd_pfn(pmd) +
1244 					((addr & ~PMD_MASK) >> PAGE_SHIFT);
1245 		}
1246 
1247 		for (; addr != end; addr += PAGE_SIZE) {
1248 			pagemap_entry_t pme = make_pme(frame, flags);
1249 
1250 			err = add_to_pagemap(addr, &pme, pm);
1251 			if (err)
1252 				break;
1253 			if (pm->show_pfn && (flags & PM_PRESENT))
1254 				frame++;
1255 		}
1256 		spin_unlock(ptl);
1257 		return err;
1258 	}
1259 
1260 	if (pmd_trans_unstable(pmdp))
1261 		return 0;
1262 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1263 
1264 	/*
1265 	 * We can assume that @vma always points to a valid one and @end never
1266 	 * goes beyond vma->vm_end.
1267 	 */
1268 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1269 	for (; addr < end; pte++, addr += PAGE_SIZE) {
1270 		pagemap_entry_t pme;
1271 
1272 		pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1273 		err = add_to_pagemap(addr, &pme, pm);
1274 		if (err)
1275 			break;
1276 	}
1277 	pte_unmap_unlock(orig_pte, ptl);
1278 
1279 	cond_resched();
1280 
1281 	return err;
1282 }
1283 
1284 #ifdef CONFIG_HUGETLB_PAGE
1285 /* This function walks within one hugetlb entry in the single call */
1286 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1287 				 unsigned long addr, unsigned long end,
1288 				 struct mm_walk *walk)
1289 {
1290 	struct pagemapread *pm = walk->private;
1291 	struct vm_area_struct *vma = walk->vma;
1292 	u64 flags = 0, frame = 0;
1293 	int err = 0;
1294 	pte_t pte;
1295 
1296 	if (vma->vm_flags & VM_SOFTDIRTY)
1297 		flags |= PM_SOFT_DIRTY;
1298 
1299 	pte = huge_ptep_get(ptep);
1300 	if (pte_present(pte)) {
1301 		struct page *page = pte_page(pte);
1302 
1303 		if (!PageAnon(page))
1304 			flags |= PM_FILE;
1305 
1306 		if (page_mapcount(page) == 1)
1307 			flags |= PM_MMAP_EXCLUSIVE;
1308 
1309 		flags |= PM_PRESENT;
1310 		if (pm->show_pfn)
1311 			frame = pte_pfn(pte) +
1312 				((addr & ~hmask) >> PAGE_SHIFT);
1313 	}
1314 
1315 	for (; addr != end; addr += PAGE_SIZE) {
1316 		pagemap_entry_t pme = make_pme(frame, flags);
1317 
1318 		err = add_to_pagemap(addr, &pme, pm);
1319 		if (err)
1320 			return err;
1321 		if (pm->show_pfn && (flags & PM_PRESENT))
1322 			frame++;
1323 	}
1324 
1325 	cond_resched();
1326 
1327 	return err;
1328 }
1329 #endif /* HUGETLB_PAGE */
1330 
1331 /*
1332  * /proc/pid/pagemap - an array mapping virtual pages to pfns
1333  *
1334  * For each page in the address space, this file contains one 64-bit entry
1335  * consisting of the following:
1336  *
1337  * Bits 0-54  page frame number (PFN) if present
1338  * Bits 0-4   swap type if swapped
1339  * Bits 5-54  swap offset if swapped
1340  * Bit  55    pte is soft-dirty (see Documentation/vm/soft-dirty.txt)
1341  * Bit  56    page exclusively mapped
1342  * Bits 57-60 zero
1343  * Bit  61    page is file-page or shared-anon
1344  * Bit  62    page swapped
1345  * Bit  63    page present
1346  *
1347  * If the page is not present but in swap, then the PFN contains an
1348  * encoding of the swap file number and the page's offset into the
1349  * swap. Unmapped pages return a null PFN. This allows determining
1350  * precisely which pages are mapped (or in swap) and comparing mapped
1351  * pages between processes.
1352  *
1353  * Efficient users of this interface will use /proc/pid/maps to
1354  * determine which areas of memory are actually mapped and llseek to
1355  * skip over unmapped regions.
1356  */
1357 static ssize_t pagemap_read(struct file *file, char __user *buf,
1358 			    size_t count, loff_t *ppos)
1359 {
1360 	struct mm_struct *mm = file->private_data;
1361 	struct pagemapread pm;
1362 	struct mm_walk pagemap_walk = {};
1363 	unsigned long src;
1364 	unsigned long svpfn;
1365 	unsigned long start_vaddr;
1366 	unsigned long end_vaddr;
1367 	int ret = 0, copied = 0;
1368 
1369 	if (!mm || !mmget_not_zero(mm))
1370 		goto out;
1371 
1372 	ret = -EINVAL;
1373 	/* file position must be aligned */
1374 	if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1375 		goto out_mm;
1376 
1377 	ret = 0;
1378 	if (!count)
1379 		goto out_mm;
1380 
1381 	/* do not disclose physical addresses: attack vector */
1382 	pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1383 
1384 	pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1385 	pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
1386 	ret = -ENOMEM;
1387 	if (!pm.buffer)
1388 		goto out_mm;
1389 
1390 	pagemap_walk.pmd_entry = pagemap_pmd_range;
1391 	pagemap_walk.pte_hole = pagemap_pte_hole;
1392 #ifdef CONFIG_HUGETLB_PAGE
1393 	pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1394 #endif
1395 	pagemap_walk.mm = mm;
1396 	pagemap_walk.private = &pm;
1397 
1398 	src = *ppos;
1399 	svpfn = src / PM_ENTRY_BYTES;
1400 	start_vaddr = svpfn << PAGE_SHIFT;
1401 	end_vaddr = mm->task_size;
1402 
1403 	/* watch out for wraparound */
1404 	if (svpfn > mm->task_size >> PAGE_SHIFT)
1405 		start_vaddr = end_vaddr;
1406 
1407 	/*
1408 	 * The odds are that this will stop walking way
1409 	 * before end_vaddr, because the length of the
1410 	 * user buffer is tracked in "pm", and the walk
1411 	 * will stop when we hit the end of the buffer.
1412 	 */
1413 	ret = 0;
1414 	while (count && (start_vaddr < end_vaddr)) {
1415 		int len;
1416 		unsigned long end;
1417 
1418 		pm.pos = 0;
1419 		end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1420 		/* overflow ? */
1421 		if (end < start_vaddr || end > end_vaddr)
1422 			end = end_vaddr;
1423 		down_read(&mm->mmap_sem);
1424 		ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1425 		up_read(&mm->mmap_sem);
1426 		start_vaddr = end;
1427 
1428 		len = min(count, PM_ENTRY_BYTES * pm.pos);
1429 		if (copy_to_user(buf, pm.buffer, len)) {
1430 			ret = -EFAULT;
1431 			goto out_free;
1432 		}
1433 		copied += len;
1434 		buf += len;
1435 		count -= len;
1436 	}
1437 	*ppos += copied;
1438 	if (!ret || ret == PM_END_OF_BUFFER)
1439 		ret = copied;
1440 
1441 out_free:
1442 	kfree(pm.buffer);
1443 out_mm:
1444 	mmput(mm);
1445 out:
1446 	return ret;
1447 }
1448 
1449 static int pagemap_open(struct inode *inode, struct file *file)
1450 {
1451 	struct mm_struct *mm;
1452 
1453 	mm = proc_mem_open(inode, PTRACE_MODE_READ);
1454 	if (IS_ERR(mm))
1455 		return PTR_ERR(mm);
1456 	file->private_data = mm;
1457 	return 0;
1458 }
1459 
1460 static int pagemap_release(struct inode *inode, struct file *file)
1461 {
1462 	struct mm_struct *mm = file->private_data;
1463 
1464 	if (mm)
1465 		mmdrop(mm);
1466 	return 0;
1467 }
1468 
1469 const struct file_operations proc_pagemap_operations = {
1470 	.llseek		= mem_lseek, /* borrow this */
1471 	.read		= pagemap_read,
1472 	.open		= pagemap_open,
1473 	.release	= pagemap_release,
1474 };
1475 #endif /* CONFIG_PROC_PAGE_MONITOR */
1476 
1477 #ifdef CONFIG_NUMA
1478 
1479 struct numa_maps {
1480 	unsigned long pages;
1481 	unsigned long anon;
1482 	unsigned long active;
1483 	unsigned long writeback;
1484 	unsigned long mapcount_max;
1485 	unsigned long dirty;
1486 	unsigned long swapcache;
1487 	unsigned long node[MAX_NUMNODES];
1488 };
1489 
1490 struct numa_maps_private {
1491 	struct proc_maps_private proc_maps;
1492 	struct numa_maps md;
1493 };
1494 
1495 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1496 			unsigned long nr_pages)
1497 {
1498 	int count = page_mapcount(page);
1499 
1500 	md->pages += nr_pages;
1501 	if (pte_dirty || PageDirty(page))
1502 		md->dirty += nr_pages;
1503 
1504 	if (PageSwapCache(page))
1505 		md->swapcache += nr_pages;
1506 
1507 	if (PageActive(page) || PageUnevictable(page))
1508 		md->active += nr_pages;
1509 
1510 	if (PageWriteback(page))
1511 		md->writeback += nr_pages;
1512 
1513 	if (PageAnon(page))
1514 		md->anon += nr_pages;
1515 
1516 	if (count > md->mapcount_max)
1517 		md->mapcount_max = count;
1518 
1519 	md->node[page_to_nid(page)] += nr_pages;
1520 }
1521 
1522 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1523 		unsigned long addr)
1524 {
1525 	struct page *page;
1526 	int nid;
1527 
1528 	if (!pte_present(pte))
1529 		return NULL;
1530 
1531 	page = vm_normal_page(vma, addr, pte);
1532 	if (!page)
1533 		return NULL;
1534 
1535 	if (PageReserved(page))
1536 		return NULL;
1537 
1538 	nid = page_to_nid(page);
1539 	if (!node_isset(nid, node_states[N_MEMORY]))
1540 		return NULL;
1541 
1542 	return page;
1543 }
1544 
1545 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1546 static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1547 					      struct vm_area_struct *vma,
1548 					      unsigned long addr)
1549 {
1550 	struct page *page;
1551 	int nid;
1552 
1553 	if (!pmd_present(pmd))
1554 		return NULL;
1555 
1556 	page = vm_normal_page_pmd(vma, addr, pmd);
1557 	if (!page)
1558 		return NULL;
1559 
1560 	if (PageReserved(page))
1561 		return NULL;
1562 
1563 	nid = page_to_nid(page);
1564 	if (!node_isset(nid, node_states[N_MEMORY]))
1565 		return NULL;
1566 
1567 	return page;
1568 }
1569 #endif
1570 
1571 static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1572 		unsigned long end, struct mm_walk *walk)
1573 {
1574 	struct numa_maps *md = walk->private;
1575 	struct vm_area_struct *vma = walk->vma;
1576 	spinlock_t *ptl;
1577 	pte_t *orig_pte;
1578 	pte_t *pte;
1579 
1580 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1581 	ptl = pmd_trans_huge_lock(pmd, vma);
1582 	if (ptl) {
1583 		struct page *page;
1584 
1585 		page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1586 		if (page)
1587 			gather_stats(page, md, pmd_dirty(*pmd),
1588 				     HPAGE_PMD_SIZE/PAGE_SIZE);
1589 		spin_unlock(ptl);
1590 		return 0;
1591 	}
1592 
1593 	if (pmd_trans_unstable(pmd))
1594 		return 0;
1595 #endif
1596 	orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1597 	do {
1598 		struct page *page = can_gather_numa_stats(*pte, vma, addr);
1599 		if (!page)
1600 			continue;
1601 		gather_stats(page, md, pte_dirty(*pte), 1);
1602 
1603 	} while (pte++, addr += PAGE_SIZE, addr != end);
1604 	pte_unmap_unlock(orig_pte, ptl);
1605 	cond_resched();
1606 	return 0;
1607 }
1608 #ifdef CONFIG_HUGETLB_PAGE
1609 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1610 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1611 {
1612 	pte_t huge_pte = huge_ptep_get(pte);
1613 	struct numa_maps *md;
1614 	struct page *page;
1615 
1616 	if (!pte_present(huge_pte))
1617 		return 0;
1618 
1619 	page = pte_page(huge_pte);
1620 	if (!page)
1621 		return 0;
1622 
1623 	md = walk->private;
1624 	gather_stats(page, md, pte_dirty(huge_pte), 1);
1625 	return 0;
1626 }
1627 
1628 #else
1629 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1630 		unsigned long addr, unsigned long end, struct mm_walk *walk)
1631 {
1632 	return 0;
1633 }
1634 #endif
1635 
1636 /*
1637  * Display pages allocated per node and memory policy via /proc.
1638  */
1639 static int show_numa_map(struct seq_file *m, void *v, int is_pid)
1640 {
1641 	struct numa_maps_private *numa_priv = m->private;
1642 	struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1643 	struct vm_area_struct *vma = v;
1644 	struct numa_maps *md = &numa_priv->md;
1645 	struct file *file = vma->vm_file;
1646 	struct mm_struct *mm = vma->vm_mm;
1647 	struct mm_walk walk = {
1648 		.hugetlb_entry = gather_hugetlb_stats,
1649 		.pmd_entry = gather_pte_stats,
1650 		.private = md,
1651 		.mm = mm,
1652 	};
1653 	struct mempolicy *pol;
1654 	char buffer[64];
1655 	int nid;
1656 
1657 	if (!mm)
1658 		return 0;
1659 
1660 	/* Ensure we start with an empty set of numa_maps statistics. */
1661 	memset(md, 0, sizeof(*md));
1662 
1663 	pol = __get_vma_policy(vma, vma->vm_start);
1664 	if (pol) {
1665 		mpol_to_str(buffer, sizeof(buffer), pol);
1666 		mpol_cond_put(pol);
1667 	} else {
1668 		mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1669 	}
1670 
1671 	seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1672 
1673 	if (file) {
1674 		seq_puts(m, " file=");
1675 		seq_file_path(m, file, "\n\t= ");
1676 	} else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1677 		seq_puts(m, " heap");
1678 	} else if (is_stack(proc_priv, vma)) {
1679 		seq_puts(m, " stack");
1680 	}
1681 
1682 	if (is_vm_hugetlb_page(vma))
1683 		seq_puts(m, " huge");
1684 
1685 	/* mmap_sem is held by m_start */
1686 	walk_page_vma(vma, &walk);
1687 
1688 	if (!md->pages)
1689 		goto out;
1690 
1691 	if (md->anon)
1692 		seq_printf(m, " anon=%lu", md->anon);
1693 
1694 	if (md->dirty)
1695 		seq_printf(m, " dirty=%lu", md->dirty);
1696 
1697 	if (md->pages != md->anon && md->pages != md->dirty)
1698 		seq_printf(m, " mapped=%lu", md->pages);
1699 
1700 	if (md->mapcount_max > 1)
1701 		seq_printf(m, " mapmax=%lu", md->mapcount_max);
1702 
1703 	if (md->swapcache)
1704 		seq_printf(m, " swapcache=%lu", md->swapcache);
1705 
1706 	if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1707 		seq_printf(m, " active=%lu", md->active);
1708 
1709 	if (md->writeback)
1710 		seq_printf(m, " writeback=%lu", md->writeback);
1711 
1712 	for_each_node_state(nid, N_MEMORY)
1713 		if (md->node[nid])
1714 			seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1715 
1716 	seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
1717 out:
1718 	seq_putc(m, '\n');
1719 	m_cache_vma(m, vma);
1720 	return 0;
1721 }
1722 
1723 static int show_pid_numa_map(struct seq_file *m, void *v)
1724 {
1725 	return show_numa_map(m, v, 1);
1726 }
1727 
1728 static int show_tid_numa_map(struct seq_file *m, void *v)
1729 {
1730 	return show_numa_map(m, v, 0);
1731 }
1732 
1733 static const struct seq_operations proc_pid_numa_maps_op = {
1734 	.start  = m_start,
1735 	.next   = m_next,
1736 	.stop   = m_stop,
1737 	.show   = show_pid_numa_map,
1738 };
1739 
1740 static const struct seq_operations proc_tid_numa_maps_op = {
1741 	.start  = m_start,
1742 	.next   = m_next,
1743 	.stop   = m_stop,
1744 	.show   = show_tid_numa_map,
1745 };
1746 
1747 static int numa_maps_open(struct inode *inode, struct file *file,
1748 			  const struct seq_operations *ops)
1749 {
1750 	return proc_maps_open(inode, file, ops,
1751 				sizeof(struct numa_maps_private));
1752 }
1753 
1754 static int pid_numa_maps_open(struct inode *inode, struct file *file)
1755 {
1756 	return numa_maps_open(inode, file, &proc_pid_numa_maps_op);
1757 }
1758 
1759 static int tid_numa_maps_open(struct inode *inode, struct file *file)
1760 {
1761 	return numa_maps_open(inode, file, &proc_tid_numa_maps_op);
1762 }
1763 
1764 const struct file_operations proc_pid_numa_maps_operations = {
1765 	.open		= pid_numa_maps_open,
1766 	.read		= seq_read,
1767 	.llseek		= seq_lseek,
1768 	.release	= proc_map_release,
1769 };
1770 
1771 const struct file_operations proc_tid_numa_maps_operations = {
1772 	.open		= tid_numa_maps_open,
1773 	.read		= seq_read,
1774 	.llseek		= seq_lseek,
1775 	.release	= proc_map_release,
1776 };
1777 #endif /* CONFIG_NUMA */
1778