hugetlbpage.c (ee01c4d72adffb7d424535adf630f2955748fa8b) hugetlbpage.c (2fb4706057bcf8261b3b0521ec7a62b54b82ce48)
1/*
2 * PPC Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>

--- 105 unchanged lines hidden (view full) ---

114
115/*
116 * At this point we do the placement change only for BOOK3S 64. This would
117 * possibly work on other subarchs.
118 */
119pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
120{
121 pgd_t *pg;
1/*
2 * PPC Huge TLB Page Support for Kernel.
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>

--- 105 unchanged lines hidden (view full) ---

114
115/*
116 * At this point we do the placement change only for BOOK3S 64. This would
117 * possibly work on other subarchs.
118 */
119pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
120{
121 pgd_t *pg;
122 p4d_t *p4;
122 pud_t *pu;
123 pmd_t *pm;
124 hugepd_t *hpdp = NULL;
125 unsigned pshift = __ffs(sz);
126 unsigned pdshift = PGDIR_SHIFT;
127 spinlock_t *ptl;
128
129 addr &= ~(sz-1);
130 pg = pgd_offset(mm, addr);
123 pud_t *pu;
124 pmd_t *pm;
125 hugepd_t *hpdp = NULL;
126 unsigned pshift = __ffs(sz);
127 unsigned pdshift = PGDIR_SHIFT;
128 spinlock_t *ptl;
129
130 addr &= ~(sz-1);
131 pg = pgd_offset(mm, addr);
132 p4 = p4d_offset(pg, addr);
131
132#ifdef CONFIG_PPC_BOOK3S_64
133 if (pshift == PGDIR_SHIFT)
134 /* 16GB huge page */
133
134#ifdef CONFIG_PPC_BOOK3S_64
135 if (pshift == PGDIR_SHIFT)
136 /* 16GB huge page */
135 return (pte_t *) pg;
137 return (pte_t *) p4;
136 else if (pshift > PUD_SHIFT) {
137 /*
138 * We need to use hugepd table
139 */
140 ptl = &mm->page_table_lock;
138 else if (pshift > PUD_SHIFT) {
139 /*
140 * We need to use hugepd table
141 */
142 ptl = &mm->page_table_lock;
141 hpdp = (hugepd_t *)pg;
143 hpdp = (hugepd_t *)p4;
142 } else {
143 pdshift = PUD_SHIFT;
144 } else {
145 pdshift = PUD_SHIFT;
144 pu = pud_alloc(mm, pg, addr);
146 pu = pud_alloc(mm, p4, addr);
145 if (!pu)
146 return NULL;
147 if (pshift == PUD_SHIFT)
148 return (pte_t *)pu;
149 else if (pshift > PMD_SHIFT) {
150 ptl = pud_lockptr(mm, pu);
151 hpdp = (hugepd_t *)pu;
152 } else {

--- 8 unchanged lines hidden (view full) ---

161 ptl = pmd_lockptr(mm, pm);
162 hpdp = (hugepd_t *)pm;
163 }
164 }
165 }
166#else
167 if (pshift >= PGDIR_SHIFT) {
168 ptl = &mm->page_table_lock;
147 if (!pu)
148 return NULL;
149 if (pshift == PUD_SHIFT)
150 return (pte_t *)pu;
151 else if (pshift > PMD_SHIFT) {
152 ptl = pud_lockptr(mm, pu);
153 hpdp = (hugepd_t *)pu;
154 } else {

--- 8 unchanged lines hidden (view full) ---

163 ptl = pmd_lockptr(mm, pm);
164 hpdp = (hugepd_t *)pm;
165 }
166 }
167 }
168#else
169 if (pshift >= PGDIR_SHIFT) {
170 ptl = &mm->page_table_lock;
169 hpdp = (hugepd_t *)pg;
171 hpdp = (hugepd_t *)p4;
170 } else {
171 pdshift = PUD_SHIFT;
172 } else {
173 pdshift = PUD_SHIFT;
172 pu = pud_alloc(mm, pg, addr);
174 pu = pud_alloc(mm, p4, addr);
173 if (!pu)
174 return NULL;
175 if (pshift >= PUD_SHIFT) {
176 ptl = pud_lockptr(mm, pu);
177 hpdp = (hugepd_t *)pu;
178 } else {
179 pdshift = PMD_SHIFT;
180 pm = pmd_alloc(mm, pu, addr);

--- 204 unchanged lines hidden (view full) ---

385 return;
386
387 pmd = pmd_offset(pud, start);
388 pud_clear(pud);
389 pmd_free_tlb(tlb, pmd, start);
390 mm_dec_nr_pmds(tlb->mm);
391}
392
175 if (!pu)
176 return NULL;
177 if (pshift >= PUD_SHIFT) {
178 ptl = pud_lockptr(mm, pu);
179 hpdp = (hugepd_t *)pu;
180 } else {
181 pdshift = PMD_SHIFT;
182 pm = pmd_alloc(mm, pu, addr);

--- 204 unchanged lines hidden (view full) ---

387 return;
388
389 pmd = pmd_offset(pud, start);
390 pud_clear(pud);
391 pmd_free_tlb(tlb, pmd, start);
392 mm_dec_nr_pmds(tlb->mm);
393}
394
393static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
395static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
394 unsigned long addr, unsigned long end,
395 unsigned long floor, unsigned long ceiling)
396{
397 pud_t *pud;
398 unsigned long next;
399 unsigned long start;
400
401 start = addr;
402 do {
396 unsigned long addr, unsigned long end,
397 unsigned long floor, unsigned long ceiling)
398{
399 pud_t *pud;
400 unsigned long next;
401 unsigned long start;
402
403 start = addr;
404 do {
403 pud = pud_offset(pgd, addr);
405 pud = pud_offset(p4d, addr);
404 next = pud_addr_end(addr, end);
405 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
406 if (pud_none_or_clear_bad(pud))
407 continue;
408 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
409 ceiling);
410 } else {
411 unsigned long more;

--- 18 unchanged lines hidden (view full) ---

430 if (ceiling) {
431 ceiling &= PGDIR_MASK;
432 if (!ceiling)
433 return;
434 }
435 if (end - 1 > ceiling - 1)
436 return;
437
406 next = pud_addr_end(addr, end);
407 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
408 if (pud_none_or_clear_bad(pud))
409 continue;
410 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
411 ceiling);
412 } else {
413 unsigned long more;

--- 18 unchanged lines hidden (view full) ---

432 if (ceiling) {
433 ceiling &= PGDIR_MASK;
434 if (!ceiling)
435 return;
436 }
437 if (end - 1 > ceiling - 1)
438 return;
439
438 pud = pud_offset(pgd, start);
439 pgd_clear(pgd);
440 pud = pud_offset(p4d, start);
441 p4d_clear(p4d);
440 pud_free_tlb(tlb, pud, start);
441 mm_dec_nr_puds(tlb->mm);
442}
443
444/*
445 * This function frees user-level page tables of a process.
446 */
447void hugetlb_free_pgd_range(struct mmu_gather *tlb,
448 unsigned long addr, unsigned long end,
449 unsigned long floor, unsigned long ceiling)
450{
451 pgd_t *pgd;
442 pud_free_tlb(tlb, pud, start);
443 mm_dec_nr_puds(tlb->mm);
444}
445
446/*
447 * This function frees user-level page tables of a process.
448 */
449void hugetlb_free_pgd_range(struct mmu_gather *tlb,
450 unsigned long addr, unsigned long end,
451 unsigned long floor, unsigned long ceiling)
452{
453 pgd_t *pgd;
454 p4d_t *p4d;
452 unsigned long next;
453
454 /*
455 * Because there are a number of different possible pagetable
456 * layouts for hugepage ranges, we limit knowledge of how
457 * things should be laid out to the allocation path
458 * (huge_pte_alloc(), above). Everything else works out the
459 * structure as it goes from information in the hugepd

--- 6 unchanged lines hidden (view full) ---

466 * To make sense of this, you should probably go read the big
467 * block comment at the top of the normal free_pgd_range(),
468 * too.
469 */
470
471 do {
472 next = pgd_addr_end(addr, end);
473 pgd = pgd_offset(tlb->mm, addr);
455 unsigned long next;
456
457 /*
458 * Because there are a number of different possible pagetable
459 * layouts for hugepage ranges, we limit knowledge of how
460 * things should be laid out to the allocation path
461 * (huge_pte_alloc(), above). Everything else works out the
462 * structure as it goes from information in the hugepd

--- 6 unchanged lines hidden (view full) ---

469 * To make sense of this, you should probably go read the big
470 * block comment at the top of the normal free_pgd_range(),
471 * too.
472 */
473
474 do {
475 next = pgd_addr_end(addr, end);
476 pgd = pgd_offset(tlb->mm, addr);
477 p4d = p4d_offset(pgd, addr);
474 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
478 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
475 if (pgd_none_or_clear_bad(pgd))
479 if (p4d_none_or_clear_bad(p4d))
476 continue;
480 continue;
477 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
481 hugetlb_free_pud_range(tlb, p4d, addr, next, floor, ceiling);
478 } else {
479 unsigned long more;
480 /*
481 * Increment next by the size of the huge mapping since
482 * there may be more than one entry at the pgd level
483 * for a single hugepage, but all of them point to the
484 * same kmem cache that holds the hugepte.
485 */
486 more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
487 if (more > next)
488 next = more;
489
482 } else {
483 unsigned long more;
484 /*
485 * Increment next by the size of the huge mapping since
486 * there may be more than one entry at the pgd level
487 * for a single hugepage, but all of them point to the
488 * same kmem cache that holds the hugepte.
489 */
490 more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
491 if (more > next)
492 next = more;
493
490 free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
494 free_hugepd_range(tlb, (hugepd_t *)p4d, PGDIR_SHIFT,
491 addr, next, floor, ceiling);
492 }
493 } while (addr = next, addr != end);
494}
495
496struct page *follow_huge_pd(struct vm_area_struct *vma,
497 unsigned long address, hugepd_t hpd,
498 int flags, int pdshift)

--- 178 unchanged lines hidden --- 		495 addr, next, floor, ceiling);
496 }
497 } while (addr = next, addr != end);
498}
499
500struct page *follow_huge_pd(struct vm_area_struct *vma,
501 unsigned long address, hugepd_t hpd,
502 int flags, int pdshift)

--- 178 unchanged lines hidden ---