| gup.c (8f66439eec46d652255b9351abebb540ee5b2fd9) | gup.c (e585513b76f7b05d08ca3fb250fed11f6ba46ee5) |
|---|---|
| 1#include <linux/kernel.h> 2#include <linux/errno.h> 3#include <linux/err.h> 4#include <linux/spinlock.h> 5 6#include <linux/mm.h> 7#include <linux/memremap.h> 8#include <linux/pagemap.h> --- 1137 unchanged lines hidden (view full) --- 1146 NULL) < 1) 1147 return NULL; 1148 flush_cache_page(vma, addr, page_to_pfn(page)); 1149 return page; 1150} 1151#endif /* CONFIG_ELF_CORE */ 1152 1153/* | 1#include <linux/kernel.h> 2#include <linux/errno.h> 3#include <linux/err.h> 4#include <linux/spinlock.h> 5 6#include <linux/mm.h> 7#include <linux/memremap.h> 8#include <linux/pagemap.h> --- 1137 unchanged lines hidden (view full) --- 1146 NULL) < 1) 1147 return NULL; 1148 flush_cache_page(vma, addr, page_to_pfn(page)); 1149 return page; 1150} 1151#endif /* CONFIG_ELF_CORE */ 1152 1153/* |
| 1154 * Generic RCU Fast GUP | 1154 * Generic Fast GUP |
| 1155 * 1156 * get_user_pages_fast attempts to pin user pages by walking the page 1157 * tables directly and avoids taking locks. Thus the walker needs to be 1158 * protected from page table pages being freed from under it, and should 1159 * block any THP splits. 1160 * 1161 * One way to achieve this is to have the walker disable interrupts, and 1162 * rely on IPIs from the TLB flushing code blocking before the page table --- 4 unchanged lines hidden (view full) --- 1167 * belonging to more than one mm_user, then rcu_sched a callback to free those 1168 * pages. Disabling interrupts will allow the fast_gup walker to both block 1169 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs 1170 * (which is a relatively rare event). The code below adopts this strategy. 1171 * 1172 * Before activating this code, please be aware that the following assumptions 1173 * are currently made: 1174 * | 1155 * 1156 * get_user_pages_fast attempts to pin user pages by walking the page 1157 * tables directly and avoids taking locks. Thus the walker needs to be 1158 * protected from page table pages being freed from under it, and should 1159 * block any THP splits. 1160 * 1161 * One way to achieve this is to have the walker disable interrupts, and 1162 * rely on IPIs from the TLB flushing code blocking before the page table --- 4 unchanged lines hidden (view full) --- 1167 * belonging to more than one mm_user, then rcu_sched a callback to free those 1168 * pages. Disabling interrupts will allow the fast_gup walker to both block 1169 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs 1170 * (which is a relatively rare event). The code below adopts this strategy. 1171 * 1172 * Before activating this code, please be aware that the following assumptions 1173 * are currently made: 1174 * |
| 1175 * *) HAVE_RCU_TABLE_FREE is enabled, and tlb_remove_table is used to free 1176 * pages containing page tables. | 1175 * *) Either HAVE_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to 1176 * free pages containing page tables or TLB flushing requires IPI broadcast. |
| 1177 * 1178 * *) ptes can be read atomically by the architecture. 1179 * 1180 * *) access_ok is sufficient to validate userspace address ranges. 1181 * 1182 * The last two assumptions can be relaxed by the addition of helper functions. 1183 * 1184 * This code is based heavily on the PowerPC implementation by Nick Piggin. 1185 */ | 1177 * 1178 * *) ptes can be read atomically by the architecture. 1179 * 1180 * *) access_ok is sufficient to validate userspace address ranges. 1181 * 1182 * The last two assumptions can be relaxed by the addition of helper functions. 1183 * 1184 * This code is based heavily on the PowerPC implementation by Nick Piggin. 1185 */ |
| 1186#ifdef CONFIG_HAVE_GENERIC_RCU_GUP | 1186#ifdef CONFIG_HAVE_GENERIC_GUP |
| 1187 1188#ifndef gup_get_pte 1189/* 1190 * We assume that the PTE can be read atomically. If this is not the case for 1191 * your architecture, please provide the helper. 1192 */ 1193static inline pte_t gup_get_pte(pte_t *ptep) 1194{ --- 473 unchanged lines hidden (view full) --- 1668 else 1669 ret += nr; 1670 } 1671 } 1672 1673 return ret; 1674} 1675 | 1187 1188#ifndef gup_get_pte 1189/* 1190 * We assume that the PTE can be read atomically. If this is not the case for 1191 * your architecture, please provide the helper. 1192 */ 1193static inline pte_t gup_get_pte(pte_t *ptep) 1194{ --- 473 unchanged lines hidden (view full) --- 1668 else 1669 ret += nr; 1670 } 1671 } 1672 1673 return ret; 1674} 1675 |
| 1676#endif /* CONFIG_HAVE_GENERIC_RCU_GUP */ | 1676#endif /* CONFIG_HAVE_GENERIC_GUP */ |