vm_map.c (d46e7d6bee6773e6e3a5a68b8d2164808e10d6a8) vm_map.c (1d7cf06c8c5e1c0232edf95a43a7c1dca598a812)
1/*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 399 unchanged lines hidden (view full) ---

408void
409_vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
410{
411
412 KASSERT(lockstatus(&map->lock, curthread) == LK_EXCLUSIVE,
413 ("%s: lock not held", __func__));
414}
415
1/*
2 * Copyright (c) 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 399 unchanged lines hidden (view full) ---

408void
409_vm_map_lock_downgrade(vm_map_t map, const char *file, int line)
410{
411
412 KASSERT(lockstatus(&map->lock, curthread) == LK_EXCLUSIVE,
413 ("%s: lock not held", __func__));
414}
415
416void
417_vm_map_set_recursive(vm_map_t map, const char *file, int line)
418{
419}
420
421void
422_vm_map_clear_recursive(vm_map_t map, const char *file, int line)
423{
424}
425
426/*
427 * vm_map_unlock_and_wait:
428 */
429static __inline int
430vm_map_unlock_and_wait(vm_map_t map, boolean_t user_wait)
431{
432 int retval;
433

--- 1364 unchanged lines hidden (view full) ---

1798 }
1799 vm_map_unlock(map);
1800 if (need_wakeup)
1801 vm_map_wakeup(map);
1802 return (rv);
1803}
1804
1805/*
416/*
417 * vm_map_unlock_and_wait:
418 */
419static __inline int
420vm_map_unlock_and_wait(vm_map_t map, boolean_t user_wait)
421{
422 int retval;
423

--- 1364 unchanged lines hidden (view full) ---

1788 }
1789 vm_map_unlock(map);
1790 if (need_wakeup)
1791 vm_map_wakeup(map);
1792 return (rv);
1793}
1794
1795/*
1806 * Implement the semantics of mlock
1807 */
1808int
1809vm_map_user_pageable(
1810 vm_map_t map,
1811 vm_offset_t start,
1812 vm_offset_t end,
1813 boolean_t new_pageable)
1814{
1815 vm_map_entry_t entry;
1816 vm_map_entry_t start_entry;
1817 vm_offset_t estart;
1818 vm_offset_t eend;
1819 int rv;
1820
1821 vm_map_lock(map);
1822 VM_MAP_RANGE_CHECK(map, start, end);
1823
1824 if (vm_map_lookup_entry(map, start, &start_entry) == FALSE) {
1825 vm_map_unlock(map);
1826 return (KERN_INVALID_ADDRESS);
1827 }
1828
1829 if (new_pageable) {
1830
1831 entry = start_entry;
1832 vm_map_clip_start(map, entry, start);
1833
1834 /*
1835 * Now decrement the wiring count for each region. If a region
1836 * becomes completely unwired, unwire its physical pages and
1837 * mappings.
1838 */
1839 while ((entry != &map->header) && (entry->start < end)) {
1840 if (entry->eflags & MAP_ENTRY_USER_WIRED) {
1841 vm_map_clip_end(map, entry, end);
1842 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
1843 entry->wired_count--;
1844 if (entry->wired_count == 0)
1845 vm_fault_unwire(map, entry->start, entry->end);
1846 }
1847 vm_map_simplify_entry(map,entry);
1848 entry = entry->next;
1849 }
1850 } else {
1851
1852 entry = start_entry;
1853
1854 while ((entry != &map->header) && (entry->start < end)) {
1855
1856 if (entry->eflags & MAP_ENTRY_USER_WIRED) {
1857 entry = entry->next;
1858 continue;
1859 }
1860
1861 if (entry->wired_count != 0) {
1862 entry->wired_count++;
1863 entry->eflags |= MAP_ENTRY_USER_WIRED;
1864 entry = entry->next;
1865 continue;
1866 }
1867
1868 /* Here on entry being newly wired */
1869
1870 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
1871 int copyflag = entry->eflags & MAP_ENTRY_NEEDS_COPY;
1872 if (copyflag && ((entry->protection & VM_PROT_WRITE) != 0)) {
1873
1874 vm_object_shadow(&entry->object.vm_object,
1875 &entry->offset,
1876 atop(entry->end - entry->start));
1877 entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
1878
1879 } else if (entry->object.vm_object == NULL &&
1880 !map->system_map) {
1881
1882 entry->object.vm_object =
1883 vm_object_allocate(OBJT_DEFAULT,
1884 atop(entry->end - entry->start));
1885 entry->offset = (vm_offset_t) 0;
1886
1887 }
1888 }
1889
1890 vm_map_clip_start(map, entry, start);
1891 vm_map_clip_end(map, entry, end);
1892
1893 entry->wired_count++;
1894 entry->eflags |= MAP_ENTRY_USER_WIRED;
1895 estart = entry->start;
1896 eend = entry->end;
1897
1898 /* First we need to allow map modifications */
1899 vm_map_set_recursive(map);
1900 vm_map_lock_downgrade(map);
1901 map->timestamp++;
1902
1903 rv = vm_fault_user_wire(map, entry->start, entry->end);
1904 if (rv) {
1905
1906 entry->wired_count--;
1907 entry->eflags &= ~MAP_ENTRY_USER_WIRED;
1908
1909 vm_map_clear_recursive(map);
1910 vm_map_unlock(map);
1911
1912 /*
1913 * At this point, the map is unlocked, and
1914 * entry might no longer be valid. Use copy
1915 * of entry start value obtained while entry
1916 * was valid.
1917 */
1918 (void) vm_map_user_pageable(map, start, estart,
1919 TRUE);
1920 return rv;
1921 }
1922
1923 vm_map_clear_recursive(map);
1924 if (vm_map_lock_upgrade(map)) {
1925 vm_map_lock(map);
1926 if (vm_map_lookup_entry(map, estart, &entry)
1927 == FALSE) {
1928 vm_map_unlock(map);
1929 /*
1930 * vm_fault_user_wire succeded, thus
1931 * the area between start and eend
1932 * is wired and has to be unwired
1933 * here as part of the cleanup.
1934 */
1935 (void) vm_map_user_pageable(map,
1936 start,
1937 eend,
1938 TRUE);
1939 return (KERN_INVALID_ADDRESS);
1940 }
1941 }
1942 vm_map_simplify_entry(map,entry);
1943 }
1944 }
1945 map->timestamp++;
1946 vm_map_unlock(map);
1947 return KERN_SUCCESS;
1948}
1949
1950/*
1951 * vm_map_pageable:
1952 *
1953 * Sets the pageability of the specified address
1954 * range in the target map. Regions specified
1955 * as not pageable require locked-down physical
1956 * memory and physical page maps.
1957 *
1958 * The map must not be locked, but a reference
1959 * must remain to the map throughout the call.
1960 */
1961int
1962vm_map_pageable(
1963 vm_map_t map,
1964 vm_offset_t start,
1965 vm_offset_t end,
1966 boolean_t new_pageable)
1967{
1968 vm_map_entry_t entry;
1969 vm_map_entry_t start_entry;
1970 vm_offset_t failed = 0;
1971 int rv;
1972
1973 GIANT_REQUIRED;
1974
1975 vm_map_lock(map);
1976
1977 VM_MAP_RANGE_CHECK(map, start, end);
1978
1979 /*
1980 * Only one pageability change may take place at one time, since
1981 * vm_fault assumes it will be called only once for each
1982 * wiring/unwiring. Therefore, we have to make sure we're actually
1983 * changing the pageability for the entire region. We do so before
1984 * making any changes.
1985 */
1986 if (vm_map_lookup_entry(map, start, &start_entry) == FALSE) {
1987 vm_map_unlock(map);
1988 return (KERN_INVALID_ADDRESS);
1989 }
1990 entry = start_entry;
1991
1992 /*
1993 * Actions are rather different for wiring and unwiring, so we have
1994 * two separate cases.
1995 */
1996 if (new_pageable) {
1997 vm_map_clip_start(map, entry, start);
1998
1999 /*
2000 * Unwiring. First ensure that the range to be unwired is
2001 * really wired down and that there are no holes.
2002 */
2003 while ((entry != &map->header) && (entry->start < end)) {
2004 if (entry->wired_count == 0 ||
2005 (entry->end < end &&
2006 (entry->next == &map->header ||
2007 entry->next->start > entry->end))) {
2008 vm_map_unlock(map);
2009 return (KERN_INVALID_ARGUMENT);
2010 }
2011 entry = entry->next;
2012 }
2013
2014 /*
2015 * Now decrement the wiring count for each region. If a region
2016 * becomes completely unwired, unwire its physical pages and
2017 * mappings.
2018 */
2019 entry = start_entry;
2020 while ((entry != &map->header) && (entry->start < end)) {
2021 vm_map_clip_end(map, entry, end);
2022
2023 entry->wired_count--;
2024 if (entry->wired_count == 0)
2025 vm_fault_unwire(map, entry->start, entry->end);
2026
2027 vm_map_simplify_entry(map, entry);
2028
2029 entry = entry->next;
2030 }
2031 } else {
2032 /*
2033 * Wiring. We must do this in two passes:
2034 *
2035 * 1. Holding the write lock, we create any shadow or zero-fill
2036 * objects that need to be created. Then we clip each map
2037 * entry to the region to be wired and increment its wiring
2038 * count. We create objects before clipping the map entries
2039 * to avoid object proliferation.
2040 *
2041 * 2. We downgrade to a read lock, and call vm_fault_wire to
2042 * fault in the pages for any newly wired area (wired_count is
2043 * 1).
2044 *
2045 * Downgrading to a read lock for vm_fault_wire avoids a possible
2046 * deadlock with another process that may have faulted on one
2047 * of the pages to be wired (it would mark the page busy,
2048 * blocking us, then in turn block on the map lock that we
2049 * hold). Because of problems in the recursive lock package,
2050 * we cannot upgrade to a write lock in vm_map_lookup. Thus,
2051 * any actions that require the write lock must be done
2052 * beforehand. Because we keep the read lock on the map, the
2053 * copy-on-write status of the entries we modify here cannot
2054 * change.
2055 */
2056
2057 /*
2058 * Pass 1.
2059 */
2060 while ((entry != &map->header) && (entry->start < end)) {
2061 if (entry->wired_count == 0) {
2062
2063 /*
2064 * Perform actions of vm_map_lookup that need
2065 * the write lock on the map: create a shadow
2066 * object for a copy-on-write region, or an
2067 * object for a zero-fill region.
2068 *
2069 * We don't have to do this for entries that
2070 * point to sub maps, because we won't
2071 * hold the lock on the sub map.
2072 */
2073 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
2074 int copyflag = entry->eflags & MAP_ENTRY_NEEDS_COPY;
2075 if (copyflag &&
2076 ((entry->protection & VM_PROT_WRITE) != 0)) {
2077
2078 vm_object_shadow(&entry->object.vm_object,
2079 &entry->offset,
2080 atop(entry->end - entry->start));
2081 entry->eflags &= ~MAP_ENTRY_NEEDS_COPY;
2082 } else if (entry->object.vm_object == NULL &&
2083 !map->system_map) {
2084 entry->object.vm_object =
2085 vm_object_allocate(OBJT_DEFAULT,
2086 atop(entry->end - entry->start));
2087 entry->offset = (vm_offset_t) 0;
2088 }
2089 }
2090 }
2091 vm_map_clip_start(map, entry, start);
2092 vm_map_clip_end(map, entry, end);
2093 entry->wired_count++;
2094
2095 /*
2096 * Check for holes
2097 */
2098 if (entry->end < end &&
2099 (entry->next == &map->header ||
2100 entry->next->start > entry->end)) {
2101 /*
2102 * Found one. Object creation actions do not
2103 * need to be undone, but the wired counts
2104 * need to be restored.
2105 */
2106 while (entry != &map->header && entry->end > start) {
2107 entry->wired_count--;
2108 entry = entry->prev;
2109 }
2110 vm_map_unlock(map);
2111 return (KERN_INVALID_ARGUMENT);
2112 }
2113 entry = entry->next;
2114 }
2115
2116 /*
2117 * Pass 2.
2118 */
2119
2120 /*
2121 * HACK HACK HACK HACK
2122 *
2123 * If we are wiring in the kernel map or a submap of it,
2124 * unlock the map to avoid deadlocks. We trust that the
2125 * kernel is well-behaved, and therefore will not do
2126 * anything destructive to this region of the map while
2127 * we have it unlocked. We cannot trust user processes
2128 * to do the same.
2129 *
2130 * HACK HACK HACK HACK
2131 */
2132 if (vm_map_pmap(map) == kernel_pmap) {
2133 vm_map_unlock(map); /* trust me ... */
2134 } else {
2135 vm_map_lock_downgrade(map);
2136 }
2137
2138 rv = 0;
2139 entry = start_entry;
2140 while (entry != &map->header && entry->start < end) {
2141 /*
2142 * If vm_fault_wire fails for any page we need to undo
2143 * what has been done. We decrement the wiring count
2144 * for those pages which have not yet been wired (now)
2145 * and unwire those that have (later).
2146 *
2147 * XXX this violates the locking protocol on the map,
2148 * needs to be fixed.
2149 */
2150 if (rv)
2151 entry->wired_count--;
2152 else if (entry->wired_count == 1) {
2153 rv = vm_fault_wire(map, entry->start, entry->end);
2154 if (rv) {
2155 failed = entry->start;
2156 entry->wired_count--;
2157 }
2158 }
2159 entry = entry->next;
2160 }
2161
2162 if (vm_map_pmap(map) == kernel_pmap) {
2163 vm_map_lock(map);
2164 }
2165 if (rv) {
2166 vm_map_unlock(map);
2167 (void) vm_map_pageable(map, start, failed, TRUE);
2168 return (rv);
2169 }
2170 /*
2171 * An exclusive lock on the map is needed in order to call
2172 * vm_map_simplify_entry(). If the current lock on the map
2173 * is only a shared lock, an upgrade is needed.
2174 */
2175 if (vm_map_pmap(map) != kernel_pmap &&
2176 vm_map_lock_upgrade(map)) {
2177 vm_map_lock(map);
2178 if (vm_map_lookup_entry(map, start, &start_entry) ==
2179 FALSE) {
2180 vm_map_unlock(map);
2181 return KERN_SUCCESS;
2182 }
2183 }
2184 vm_map_simplify_entry(map, start_entry);
2185 }
2186
2187 vm_map_unlock(map);
2188
2189 return (KERN_SUCCESS);
2190}
2191
2192/*
2193 * vm_map_clean
2194 *
2195 * Push any dirty cached pages in the address range to their pager.
2196 * If syncio is TRUE, dirty pages are written synchronously.
2197 * If invalidate is TRUE, any cached pages are freed as well.
2198 *
2199 * Returns an error if any part of the specified range is not mapped.
2200 */

--- 1330 unchanged lines hidden --- 		1796 * vm_map_clean
1797 *
1798 * Push any dirty cached pages in the address range to their pager.
1799 * If syncio is TRUE, dirty pages are written synchronously.
1800 * If invalidate is TRUE, any cached pages are freed as well.
1801 *
1802 * Returns an error if any part of the specified range is not mapped.
1803 */

--- 1330 unchanged lines hidden ---