pmap.c (033e1e35d38d38a582c4acc72e3745dbaf27ab53) pmap.c (250a44f6a22f43ad95ca119e876370e82f8cefc1)
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>

--- 918 unchanged lines hidden (view full) ---

927 *
928 * The kernel page table is exempt because its pm_active field is
929 * immutable. The kernel page table is always active on every
930 * processor.
931 */
932void
933pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
934{
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005-2010 Alan L. Cox <alc@cs.rice.edu>

--- 918 unchanged lines hidden (view full) ---

927 *
928 * The kernel page table is exempt because its pm_active field is
929 * immutable. The kernel page table is always active on every
930 * processor.
931 */
932void
933pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
934{
935 cpuset_t cpumask, other_cpus;
935 cpuset_t other_cpus;
936 u_int cpuid;
936
937 sched_pin();
938 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
939 invlpg(va);
940 smp_invlpg(va);
941 } else {
937
938 sched_pin();
939 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
940 invlpg(va);
941 smp_invlpg(va);
942 } else {
942 cpumask = PCPU_GET(cpumask);
943 other_cpus = PCPU_GET(other_cpus);
944 if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
943 cpuid = PCPU_GET(cpuid);
944 other_cpus = all_cpus;
945 CPU_CLR(cpuid, &other_cpus);
946 if (CPU_ISSET(cpuid, &pmap->pm_active))
945 invlpg(va);
946 CPU_AND(&other_cpus, &pmap->pm_active);
947 if (!CPU_EMPTY(&other_cpus))
948 smp_masked_invlpg(other_cpus, va);
949 }
950 sched_unpin();
951}
952
953void
954pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
955{
947 invlpg(va);
948 CPU_AND(&other_cpus, &pmap->pm_active);
949 if (!CPU_EMPTY(&other_cpus))
950 smp_masked_invlpg(other_cpus, va);
951 }
952 sched_unpin();
953}
954
955void
956pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
957{
956 cpuset_t cpumask, other_cpus;
958 cpuset_t other_cpus;
957 vm_offset_t addr;
959 vm_offset_t addr;
960 u_int cpuid;
958
959 sched_pin();
960 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
961 for (addr = sva; addr < eva; addr += PAGE_SIZE)
962 invlpg(addr);
963 smp_invlpg_range(sva, eva);
964 } else {
961
962 sched_pin();
963 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
964 for (addr = sva; addr < eva; addr += PAGE_SIZE)
965 invlpg(addr);
966 smp_invlpg_range(sva, eva);
967 } else {
965 cpumask = PCPU_GET(cpumask);
966 other_cpus = PCPU_GET(other_cpus);
967 if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
968 cpuid = PCPU_GET(cpuid);
969 other_cpus = all_cpus;
970 CPU_CLR(cpuid, &other_cpus);
971 if (CPU_ISSET(cpuid, &pmap->pm_active))
968 for (addr = sva; addr < eva; addr += PAGE_SIZE)
969 invlpg(addr);
970 CPU_AND(&other_cpus, &pmap->pm_active);
971 if (!CPU_EMPTY(&other_cpus))
972 smp_masked_invlpg_range(other_cpus, sva, eva);
973 }
974 sched_unpin();
975}
976
977void
978pmap_invalidate_all(pmap_t pmap)
979{
972 for (addr = sva; addr < eva; addr += PAGE_SIZE)
973 invlpg(addr);
974 CPU_AND(&other_cpus, &pmap->pm_active);
975 if (!CPU_EMPTY(&other_cpus))
976 smp_masked_invlpg_range(other_cpus, sva, eva);
977 }
978 sched_unpin();
979}
980
981void
982pmap_invalidate_all(pmap_t pmap)
983{
980 cpuset_t cpumask, other_cpus;
984 cpuset_t other_cpus;
985 u_int cpuid;
981
982 sched_pin();
983 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
984 invltlb();
985 smp_invltlb();
986 } else {
986
987 sched_pin();
988 if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) {
989 invltlb();
990 smp_invltlb();
991 } else {
987 cpumask = PCPU_GET(cpumask);
988 other_cpus = PCPU_GET(other_cpus);
989 if (CPU_OVERLAP(&pmap->pm_active, &cpumask))
992 cpuid = PCPU_GET(cpuid);
993 other_cpus = all_cpus;
994 CPU_CLR(cpuid, &other_cpus);
995 if (CPU_ISSET(cpuid, &pmap->pm_active))
990 invltlb();
991 CPU_AND(&other_cpus, &pmap->pm_active);
992 if (!CPU_EMPTY(&other_cpus))
993 smp_masked_invltlb(other_cpus);
994 }
995 sched_unpin();
996}
997

--- 52 unchanged lines hidden (view full) ---

1050 sched_unpin();
1051}
1052
1053static void
1054pmap_update_pde_teardown(void *arg)
1055{
1056 struct pde_action *act = arg;
1057
996 invltlb();
997 CPU_AND(&other_cpus, &pmap->pm_active);
998 if (!CPU_EMPTY(&other_cpus))
999 smp_masked_invltlb(other_cpus);
1000 }
1001 sched_unpin();
1002}
1003

--- 52 unchanged lines hidden (view full) ---

1056 sched_unpin();
1057}
1058
1059static void
1060pmap_update_pde_teardown(void *arg)
1061{
1062 struct pde_action *act = arg;
1063
1058 sched_pin();
1059 if (CPU_OVERLAP(&act->invalidate, PCPU_PTR(cpumask))) {
1060 sched_unpin();
1064 if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate))
1061 pmap_update_pde_invalidate(act->va, act->newpde);
1065 pmap_update_pde_invalidate(act->va, act->newpde);
1062 } else
1063 sched_unpin();
1064}
1065
1066/*
1067 * Change the page size for the specified virtual address in a way that
1068 * prevents any possibility of the TLB ever having two entries that map the
1069 * same virtual address using different page sizes. This is the recommended
1070 * workaround for Erratum 383 on AMD Family 10h processors. It prevents a
1071 * machine check exception for a TLB state that is improperly diagnosed as a
1072 * hardware error.
1073 */
1074static void
1075pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
1076{
1077 struct pde_action act;
1078 cpuset_t active, cpumask, other_cpus;
1079
1080 sched_pin();
1081 cpumask = PCPU_GET(cpumask);
1066}
1067
1068/*
1069 * Change the page size for the specified virtual address in a way that
1070 * prevents any possibility of the TLB ever having two entries that map the
1071 * same virtual address using different page sizes. This is the recommended
1072 * workaround for Erratum 383 on AMD Family 10h processors. It prevents a
1073 * machine check exception for a TLB state that is improperly diagnosed as a
1074 * hardware error.
1075 */
1076static void
1077pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde)
1078{
1079 struct pde_action act;
1080 cpuset_t active, cpumask, other_cpus;
1081
1082 sched_pin();
1083 cpumask = PCPU_GET(cpumask);
1082 other_cpus = PCPU_GET(other_cpus);
1084 other_cpus = all_cpus;
1085 CPU_CLR(PCPU_GET(cpuid), &other_cpus);
1083 if (pmap == kernel_pmap)
1084 active = all_cpus;
1085 else
1086 active = pmap->pm_active;
1087 if (CPU_OVERLAP(&active, &other_cpus)) {
1088 act.store = cpumask;
1089 act.invalidate = active;
1090 act.va = va;

--- 4174 unchanged lines hidden --- 		1086 if (pmap == kernel_pmap)
1087 active = all_cpus;
1088 else
1089 active = pmap->pm_active;
1090 if (CPU_OVERLAP(&active, &other_cpus)) {
1091 act.store = cpumask;
1092 act.invalidate = active;
1093 act.va = va;

--- 4174 unchanged lines hidden ---