pmap.c (013818111a1477055223a69d538ebcd1c23b955e) pmap.c (8a5ac5d56f6fa2d4569c90ce3efe3077588dd9f9)
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>

--- 105 unchanged lines hidden (view full) ---

114#include <sys/kernel.h>
115#include <sys/ktr.h>
116#include <sys/lock.h>
117#include <sys/malloc.h>
118#include <sys/mman.h>
119#include <sys/msgbuf.h>
120#include <sys/mutex.h>
121#include <sys/proc.h>
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005-2008 Alan L. Cox <alc@cs.rice.edu>

--- 105 unchanged lines hidden (view full) ---

114#include <sys/kernel.h>
115#include <sys/ktr.h>
116#include <sys/lock.h>
117#include <sys/malloc.h>
118#include <sys/mman.h>
119#include <sys/msgbuf.h>
120#include <sys/mutex.h>
121#include <sys/proc.h>
122#include <sys/sf_buf.h>
122#include <sys/sx.h>
123#include <sys/vmmeter.h>
124#include <sys/sched.h>
125#include <sys/sysctl.h>
126#ifdef SMP
127#include <sys/smp.h>
128#endif
129

--- 597 unchanged lines hidden (view full) ---

727/***************************************************
728 * Low level helper routines.....
729 ***************************************************/
730
731/*
732 * Determine the appropriate bits to set in a PTE or PDE for a specified
733 * caching mode.
734 */
123#include <sys/sx.h>
124#include <sys/vmmeter.h>
125#include <sys/sched.h>
126#include <sys/sysctl.h>
127#ifdef SMP
128#include <sys/smp.h>
129#endif
130

--- 597 unchanged lines hidden (view full) ---

728/***************************************************
729 * Low level helper routines.....
730 ***************************************************/
731
732/*
733 * Determine the appropriate bits to set in a PTE or PDE for a specified
734 * caching mode.
735 */
735static int
736int
736pmap_cache_bits(int mode, boolean_t is_pde)
737{
738 int pat_flag, pat_index, cache_bits;
739
740 /* The PAT bit is different for PTE's and PDE's. */
741 pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
742
743 /* If we don't support PAT, map extended modes to older ones. */

--- 196 unchanged lines hidden (view full) ---

940PMAP_INLINE void
941pmap_invalidate_cache(void)
942{
943
944 wbinvd();
945}
946#endif /* !SMP */
947
737pmap_cache_bits(int mode, boolean_t is_pde)
738{
739 int pat_flag, pat_index, cache_bits;
740
741 /* The PAT bit is different for PTE's and PDE's. */
742 pat_flag = is_pde ? PG_PDE_PAT : PG_PTE_PAT;
743
744 /* If we don't support PAT, map extended modes to older ones. */

--- 196 unchanged lines hidden (view full) ---

941PMAP_INLINE void
942pmap_invalidate_cache(void)
943{
944
945 wbinvd();
946}
947#endif /* !SMP */
948
949void
950pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
951{
952
953 KASSERT((sva & PAGE_MASK) == 0,
954 ("pmap_invalidate_cache_range: sva not page-aligned"));
955 KASSERT((eva & PAGE_MASK) == 0,
956 ("pmap_invalidate_cache_range: eva not page-aligned"));
957
958 if (cpu_feature & CPUID_SS)
959 ; /* If "Self Snoop" is supported, do nothing. */
960 else if (cpu_feature & CPUID_CLFSH) {
961
962 /*
963 * Otherwise, do per-cache line flush. Use the mfence
964 * instruction to insure that previous stores are
965 * included in the write-back. The processor
966 * propagates flush to other processors in the cache
967 * coherence domain.
968 */
969 mfence();
970 for (; eva < sva; eva += cpu_clflush_line_size)
971 clflush(eva);
972 mfence();
973 } else {
974
975 /*
976 * No targeted cache flush methods are supported by CPU,
977 * globally invalidate cache as a last resort.
978 */
979 pmap_invalidate_cache();
980 }
981}
982
948/*
949 * Are we current address space or kernel? N.B. We return FALSE when
950 * a pmap's page table is in use because a kernel thread is borrowing
951 * it. The borrowed page table can change spontaneously, making any
952 * dependence on its continued use subject to a race condition.
953 */
954static __inline int
955pmap_is_current(pmap_t pmap)

--- 3439 unchanged lines hidden (view full) ---

4395 * Map a set of physical memory pages into the kernel virtual
4396 * address space. Return a pointer to where it is mapped. This
4397 * routine is intended to be used for mapping device memory,
4398 * NOT real memory.
4399 */
4400void *
4401pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
4402{
983/*
984 * Are we current address space or kernel? N.B. We return FALSE when
985 * a pmap's page table is in use because a kernel thread is borrowing
986 * it. The borrowed page table can change spontaneously, making any
987 * dependence on its continued use subject to a race condition.
988 */
989static __inline int
990pmap_is_current(pmap_t pmap)

--- 3439 unchanged lines hidden (view full) ---

4430 * Map a set of physical memory pages into the kernel virtual
4431 * address space. Return a pointer to where it is mapped. This
4432 * routine is intended to be used for mapping device memory,
4433 * NOT real memory.
4434 */
4435void *
4436pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
4437{
4403 vm_offset_t va, tmpva, offset;
4438 vm_offset_t va, offset;
4439 vm_size_t tmpsize;
4404
4405 offset = pa & PAGE_MASK;
4406 size = roundup(offset + size, PAGE_SIZE);
4407 pa = pa & PG_FRAME;
4408
4409 if (pa < KERNLOAD && pa + size <= KERNLOAD)
4410 va = KERNBASE + pa;
4411 else
4412 va = kmem_alloc_nofault(kernel_map, size);
4413 if (!va)
4414 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
4415
4440
4441 offset = pa & PAGE_MASK;
4442 size = roundup(offset + size, PAGE_SIZE);
4443 pa = pa & PG_FRAME;
4444
4445 if (pa < KERNLOAD && pa + size <= KERNLOAD)
4446 va = KERNBASE + pa;
4447 else
4448 va = kmem_alloc_nofault(kernel_map, size);
4449 if (!va)
4450 panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
4451
4416 for (tmpva = va; size > 0; ) {
4417 pmap_kenter_attr(tmpva, pa, mode);
4418 size -= PAGE_SIZE;
4419 tmpva += PAGE_SIZE;
4420 pa += PAGE_SIZE;
4421 }
4422 pmap_invalidate_range(kernel_pmap, va, tmpva);
4423 /* If "Self Snoop" is supported, do nothing. */
4424 if (!(cpu_feature & CPUID_SS))
4425 pmap_invalidate_cache();
4452 for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
4453 pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
4454 pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
4455 pmap_invalidate_cache_range(va, va + size);
4426 return ((void *)(va + offset));
4427}
4428
4429void *
4430pmap_mapdev(vm_paddr_t pa, vm_size_t size)
4431{
4432
4433 return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));

--- 23 unchanged lines hidden (view full) ---

4457}
4458
4459/*
4460 * Sets the memory attribute for the specified page.
4461 */
4462void
4463pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
4464{
4456 return ((void *)(va + offset));
4457}
4458
4459void *
4460pmap_mapdev(vm_paddr_t pa, vm_size_t size)
4461{
4462
4463 return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE));

--- 23 unchanged lines hidden (view full) ---

4487}
4488
4489/*
4490 * Sets the memory attribute for the specified page.
4491 */
4492void
4493pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma)
4494{
4495 struct sysmaps *sysmaps;
4496 vm_offset_t sva, eva;
4465
4466 m->md.pat_mode = ma;
4497
4498 m->md.pat_mode = ma;
4499 if ((m->flags & PG_FICTITIOUS) != 0)
4500 return;
4467
4468 /*
4469 * If "m" is a normal page, flush it from the cache.
4501
4502 /*
4503 * If "m" is a normal page, flush it from the cache.
4504 * See pmap_invalidate_cache_range().
4505 *
4506 * First, try to find an existing mapping of the page by sf
4507 * buffer. sf_buf_invalidate_cache() modifies mapping and
4508 * flushes the cache.
4470 */
4509 */
4471 if ((m->flags & PG_FICTITIOUS) == 0) {
4472 /* If "Self Snoop" is supported, do nothing. */
4473 if (!(cpu_feature & CPUID_SS))
4474 pmap_invalidate_cache();
4510 if (sf_buf_invalidate_cache(m))
4511 return;
4512
4513 /*
4514 * If page is not mapped by sf buffer, but CPU does not
4515 * support self snoop, map the page transient and do
4516 * invalidation. In the worst case, whole cache is flushed by
4517 * pmap_invalidate_cache_range().
4518 */
4519 if ((cpu_feature & (CPUID_SS|CPUID_CLFSH)) == CPUID_CLFSH) {
4520 sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
4521 mtx_lock(&sysmaps->lock);
4522 if (*sysmaps->CMAP2)
4523 panic("pmap_page_set_memattr: CMAP2 busy");
4524 sched_pin();
4525 *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) |
4526 PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0);
4527 invlcaddr(sysmaps->CADDR2);
4528 sva = (vm_offset_t)sysmaps->CADDR2;
4529 eva = sva + PAGE_SIZE;
4530 } else
4531 sva = eva = 0; /* gcc */
4532 pmap_invalidate_cache_range(sva, eva);
4533 if (sva != 0) {
4534 *sysmaps->CMAP2 = 0;
4535 sched_unpin();
4536 mtx_unlock(&sysmaps->lock);
4475 }
4476}
4477
4478int
4479pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
4480{
4481 vm_offset_t base, offset, tmpva;
4482 pt_entry_t *pte;

--- 49 unchanged lines hidden (view full) ---

4532 }
4533
4534 /*
4535 * Flush CPU caches to make sure any data isn't cached that shouldn't
4536 * be, etc.
4537 */
4538 if (changed) {
4539 pmap_invalidate_range(kernel_pmap, base, tmpva);
4537 }
4538}
4539
4540int
4541pmap_change_attr(vm_offset_t va, vm_size_t size, int mode)
4542{
4543 vm_offset_t base, offset, tmpva;
4544 pt_entry_t *pte;

--- 49 unchanged lines hidden (view full) ---

4594 }
4595
4596 /*
4597 * Flush CPU caches to make sure any data isn't cached that shouldn't
4598 * be, etc.
4599 */
4600 if (changed) {
4601 pmap_invalidate_range(kernel_pmap, base, tmpva);
4540 /* If "Self Snoop" is supported, do nothing. */
4541 if (!(cpu_feature & CPUID_SS))
4542 pmap_invalidate_cache();
4602 pmap_invalidate_cache_range(base, tmpva);
4543 }
4544 return (0);
4545}
4546
4547/*
4548 * perform the pmap work for mincore
4549 */
4550int

--- 234 unchanged lines hidden --- 		4603 }
4604 return (0);
4605}
4606
4607/*
4608 * perform the pmap work for mincore
4609 */
4610int

--- 234 unchanged lines hidden ---