Annotation of sys/arch/amd64/amd64/pmap.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: pmap.c,v 1.27 2007/06/01 20:10:04 tedu Exp $ */
! 2: /* $NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $ */
! 3:
! 4: /*
! 5: *
! 6: * Copyright (c) 1997 Charles D. Cranor and Washington University.
! 7: * All rights reserved.
! 8: *
! 9: * Redistribution and use in source and binary forms, with or without
! 10: * modification, are permitted provided that the following conditions
! 11: * are met:
! 12: * 1. Redistributions of source code must retain the above copyright
! 13: * notice, this list of conditions and the following disclaimer.
! 14: * 2. Redistributions in binary form must reproduce the above copyright
! 15: * notice, this list of conditions and the following disclaimer in the
! 16: * documentation and/or other materials provided with the distribution.
! 17: * 3. All advertising materials mentioning features or use of this software
! 18: * must display the following acknowledgement:
! 19: * This product includes software developed by Charles D. Cranor and
! 20: * Washington University.
! 21: * 4. The name of the author may not be used to endorse or promote products
! 22: * derived from this software without specific prior written permission.
! 23: *
! 24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
! 25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
! 26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
! 27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
! 28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
! 29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
! 30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
! 31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
! 32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
! 33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
! 34: */
! 35:
! 36: /*
! 37: * Copyright 2001 (c) Wasabi Systems, Inc.
! 38: * All rights reserved.
! 39: *
! 40: * Written by Frank van der Linden for Wasabi Systems, Inc.
! 41: *
! 42: * Redistribution and use in source and binary forms, with or without
! 43: * modification, are permitted provided that the following conditions
! 44: * are met:
! 45: * 1. Redistributions of source code must retain the above copyright
! 46: * notice, this list of conditions and the following disclaimer.
! 47: * 2. Redistributions in binary form must reproduce the above copyright
! 48: * notice, this list of conditions and the following disclaimer in the
! 49: * documentation and/or other materials provided with the distribution.
! 50: * 3. All advertising materials mentioning features or use of this software
! 51: * must display the following acknowledgement:
! 52: * This product includes software developed for the NetBSD Project by
! 53: * Wasabi Systems, Inc.
! 54: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
! 55: * or promote products derived from this software without specific prior
! 56: * written permission.
! 57: *
! 58: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
! 59: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
! 60: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
! 61: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
! 62: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
! 63: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
! 64: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
! 65: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
! 66: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
! 67: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
! 68: * POSSIBILITY OF SUCH DAMAGE.
! 69: */
! 70:
! 71: /*
! 72: * This is the i386 pmap modified and generalized to support x86-64
! 73: * as well. The idea is to hide the upper N levels of the page tables
! 74: * inside pmap_get_ptp, pmap_free_ptp and pmap_growkernel. The rest
! 75: * is mostly untouched, except that it uses some more generalized
! 76: * macros and interfaces.
! 77: *
! 78: * This pmap has been tested on the i386 as well, and it can be easily
! 79: * adapted to PAE.
! 80: *
! 81: * fvdl@wasabisystems.com 18-Jun-2001
! 82: */
! 83:
! 84: /*
! 85: * pmap.c: i386 pmap module rewrite
! 86: * Chuck Cranor <chuck@ccrc.wustl.edu>
! 87: * 11-Aug-97
! 88: *
! 89: * history of this pmap module: in addition to my own input, i used
! 90: * the following references for this rewrite of the i386 pmap:
! 91: *
! 92: * [1] the NetBSD i386 pmap. this pmap appears to be based on the
! 93: * BSD hp300 pmap done by Mike Hibler at University of Utah.
! 94: * it was then ported to the i386 by William Jolitz of UUNET
! 95: * Technologies, Inc. Then Charles M. Hannum of the NetBSD
! 96: * project fixed some bugs and provided some speed ups.
! 97: *
! 98: * [2] the FreeBSD i386 pmap. this pmap seems to be the
! 99: * Hibler/Jolitz pmap, as modified for FreeBSD by John S. Dyson
! 100: * and David Greenman.
! 101: *
! 102: * [3] the Mach pmap. this pmap, from CMU, seems to have migrated
! 103: * between several processors. the VAX version was done by
! 104: * Avadis Tevanian, Jr., and Michael Wayne Young. the i386
! 105: * version was done by Lance Berc, Mike Kupfer, Bob Baron,
! 106: * David Golub, and Richard Draves. the alpha version was
! 107: * done by Alessandro Forin (CMU/Mach) and Chris Demetriou
! 108: * (NetBSD/alpha).
! 109: */
! 110:
! 111: #include <sys/param.h>
! 112: #include <sys/systm.h>
! 113: #include <sys/proc.h>
! 114: #include <sys/malloc.h>
! 115: #include <sys/pool.h>
! 116: #include <sys/user.h>
! 117: #include <sys/kernel.h>
! 118: #include <sys/mutex.h>
! 119: #include <sys/sched.h>
! 120:
! 121: #include <uvm/uvm.h>
! 122:
! 123: #include <machine/atomic.h>
! 124: #include <machine/cpu.h>
! 125: #include <machine/specialreg.h>
! 126: #include <machine/gdt.h>
! 127:
! 128: #include <dev/isa/isareg.h>
! 129: #include <machine/isa_machdep.h>
! 130:
! 131: /*
! 132: * general info:
! 133: *
! 134: * - for an explanation of how the i386 MMU hardware works see
! 135: * the comments in <machine/pte.h>.
! 136: *
! 137: * - for an explanation of the general memory structure used by
! 138: * this pmap (including the recursive mapping), see the comments
! 139: * in <machine/pmap.h>.
! 140: *
! 141: * this file contains the code for the "pmap module." the module's
! 142: * job is to manage the hardware's virtual to physical address mappings.
! 143: * note that there are two levels of mapping in the VM system:
! 144: *
! 145: * [1] the upper layer of the VM system uses vm_map's and vm_map_entry's
! 146: * to map ranges of virtual address space to objects/files. for
! 147: * example, the vm_map may say: "map VA 0x1000 to 0x22000 read-only
! 148: * to the file /bin/ls starting at offset zero." note that
! 149: * the upper layer mapping is not concerned with how individual
! 150: * vm_pages are mapped.
! 151: *
! 152: * [2] the lower layer of the VM system (the pmap) maintains the mappings
! 153: * from virtual addresses. it is concerned with which vm_page is
! 154: * mapped where. for example, when you run /bin/ls and start
! 155: * at page 0x1000 the fault routine may lookup the correct page
! 156: * of the /bin/ls file and then ask the pmap layer to establish
! 157: * a mapping for it.
! 158: *
! 159: * note that information in the lower layer of the VM system can be
! 160: * thrown away since it can easily be reconstructed from the info
! 161: * in the upper layer.
! 162: *
! 163: * data structures we use include:
! 164: *
! 165: * - struct pmap: describes the address space of one thread
! 166: * - struct pv_entry: describes one <PMAP,VA> mapping of a PA
! 167: * - pmap_remove_record: a list of virtual addresses whose mappings
! 168: * have been changed. used for TLB flushing.
! 169: */
! 170:
! 171: /*
! 172: * memory allocation
! 173: *
! 174: * - there are three data structures that we must dynamically allocate:
! 175: *
! 176: * [A] new process' page directory page (PDP)
! 177: * - plan 1: done at pmap_create() we use
! 178: * uvm_km_alloc(kernel_map, PAGE_SIZE) [fka kmem_alloc] to do this
! 179: * allocation.
! 180: *
! 181: * if we are low in free physical memory then we sleep in
! 182: * uvm_km_alloc -- in this case this is ok since we are creating
! 183: * a new pmap and should not be holding any locks.
! 184: *
! 185: * if the kernel is totally out of virtual space
! 186: * (i.e. uvm_km_alloc returns NULL), then we panic.
! 187: *
! 188: * XXX: the fork code currently has no way to return an "out of
! 189: * memory, try again" error code since uvm_fork [fka vm_fork]
! 190: * is a void function.
! 191: *
! 192: * [B] new page tables pages (PTP)
! 193: * call uvm_pagealloc()
! 194: * => success: zero page, add to pm_pdir
! 195: * => failure: we are out of free vm_pages, let pmap_enter()
! 196: * tell UVM about it.
! 197: *
! 198: * note: for kernel PTPs, we start with NKPTP of them. as we map
! 199: * kernel memory (at uvm_map time) we check to see if we've grown
! 200: * the kernel pmap. if so, we call the optional function
! 201: * pmap_growkernel() to grow the kernel PTPs in advance.
! 202: *
! 203: * [C] pv_entry structures
! 204: * - try to allocate one from the pool.
! 205: * If we fail, we simply let pmap_enter() tell UVM about it.
! 206: */
! 207:
! 208: /*
! 209: * locking
! 210: *
! 211: * we have the following locks that we must contend with:
! 212: *
! 213: * "simple" locks:
! 214: *
! 215: * - pmap lock (per pmap, part of uvm_object)
! 216: * this lock protects the fields in the pmap structure including
! 217: * the non-kernel PDEs in the PDP, and the PTEs. it also locks
! 218: * in the alternate PTE space (since that is determined by the
! 219: * entry in the PDP).
! 220: *
! 221: * - pmaps_lock
! 222: * this lock protects the list of active pmaps (headed by "pmaps").
! 223: * we lock it when adding or removing pmaps from this list.
! 224: *
! 225: * XXX: would be nice to have per-CPU VAs for the above 4
! 226: */
! 227:
! 228: /*
! 229: * locking data structures
! 230: */
! 231:
! 232: vaddr_t ptp_masks[] = PTP_MASK_INITIALIZER;
! 233: int ptp_shifts[] = PTP_SHIFT_INITIALIZER;
! 234: long nkptp[] = NKPTP_INITIALIZER;
! 235: long nkptpmax[] = NKPTPMAX_INITIALIZER;
! 236: long nbpd[] = NBPD_INITIALIZER;
! 237: pd_entry_t *normal_pdes[] = PDES_INITIALIZER;
! 238: pd_entry_t *alternate_pdes[] = APDES_INITIALIZER;
! 239:
! 240: /* int nkpde = NKPTP; */
! 241:
! 242: struct simplelock pmaps_lock;
! 243:
! 244: #define PMAP_MAP_TO_HEAD_LOCK() /* null */
! 245: #define PMAP_MAP_TO_HEAD_UNLOCK() /* null */
! 246:
! 247: #define PMAP_HEAD_TO_MAP_LOCK() /* null */
! 248: #define PMAP_HEAD_TO_MAP_UNLOCK() /* null */
! 249:
! 250: #define COUNT(x) /* nothing */
! 251:
! 252: /*
! 253: * global data structures
! 254: */
! 255:
! 256: struct pmap kernel_pmap_store; /* the kernel's pmap (proc0) */
! 257:
! 258: /*
! 259: * pmap_pg_g: if our processor supports PG_G in the PTE then we
! 260: * set pmap_pg_g to PG_G (otherwise it is zero).
! 261: */
! 262:
! 263: int pmap_pg_g = 0;
! 264:
! 265: /*
! 266: * i386 physical memory comes in a big contig chunk with a small
! 267: * hole toward the front of it... the following 4 paddr_t's
! 268: * (shared with machdep.c) describe the physical address space
! 269: * of this machine.
! 270: */
! 271: paddr_t avail_start; /* PA of first available physical page */
! 272: paddr_t avail_end; /* PA of last available physical page */
! 273:
! 274: /*
! 275: * other data structures
! 276: */
! 277:
! 278: pt_entry_t protection_codes[8]; /* maps MI prot to i386 prot code */
! 279: boolean_t pmap_initialized = FALSE; /* pmap_init done yet? */
! 280:
! 281: /*
! 282: * pv management structures.
! 283: */
! 284: struct pool pmap_pv_pool;
! 285:
! 286: /*
! 287: * linked list of all non-kernel pmaps
! 288: */
! 289:
! 290: struct pmap_head pmaps;
! 291:
! 292: /*
! 293: * pool that pmap structures are allocated from
! 294: */
! 295:
! 296: struct pool pmap_pmap_pool;
! 297:
! 298: /*
! 299: * When we're freeing a ptp, we need to delay the freeing until all
! 300: * tlb shootdown has been done. This is the list of the to-be-freed pages.
! 301: */
! 302: TAILQ_HEAD(pg_to_free, vm_page);
! 303:
! 304: /*
! 305: * pool and cache that PDPs are allocated from
! 306: */
! 307:
! 308: struct pool pmap_pdp_pool;
! 309: struct pool_cache pmap_pdp_cache;
! 310: u_int pmap_pdp_cache_generation;
! 311:
! 312: int pmap_pdp_ctor(void *, void *, int);
! 313:
! 314: extern vaddr_t msgbuf_vaddr;
! 315: extern paddr_t msgbuf_paddr;
! 316:
! 317: extern vaddr_t idt_vaddr; /* we allocate IDT early */
! 318: extern paddr_t idt_paddr;
! 319:
! 320: #ifdef _LP64
! 321: extern vaddr_t lo32_vaddr;
! 322: extern vaddr_t lo32_paddr;
! 323: #endif
! 324:
! 325: vaddr_t virtual_avail;
! 326: extern int end;
! 327:
! 328: /*
! 329: * local prototypes
! 330: */
! 331:
! 332: void pmap_enter_pv(struct vm_page *, struct pv_entry *, struct pmap *,
! 333: vaddr_t, struct vm_page *);
! 334: struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t, pd_entry_t **);
! 335: struct vm_page *pmap_find_ptp(struct pmap *, vaddr_t, paddr_t, int);
! 336: void pmap_free_ptp(struct pmap *, struct vm_page *,
! 337: vaddr_t, pt_entry_t *, pd_entry_t **, struct pg_to_free *);
! 338: void pmap_freepage(struct pmap *, struct vm_page *, int, struct pg_to_free *);
! 339: static boolean_t pmap_is_active(struct pmap *, int);
! 340: void pmap_map_ptes(struct pmap *, pt_entry_t **, pd_entry_t ***);
! 341: struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t);
! 342: void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int);
! 343: boolean_t pmap_remove_pte(struct pmap *, struct vm_page *, pt_entry_t *,
! 344: vaddr_t, int);
! 345: void pmap_remove_ptes(struct pmap *, struct vm_page *, vaddr_t,
! 346: vaddr_t, vaddr_t, int);
! 347: #define PMAP_REMOVE_ALL 0 /* remove all mappings */
! 348: #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */
! 349:
! 350: void pmap_unmap_ptes(struct pmap *);
! 351: boolean_t pmap_get_physpage(vaddr_t, int, paddr_t *);
! 352: boolean_t pmap_pdes_valid(vaddr_t, pd_entry_t **, pd_entry_t *);
! 353: void pmap_alloc_level(pd_entry_t **, vaddr_t, int, long *);
! 354: void pmap_apte_flush(struct pmap *pmap);
! 355:
! 356: void pmap_sync_flags_pte(struct vm_page *, u_long);
! 357:
! 358: /*
! 359: * p m a p i n l i n e h e l p e r f u n c t i o n s
! 360: */
! 361:
! 362: /*
! 363: * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
! 364: * of course the kernel is always loaded
! 365: */
! 366:
! 367: static __inline boolean_t
! 368: pmap_is_curpmap(struct pmap *pmap)
! 369: {
! 370: return((pmap == pmap_kernel()) ||
! 371: (pmap->pm_pdirpa == (paddr_t) rcr3()));
! 372: }
! 373:
! 374: /*
! 375: * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
! 376: */
! 377:
! 378: static __inline boolean_t
! 379: pmap_is_active(struct pmap *pmap, int cpu_id)
! 380: {
! 381: return (pmap == pmap_kernel() ||
! 382: (pmap->pm_cpus & (1U << cpu_id)) != 0);
! 383: }
! 384:
! 385: static __inline u_int
! 386: pmap_pte2flags(u_long pte)
! 387: {
! 388: return (((pte & PG_U) ? PG_PMAP_REF : 0) |
! 389: ((pte & PG_M) ? PG_PMAP_MOD : 0));
! 390: }
! 391:
! 392: void
! 393: pmap_sync_flags_pte(struct vm_page *pg, u_long pte)
! 394: {
! 395: if (pte & (PG_U|PG_M)) {
! 396: atomic_setbits_int(&pg->pg_flags, pmap_pte2flags(pte));
! 397: }
! 398: }
! 399:
! 400: void
! 401: pmap_apte_flush(struct pmap *pmap)
! 402: {
! 403: pmap_tlb_shoottlb();
! 404: pmap_tlb_shootwait();
! 405: }
! 406:
! 407: /*
! 408: * pmap_map_ptes: map a pmap's PTEs into KVM and lock them in
! 409: *
! 410: * => we lock enough pmaps to keep things locked in
! 411: * => must be undone with pmap_unmap_ptes before returning
! 412: */
! 413:
! 414: void
! 415: pmap_map_ptes(struct pmap *pmap, pt_entry_t **ptepp, pd_entry_t ***pdeppp)
! 416: {
! 417: pd_entry_t opde, npde;
! 418:
! 419: /* the kernel's pmap is always accessible */
! 420: if (pmap == pmap_kernel()) {
! 421: *ptepp = PTE_BASE;
! 422: *pdeppp = normal_pdes;
! 423: return;
! 424: }
! 425:
! 426: /* if curpmap then we are always mapped */
! 427: if (pmap_is_curpmap(pmap)) {
! 428: simple_lock(&pmap->pm_lock);
! 429: *ptepp = PTE_BASE;
! 430: *pdeppp = normal_pdes;
! 431: return;
! 432: }
! 433:
! 434: /* need to lock both curpmap and pmap: use ordered locking */
! 435: if ((unsigned long) pmap < (unsigned long) curpcb->pcb_pmap) {
! 436: simple_lock(&pmap->pm_lock);
! 437: simple_lock(&curpcb->pcb_pmap->pm_lock);
! 438: } else {
! 439: simple_lock(&curpcb->pcb_pmap->pm_lock);
! 440: simple_lock(&pmap->pm_lock);
! 441: }
! 442:
! 443: /* need to load a new alternate pt space into curpmap? */
! 444: opde = *APDP_PDE;
! 445: if (!pmap_valid_entry(opde) || (opde & PG_FRAME) != pmap->pm_pdirpa) {
! 446: npde = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V);
! 447: *APDP_PDE = npde;
! 448: if (pmap_valid_entry(opde))
! 449: pmap_apte_flush(curpcb->pcb_pmap);
! 450: }
! 451: *ptepp = APTE_BASE;
! 452: *pdeppp = alternate_pdes;
! 453: }
! 454:
! 455: /*
! 456: * pmap_unmap_ptes: unlock the PTE mapping of "pmap"
! 457: */
! 458:
! 459: void
! 460: pmap_unmap_ptes(struct pmap *pmap)
! 461: {
! 462: if (pmap == pmap_kernel()) {
! 463: return;
! 464: }
! 465: if (pmap_is_curpmap(pmap)) {
! 466: simple_unlock(&pmap->pm_lock);
! 467: } else {
! 468: #if defined(MULTIPROCESSOR)
! 469: *APDP_PDE = 0;
! 470: pmap_apte_flush(curpcb->pcb_pmap);
! 471: #endif
! 472: COUNT(apdp_pde_unmap);
! 473: simple_unlock(&pmap->pm_lock);
! 474: simple_unlock(&curpcb->pcb_pmap->pm_lock);
! 475: }
! 476: }
! 477:
! 478: /*
! 479: * p m a p k e n t e r f u n c t i o n s
! 480: *
! 481: * functions to quickly enter/remove pages from the kernel address
! 482: * space. pmap_kremove is exported to MI kernel. we make use of
! 483: * the recursive PTE mappings.
! 484: */
! 485:
! 486: /*
! 487: * pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking
! 488: *
! 489: * => no need to lock anything, assume va is already allocated
! 490: * => should be faster than normal pmap enter function
! 491: */
! 492:
! 493: void
! 494: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
! 495: {
! 496: pt_entry_t *pte, opte, npte;
! 497:
! 498: pte = kvtopte(va);
! 499:
! 500: npte = pa | ((prot & VM_PROT_WRITE) ? PG_RW : PG_RO) |
! 501: PG_V | pmap_pg_g;
! 502: if ((cpu_feature & CPUID_NXE) && !(prot & VM_PROT_EXECUTE))
! 503: npte |= PG_NX;
! 504: opte = pmap_pte_set(pte, npte);
! 505: #ifdef LARGEPAGES
! 506: /* XXX For now... */
! 507: if (opte & PG_PS)
! 508: panic("pmap_kenter_pa: PG_PS");
! 509: #endif
! 510: if (pmap_valid_entry(opte)) {
! 511: /* This shouldn't happen */
! 512: pmap_tlb_shootpage(pmap_kernel(), va);
! 513: pmap_tlb_shootwait();
! 514: }
! 515: }
! 516:
! 517: /*
! 518: * pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking
! 519: *
! 520: * => no need to lock anything
! 521: * => caller must dispose of any vm_page mapped in the va range
! 522: * => note: not an inline function
! 523: * => we assume the va is page aligned and the len is a multiple of PAGE_SIZE
! 524: * => we assume kernel only unmaps valid addresses and thus don't bother
! 525: * checking the valid bit before doing TLB flushing
! 526: */
! 527:
! 528: void
! 529: pmap_kremove(vaddr_t sva, vsize_t len)
! 530: {
! 531: pt_entry_t *pte, opte;
! 532: vaddr_t va, eva;
! 533:
! 534: eva = sva + len;
! 535:
! 536: for (va = sva; va != eva; va += PAGE_SIZE) {
! 537: pte = kvtopte(va);
! 538:
! 539: opte = pmap_pte_set(pte, 0);
! 540: #ifdef LARGEPAGES
! 541: KASSERT((opte & PG_PS) == 0);
! 542: #endif
! 543: KASSERT((opte & PG_PVLIST) == 0);
! 544: }
! 545:
! 546: pmap_tlb_shootrange(pmap_kernel(), sva, eva);
! 547: pmap_tlb_shootwait();
! 548: }
! 549:
! 550: /*
! 551: * p m a p i n i t f u n c t i o n s
! 552: *
! 553: * pmap_bootstrap and pmap_init are called during system startup
! 554: * to init the pmap module. pmap_bootstrap() does a low level
! 555: * init just to get things rolling. pmap_init() finishes the job.
! 556: */
! 557:
! 558: /*
! 559: * pmap_bootstrap: get the system in a state where it can run with VM
! 560: * properly enabled (called before main()). the VM system is
! 561: * fully init'd later...
! 562: *
! 563: * => on i386, locore.s has already enabled the MMU by allocating
! 564: * a PDP for the kernel, and nkpde PTP's for the kernel.
! 565: * => kva_start is the first free virtual address in kernel space
! 566: */
! 567:
! 568: void
! 569: pmap_bootstrap(vaddr_t kva_start, paddr_t max_pa)
! 570: {
! 571: vaddr_t kva, kva_end;
! 572: struct pmap *kpm;
! 573: int i;
! 574: unsigned long p1i;
! 575: pt_entry_t pg_nx = (cpu_feature & CPUID_NXE? PG_NX : 0);
! 576: long ndmpdp;
! 577: paddr_t dmpd, dmpdp;
! 578:
! 579: /*
! 580: * define the voundaries of the managed kernel virtual address
! 581: * space.
! 582: */
! 583:
! 584: virtual_avail = kva_start; /* first free KVA */
! 585:
! 586: /*
! 587: * set up protection_codes: we need to be able to convert from
! 588: * a MI protection code (some combo of VM_PROT...) to something
! 589: * we can jam into a i386 PTE.
! 590: */
! 591:
! 592: protection_codes[VM_PROT_NONE] = pg_nx; /* --- */
! 593: protection_codes[VM_PROT_EXECUTE] = PG_RO; /* --x */
! 594: protection_codes[VM_PROT_READ] = PG_RO | pg_nx; /* -r- */
! 595: protection_codes[VM_PROT_READ|VM_PROT_EXECUTE] = PG_RO; /* -rx */
! 596: protection_codes[VM_PROT_WRITE] = PG_RW | pg_nx; /* w-- */
! 597: protection_codes[VM_PROT_WRITE|VM_PROT_EXECUTE] = PG_RW;/* w-x */
! 598: protection_codes[VM_PROT_WRITE|VM_PROT_READ] = PG_RW | pg_nx;
! 599: /* wr- */
! 600: protection_codes[VM_PROT_ALL] = PG_RW; /* wrx */
! 601:
! 602: /*
! 603: * now we init the kernel's pmap
! 604: *
! 605: * the kernel pmap's pm_obj is not used for much. however, in
! 606: * user pmaps the pm_obj contains the list of active PTPs.
! 607: * the pm_obj currently does not have a pager. it might be possible
! 608: * to add a pager that would allow a process to read-only mmap its
! 609: * own page tables (fast user level vtophys?). this may or may not
! 610: * be useful.
! 611: */
! 612:
! 613: kpm = pmap_kernel();
! 614: for (i = 0; i < PTP_LEVELS - 1; i++) {
! 615: simple_lock_init(&kpm->pm_obj[i].vmobjlock);
! 616: kpm->pm_obj[i].pgops = NULL;
! 617: TAILQ_INIT(&kpm->pm_obj[i].memq);
! 618: kpm->pm_obj[i].uo_npages = 0;
! 619: kpm->pm_obj[i].uo_refs = 1;
! 620: kpm->pm_ptphint[i] = NULL;
! 621: }
! 622: memset(&kpm->pm_list, 0, sizeof(kpm->pm_list)); /* pm_list not used */
! 623: kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE);
! 624: kpm->pm_pdirpa = (u_int32_t) proc0.p_addr->u_pcb.pcb_cr3;
! 625: kpm->pm_stats.wired_count = kpm->pm_stats.resident_count =
! 626: atop(kva_start - VM_MIN_KERNEL_ADDRESS);
! 627:
! 628: /*
! 629: * the above is just a rough estimate and not critical to the proper
! 630: * operation of the system.
! 631: */
! 632:
! 633: curpcb->pcb_pmap = kpm; /* proc0's pcb */
! 634:
! 635: /*
! 636: * enable global TLB entries.
! 637: */
! 638: pmap_pg_g = PG_G; /* enable software */
! 639:
! 640: /* add PG_G attribute to already mapped kernel pages */
! 641: #if KERNBASE == VM_MIN_KERNEL_ADDRESS
! 642: for (kva = VM_MIN_KERNEL_ADDRESS ; kva < virtual_avail ;
! 643: #else
! 644: kva_end = roundup((vaddr_t)&end, PAGE_SIZE);
! 645: for (kva = KERNBASE; kva < kva_end ;
! 646: #endif
! 647: kva += PAGE_SIZE) {
! 648: p1i = pl1_i(kva);
! 649: if (pmap_valid_entry(PTE_BASE[p1i]))
! 650: PTE_BASE[p1i] |= PG_G;
! 651: }
! 652:
! 653: /*
! 654: * Map the direct map. The first 4GB were mapped in locore, here
! 655: * we map the rest if it exists. We actually use the direct map
! 656: * here to set up the page tables, we're assuming that we're still
! 657: * operating in the lower 4GB of memory.
! 658: */
! 659: ndmpdp = (max_pa + NBPD_L3 - 1) >> L3_SHIFT;
! 660: if (ndmpdp < NDML2_ENTRIES)
! 661: ndmpdp = NDML2_ENTRIES; /* At least 4GB */
! 662:
! 663: dmpdp = kpm->pm_pdir[PDIR_SLOT_DIRECT] & PG_FRAME;
! 664:
! 665: dmpd = avail_start; avail_start += ndmpdp * PAGE_SIZE;
! 666:
! 667: for (i = NDML2_ENTRIES; i < NPDPG * ndmpdp; i++) {
! 668: paddr_t pdp;
! 669: vaddr_t va;
! 670:
! 671: pdp = (paddr_t)&(((pd_entry_t *)dmpd)[i]);
! 672: va = PMAP_DIRECT_MAP(pdp);
! 673:
! 674: *((pd_entry_t *)va) = ((paddr_t)i << L2_SHIFT);
! 675: *((pd_entry_t *)va) |= PG_RW | PG_V | PG_PS | PG_G | PG_U |
! 676: PG_M;
! 677: }
! 678:
! 679: for (i = NDML2_ENTRIES; i < ndmpdp; i++) {
! 680: paddr_t pdp;
! 681: vaddr_t va;
! 682:
! 683: pdp = (paddr_t)&(((pd_entry_t *)dmpdp)[i]);
! 684: va = PMAP_DIRECT_MAP(pdp);
! 685:
! 686: *((pd_entry_t *)va) = dmpd + (i << PAGE_SHIFT);
! 687: *((pd_entry_t *)va) |= PG_RW | PG_V | PG_U | PG_M;
! 688: }
! 689:
! 690: kpm->pm_pdir[PDIR_SLOT_DIRECT] = dmpdp | PG_V | PG_KW | PG_U |
! 691: PG_M;
! 692:
! 693: /*
! 694: * Now do the same thing, but for the direct uncached map.
! 695: */
! 696: ndmpdp = (max_pa + NBPD_L3 - 1) >> L3_SHIFT;
! 697: if (ndmpdp < NDML2_ENTRIES)
! 698: ndmpdp = NDML2_ENTRIES; /* At least 4GB */
! 699:
! 700: dmpdp = avail_start; avail_start += PAGE_SIZE;
! 701: dmpd = avail_start; avail_start += ndmpdp * PAGE_SIZE;
! 702:
! 703: for (i = 0; i < NPDPG * ndmpdp; i++) {
! 704: paddr_t pdp;
! 705: vaddr_t va;
! 706:
! 707: pdp = (paddr_t)&(((pd_entry_t *)dmpd)[i]);
! 708: va = PMAP_DIRECT_MAP(pdp);
! 709:
! 710: *((pd_entry_t *)va) = (paddr_t)i << L2_SHIFT;
! 711: *((pd_entry_t *)va) |= PG_RW | PG_V | PG_PS | PG_G | PG_N |
! 712: PG_U | PG_M;
! 713: }
! 714:
! 715: for (i = 0; i < ndmpdp; i++) {
! 716: paddr_t pdp;
! 717: vaddr_t va;
! 718:
! 719: pdp = (paddr_t)&(((pd_entry_t *)dmpdp)[i]);
! 720: va = PMAP_DIRECT_MAP(pdp);
! 721:
! 722: *((pd_entry_t *)va) = dmpd + (i << PAGE_SHIFT);
! 723: *((pd_entry_t *)va) |= PG_RW | PG_V | PG_U | PG_M;
! 724: }
! 725:
! 726: kpm->pm_pdir[PDIR_SLOT_DIRECT_NC] = dmpdp | PG_V | PG_KW | PG_U |
! 727: PG_M;
! 728:
! 729: tlbflush();
! 730:
! 731: msgbuf_vaddr = virtual_avail;
! 732: virtual_avail += round_page(MSGBUFSIZE);
! 733:
! 734: idt_vaddr = virtual_avail;
! 735: virtual_avail += 2 * PAGE_SIZE;
! 736: idt_paddr = avail_start; /* steal a page */
! 737: avail_start += 2 * PAGE_SIZE;
! 738:
! 739: #ifdef _LP64
! 740: /*
! 741: * Grab a page below 4G for things that need it (i.e.
! 742: * having an initial %cr3 for the MP trampoline).
! 743: */
! 744: lo32_vaddr = virtual_avail;
! 745: virtual_avail += PAGE_SIZE;
! 746: lo32_paddr = avail_start;
! 747: avail_start += PAGE_SIZE;
! 748: #endif
! 749:
! 750: /*
! 751: * now we reserve some VM for mapping pages when doing a crash dump
! 752: */
! 753:
! 754: virtual_avail = reserve_dumppages(virtual_avail);
! 755:
! 756: /*
! 757: * init the static-global locks and global lists.
! 758: */
! 759:
! 760: simple_lock_init(&pmaps_lock);
! 761: LIST_INIT(&pmaps);
! 762:
! 763: /*
! 764: * initialize the pmap pool.
! 765: */
! 766:
! 767: pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
! 768: &pool_allocator_nointr);
! 769: pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
! 770: &pool_allocator_nointr);
! 771: pool_sethiwat(&pmap_pv_pool, 32 * 1024);
! 772:
! 773: /*
! 774: * initialize the PDE pool and cache.
! 775: */
! 776:
! 777: pool_init(&pmap_pdp_pool, PAGE_SIZE, 0, 0, 0, "pdppl",
! 778: &pool_allocator_nointr);
! 779: pool_cache_init(&pmap_pdp_cache, &pmap_pdp_pool,
! 780: pmap_pdp_ctor, NULL, NULL);
! 781:
! 782: /*
! 783: * ensure the TLB is sync'd with reality by flushing it...
! 784: */
! 785:
! 786: tlbflush();
! 787: }
! 788:
! 789: /*
! 790: * Pre-allocate PTPs for low memory, so that 1:1 mappings for various
! 791: * trampoline code can be entered.
! 792: */
! 793: void
! 794: pmap_prealloc_lowmem_ptps(void)
! 795: {
! 796: pd_entry_t *pdes;
! 797: int level;
! 798: paddr_t newp;
! 799:
! 800: pdes = pmap_kernel()->pm_pdir;
! 801: level = PTP_LEVELS;
! 802: for (;;) {
! 803: newp = avail_start;
! 804: avail_start += PAGE_SIZE;
! 805: memset((void *)PMAP_DIRECT_MAP(newp), 0, PAGE_SIZE);
! 806: pdes[pl_i(0, level)] = (newp & PG_FRAME) | PG_V | PG_RW;
! 807: level--;
! 808: if (level <= 1)
! 809: break;
! 810: pdes = normal_pdes[level - 2];
! 811: }
! 812: }
! 813:
! 814: /*
! 815: * pmap_init: called from uvm_init, our job is to get the pmap
! 816: * system ready to manage mappings... this mainly means initing
! 817: * the pv_entry stuff.
! 818: */
! 819:
! 820: void
! 821: pmap_init(void)
! 822: {
! 823: /*
! 824: * done: pmap module is up (and ready for business)
! 825: */
! 826:
! 827: pmap_initialized = TRUE;
! 828: }
! 829:
! 830: /*
! 831: * p v _ e n t r y f u n c t i o n s
! 832: */
! 833:
! 834: /*
! 835: * main pv_entry manipulation functions:
! 836: * pmap_enter_pv: enter a mapping onto a pv list
! 837: * pmap_remove_pv: remove a mapping from a pv list
! 838: */
! 839:
! 840: /*
! 841: * pmap_enter_pv: enter a mapping onto a pv list
! 842: *
! 843: * => caller should have pmap locked
! 844: * => caller should adjust ptp's wire_count before calling
! 845: *
! 846: * pve: preallocated pve for us to use
! 847: * ptp: PTP in pmap that maps this VA
! 848: */
! 849:
! 850: void
! 851: pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap,
! 852: vaddr_t va, struct vm_page *ptp)
! 853: {
! 854: pve->pv_pmap = pmap;
! 855: pve->pv_va = va;
! 856: pve->pv_ptp = ptp; /* NULL for kernel pmap */
! 857: pve->pv_next = pg->mdpage.pv_list; /* add to ... */
! 858: pg->mdpage.pv_list = pve; /* ... list */
! 859: }
! 860:
! 861: /*
! 862: * pmap_remove_pv: try to remove a mapping from a pv_list
! 863: *
! 864: * => pmap should be locked
! 865: * => caller should adjust ptp's wire_count and free PTP if needed
! 866: * => we return the removed pve
! 867: */
! 868:
! 869: struct pv_entry *
! 870: pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va)
! 871: {
! 872: struct pv_entry *pve, **prevptr;
! 873:
! 874: prevptr = &pg->mdpage.pv_list;
! 875: while ((pve = *prevptr) != NULL) {
! 876: if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */
! 877: *prevptr = pve->pv_next; /* remove it! */
! 878: break;
! 879: }
! 880: prevptr = &pve->pv_next; /* previous pointer */
! 881: }
! 882: return(pve); /* return removed pve */
! 883: }
! 884:
! 885: /*
! 886: * p t p f u n c t i o n s
! 887: */
! 888:
! 889: struct vm_page *
! 890: pmap_find_ptp(struct pmap *pmap, vaddr_t va, paddr_t pa, int level)
! 891: {
! 892: int lidx = level - 1;
! 893: struct vm_page *pg;
! 894:
! 895: if (pa != (paddr_t)-1 && pmap->pm_ptphint[lidx] &&
! 896: pa == VM_PAGE_TO_PHYS(pmap->pm_ptphint[lidx])) {
! 897: return (pmap->pm_ptphint[lidx]);
! 898: }
! 899: if (lidx == 0)
! 900: pg = uvm_pagelookup(&pmap->pm_obj[lidx], ptp_va2o(va, level));
! 901: else {
! 902: simple_lock(&pmap->pm_obj[lidx].vmobjlock);
! 903: pg = uvm_pagelookup(&pmap->pm_obj[lidx], ptp_va2o(va, level));
! 904: simple_unlock(&pmap->pm_obj[lidx].vmobjlock);
! 905: }
! 906: return pg;
! 907: }
! 908:
! 909: void
! 910: pmap_freepage(struct pmap *pmap, struct vm_page *ptp, int level,
! 911: struct pg_to_free *pagelist)
! 912: {
! 913: int lidx;
! 914: struct uvm_object *obj;
! 915:
! 916: lidx = level - 1;
! 917:
! 918: obj = &pmap->pm_obj[lidx];
! 919: pmap->pm_stats.resident_count--;
! 920: if (pmap->pm_ptphint[lidx] == ptp)
! 921: pmap->pm_ptphint[lidx] = TAILQ_FIRST(&obj->memq);
! 922: ptp->wire_count = 0;
! 923: uvm_pagerealloc(ptp, NULL, 0);
! 924: TAILQ_INSERT_TAIL(pagelist, ptp, listq);
! 925: }
! 926:
! 927: void
! 928: pmap_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va,
! 929: pt_entry_t *ptes, pd_entry_t **pdes, struct pg_to_free *pagelist)
! 930: {
! 931: unsigned long index;
! 932: int level;
! 933: vaddr_t invaladdr;
! 934: pd_entry_t opde;
! 935:
! 936: level = 1;
! 937: do {
! 938: pmap_freepage(pmap, ptp, level, pagelist);
! 939: index = pl_i(va, level + 1);
! 940: opde = pmap_pte_set(&pdes[level - 1][index], 0);
! 941: invaladdr = level == 1 ? (vaddr_t)ptes :
! 942: (vaddr_t)pdes[level - 2];
! 943: pmap_tlb_shootpage(curpcb->pcb_pmap,
! 944: invaladdr + index * PAGE_SIZE);
! 945: #if defined(MULTIPROCESSOR)
! 946: invaladdr = level == 1 ? (vaddr_t)PTE_BASE :
! 947: (vaddr_t)normal_pdes[level - 2];
! 948: pmap_tlb_shootpage(pmap, invaladdr + index * PAGE_SIZE);
! 949: #endif
! 950: if (level < PTP_LEVELS - 1) {
! 951: ptp = pmap_find_ptp(pmap, va, (paddr_t)-1, level + 1);
! 952: ptp->wire_count--;
! 953: if (ptp->wire_count > 1)
! 954: break;
! 955: }
! 956: } while (++level < PTP_LEVELS);
! 957: }
! 958:
! 959: /*
! 960: * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one)
! 961: *
! 962: * => pmap should NOT be pmap_kernel()
! 963: * => pmap should be locked
! 964: */
! 965:
! 966:
! 967: struct vm_page *
! 968: pmap_get_ptp(struct pmap *pmap, vaddr_t va, pd_entry_t **pdes)
! 969: {
! 970: struct vm_page *ptp, *pptp;
! 971: int i;
! 972: unsigned long index;
! 973: pd_entry_t *pva;
! 974: paddr_t ppa, pa;
! 975: struct uvm_object *obj;
! 976:
! 977: ptp = NULL;
! 978: pa = (paddr_t)-1;
! 979:
! 980: /*
! 981: * Loop through all page table levels seeing if we need to
! 982: * add a new page to that level.
! 983: */
! 984: for (i = PTP_LEVELS; i > 1; i--) {
! 985: /*
! 986: * Save values from previous round.
! 987: */
! 988: pptp = ptp;
! 989: ppa = pa;
! 990:
! 991: index = pl_i(va, i);
! 992: pva = pdes[i - 2];
! 993:
! 994: if (pmap_valid_entry(pva[index])) {
! 995: ppa = pva[index] & PG_FRAME;
! 996: ptp = NULL;
! 997: continue;
! 998: }
! 999:
! 1000: obj = &pmap->pm_obj[i-2];
! 1001: /*
! 1002: * XXX pm_obj[0] is pm_lock, which is already locked.
! 1003: */
! 1004: if (i != 2)
! 1005: simple_lock(&obj->vmobjlock);
! 1006: ptp = uvm_pagealloc(obj, ptp_va2o(va, i - 1), NULL,
! 1007: UVM_PGA_USERESERVE|UVM_PGA_ZERO);
! 1008: if (i != 2)
! 1009: simple_unlock(&obj->vmobjlock);
! 1010:
! 1011: if (ptp == NULL)
! 1012: return NULL;
! 1013:
! 1014: atomic_clearbits_int(&ptp->pg_flags, PG_BUSY);
! 1015: ptp->wire_count = 1;
! 1016: pmap->pm_ptphint[i - 2] = ptp;
! 1017: pa = VM_PAGE_TO_PHYS(ptp);
! 1018: pva[index] = (pd_entry_t) (pa | PG_u | PG_RW | PG_V);
! 1019: pmap->pm_stats.resident_count++;
! 1020: /*
! 1021: * If we're not in the top level, increase the
! 1022: * wire count of the parent page.
! 1023: */
! 1024: if (i < PTP_LEVELS) {
! 1025: if (pptp == NULL)
! 1026: pptp = pmap_find_ptp(pmap, va, ppa, i);
! 1027: #ifdef DIAGNOSTIC
! 1028: if (pptp == NULL)
! 1029: panic("pde page disappeared");
! 1030: #endif
! 1031: pptp->wire_count++;
! 1032: }
! 1033: }
! 1034:
! 1035: /*
! 1036: * ptp is not NULL if we just allocated a new ptp. If it's
! 1037: * still NULL, we must look up the existing one.
! 1038: */
! 1039: if (ptp == NULL) {
! 1040: ptp = pmap_find_ptp(pmap, va, ppa, 1);
! 1041: #ifdef DIAGNOSTIC
! 1042: if (ptp == NULL) {
! 1043: printf("va %lx ppa %lx\n", (unsigned long)va,
! 1044: (unsigned long)ppa);
! 1045: panic("pmap_get_ptp: unmanaged user PTP");
! 1046: }
! 1047: #endif
! 1048: }
! 1049:
! 1050: pmap->pm_ptphint[0] = ptp;
! 1051: return(ptp);
! 1052: }
! 1053:
! 1054: /*
! 1055: * p m a p l i f e c y c l e f u n c t i o n s
! 1056: */
! 1057:
! 1058: /*
! 1059: * pmap_pdp_ctor: constructor for the PDP cache.
! 1060: */
! 1061:
! 1062: int
! 1063: pmap_pdp_ctor(void *arg, void *object, int flags)
! 1064: {
! 1065: pd_entry_t *pdir = object;
! 1066: paddr_t pdirpa;
! 1067: int npde;
! 1068:
! 1069: /*
! 1070: * NOTE: The `pmap_lock' is held when the PDP is allocated.
! 1071: * WE MUST NOT BLOCK!
! 1072: */
! 1073:
! 1074: /* fetch the physical address of the page directory. */
! 1075: (void) pmap_extract(pmap_kernel(), (vaddr_t) pdir, &pdirpa);
! 1076:
! 1077: /* zero init area */
! 1078: memset(pdir, 0, PDIR_SLOT_PTE * sizeof(pd_entry_t));
! 1079:
! 1080: /* put in recursive PDE to map the PTEs */
! 1081: pdir[PDIR_SLOT_PTE] = pdirpa | PG_V | PG_KW;
! 1082:
! 1083: npde = nkptp[PTP_LEVELS - 1];
! 1084:
! 1085: /* put in kernel VM PDEs */
! 1086: memcpy(&pdir[PDIR_SLOT_KERN], &PDP_BASE[PDIR_SLOT_KERN],
! 1087: npde * sizeof(pd_entry_t));
! 1088:
! 1089: /* zero the rest */
! 1090: memset(&pdir[PDIR_SLOT_KERN + npde], 0,
! 1091: (NTOPLEVEL_PDES - (PDIR_SLOT_KERN + npde)) * sizeof(pd_entry_t));
! 1092:
! 1093: pdir[PDIR_SLOT_DIRECT] = pmap_kernel()->pm_pdir[PDIR_SLOT_DIRECT];
! 1094: pdir[PDIR_SLOT_DIRECT_NC] = pmap_kernel()->pm_pdir[PDIR_SLOT_DIRECT_NC];
! 1095:
! 1096: #if VM_MIN_KERNEL_ADDRESS != KERNBASE
! 1097: pdir[pl4_pi(KERNBASE)] = PDP_BASE[pl4_pi(KERNBASE)];
! 1098: #endif
! 1099:
! 1100: return (0);
! 1101: }
! 1102:
! 1103: /*
! 1104: * pmap_create: create a pmap
! 1105: *
! 1106: * => note: old pmap interface took a "size" args which allowed for
! 1107: * the creation of "software only" pmaps (not in bsd).
! 1108: */
! 1109:
! 1110: struct pmap *
! 1111: pmap_create(void)
! 1112: {
! 1113: struct pmap *pmap;
! 1114: int i;
! 1115: u_int gen;
! 1116:
! 1117: pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
! 1118:
! 1119: /* init uvm_object */
! 1120: for (i = 0; i < PTP_LEVELS - 1; i++) {
! 1121: simple_lock_init(&pmap->pm_obj[i].vmobjlock);
! 1122: pmap->pm_obj[i].pgops = NULL; /* not a mappable object */
! 1123: TAILQ_INIT(&pmap->pm_obj[i].memq);
! 1124: pmap->pm_obj[i].uo_npages = 0;
! 1125: pmap->pm_obj[i].uo_refs = 1;
! 1126: pmap->pm_ptphint[i] = NULL;
! 1127: }
! 1128: pmap->pm_stats.wired_count = 0;
! 1129: pmap->pm_stats.resident_count = 1; /* count the PDP allocd below */
! 1130: pmap->pm_flags = 0;
! 1131:
! 1132: /* init the LDT */
! 1133: pmap->pm_ldt = NULL;
! 1134: pmap->pm_ldt_len = 0;
! 1135: pmap->pm_ldt_sel = GSYSSEL(GLDT_SEL, SEL_KPL);
! 1136:
! 1137: /* allocate PDP */
! 1138:
! 1139: /*
! 1140: * we need to lock pmaps_lock to prevent nkpde from changing on
! 1141: * us. note that there is no need to splvm to protect us from
! 1142: * malloc since malloc allocates out of a submap and we should
! 1143: * have already allocated kernel PTPs to cover the range...
! 1144: *
! 1145: * NOTE: WE MUST NOT BLOCK WHILE HOLDING THE `pmap_lock', nor
! 1146: * ust we call pmap_growkernel() while holding it!
! 1147: */
! 1148:
! 1149: try_again:
! 1150: gen = pmap_pdp_cache_generation;
! 1151: pmap->pm_pdir = pool_cache_get(&pmap_pdp_cache, PR_WAITOK);
! 1152:
! 1153: simple_lock(&pmaps_lock);
! 1154:
! 1155: if (gen != pmap_pdp_cache_generation) {
! 1156: simple_unlock(&pmaps_lock);
! 1157: pool_cache_destruct_object(&pmap_pdp_cache, pmap->pm_pdir);
! 1158: goto try_again;
! 1159: }
! 1160:
! 1161: pmap->pm_pdirpa = pmap->pm_pdir[PDIR_SLOT_PTE] & PG_FRAME;
! 1162:
! 1163: LIST_INSERT_HEAD(&pmaps, pmap, pm_list);
! 1164:
! 1165: simple_unlock(&pmaps_lock);
! 1166:
! 1167: return (pmap);
! 1168: }
! 1169:
! 1170: /*
! 1171: * pmap_destroy: drop reference count on pmap. free pmap if
! 1172: * reference count goes to zero.
! 1173: */
! 1174:
! 1175: void
! 1176: pmap_destroy(struct pmap *pmap)
! 1177: {
! 1178: struct vm_page *pg;
! 1179: int refs;
! 1180: int i;
! 1181:
! 1182: /*
! 1183: * drop reference count
! 1184: */
! 1185:
! 1186: simple_lock(&pmap->pm_lock);
! 1187: refs = --pmap->pm_obj[0].uo_refs;
! 1188: simple_unlock(&pmap->pm_lock);
! 1189: if (refs > 0) {
! 1190: return;
! 1191: }
! 1192:
! 1193: /*
! 1194: * reference count is zero, free pmap resources and then free pmap.
! 1195: */
! 1196:
! 1197: /*
! 1198: * remove it from global list of pmaps
! 1199: */
! 1200:
! 1201: simple_lock(&pmaps_lock);
! 1202: LIST_REMOVE(pmap, pm_list);
! 1203: simple_unlock(&pmaps_lock);
! 1204:
! 1205: /*
! 1206: * free any remaining PTPs
! 1207: */
! 1208:
! 1209: for (i = 0; i < PTP_LEVELS - 1; i++) {
! 1210: while ((pg = TAILQ_FIRST(&pmap->pm_obj[i].memq)) != NULL) {
! 1211: KASSERT((pg->pg_flags & PG_BUSY) == 0);
! 1212:
! 1213: pg->wire_count = 0;
! 1214: uvm_pagefree(pg);
! 1215: }
! 1216: }
! 1217:
! 1218: /*
! 1219: * MULTIPROCESSOR -- no need to flush out of other processors'
! 1220: * APTE space because we do that in pmap_unmap_ptes().
! 1221: */
! 1222: /* XXX: need to flush it out of other processor's APTE space? */
! 1223: pool_cache_put(&pmap_pdp_cache, pmap->pm_pdir);
! 1224:
! 1225: #ifdef USER_LDT
! 1226: if (pmap->pm_flags & PMF_USER_LDT) {
! 1227: /*
! 1228: * no need to switch the LDT; this address space is gone,
! 1229: * nothing is using it.
! 1230: *
! 1231: * No need to lock the pmap for ldt_free (or anything else),
! 1232: * we're the last one to use it.
! 1233: */
! 1234: ldt_free(pmap);
! 1235: uvm_km_free(kernel_map, (vaddr_t)pmap->pm_ldt,
! 1236: pmap->pm_ldt_len);
! 1237: }
! 1238: #endif
! 1239:
! 1240: pool_put(&pmap_pmap_pool, pmap);
! 1241: }
! 1242:
! 1243: /*
! 1244: * Add a reference to the specified pmap.
! 1245: */
! 1246:
! 1247: void
! 1248: pmap_reference(struct pmap *pmap)
! 1249: {
! 1250: simple_lock(&pmap->pm_lock);
! 1251: pmap->pm_obj[0].uo_refs++;
! 1252: simple_unlock(&pmap->pm_lock);
! 1253: }
! 1254:
! 1255: #if defined(PMAP_FORK)
! 1256: /*
! 1257: * pmap_fork: perform any necessary data structure manipulation when
! 1258: * a VM space is forked.
! 1259: */
! 1260:
! 1261: void
! 1262: pmap_fork(struct pmap *pmap1, struct pmap *pmap2)
! 1263: {
! 1264: simple_lock(&pmap1->pm_lock);
! 1265: simple_lock(&pmap2->pm_lock);
! 1266:
! 1267: #ifdef USER_LDT
! 1268: /* Copy the LDT, if necessary. */
! 1269: if (pmap1->pm_flags & PMF_USER_LDT) {
! 1270: char *new_ldt;
! 1271: size_t len;
! 1272:
! 1273: len = pmap1->pm_ldt_len;
! 1274: new_ldt = (char *)uvm_km_alloc(kernel_map, len);
! 1275: memcpy(new_ldt, pmap1->pm_ldt, len);
! 1276: pmap2->pm_ldt = new_ldt;
! 1277: pmap2->pm_ldt_len = pmap1->pm_ldt_len;
! 1278: pmap2->pm_flags |= PMF_USER_LDT;
! 1279: ldt_alloc(pmap2, new_ldt, len);
! 1280: }
! 1281: #endif /* USER_LDT */
! 1282:
! 1283: simple_unlock(&pmap2->pm_lock);
! 1284: simple_unlock(&pmap1->pm_lock);
! 1285: }
! 1286: #endif /* PMAP_FORK */
! 1287:
! 1288: #ifdef USER_LDT
! 1289: /*
! 1290: * pmap_ldt_cleanup: if the pmap has a local LDT, deallocate it, and
! 1291: * restore the default.
! 1292: */
! 1293:
! 1294: void
! 1295: pmap_ldt_cleanup(struct proc *p)
! 1296: {
! 1297: struct pcb *pcb = &p->p_addr->u_pcb;
! 1298: pmap_t pmap = p->->p_vmspace->vm_map.pmap;
! 1299: char *old_ldt = NULL;
! 1300: size_t len = 0;
! 1301:
! 1302: simple_lock(&pmap->pm_lock);
! 1303:
! 1304: if (pmap->pm_flags & PMF_USER_LDT) {
! 1305: ldt_free(pmap);
! 1306: pmap->pm_ldt_sel = GSYSSEL(GLDT_SEL, SEL_KPL);
! 1307: pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
! 1308: if (pcb == curpcb)
! 1309: lldt(pcb->pcb_ldt_sel);
! 1310: old_ldt = pmap->pm_ldt;
! 1311: len = pmap->pm_ldt_len;
! 1312: pmap->pm_ldt = NULL;
! 1313: pmap->pm_ldt_len = 0;
! 1314: pmap->pm_flags &= ~PMF_USER_LDT;
! 1315: }
! 1316:
! 1317: simple_unlock(&pmap->pm_lock);
! 1318:
! 1319: if (old_ldt != NULL)
! 1320: uvm_km_free(kernel_map, (vaddr_t)old_ldt, len);
! 1321: }
! 1322: #endif /* USER_LDT */
! 1323:
! 1324: /*
! 1325: * pmap_activate: activate a process' pmap (fill in %cr3 and LDT info)
! 1326: *
! 1327: * => called from cpu_switch()
! 1328: * => if p is the curproc, then load it into the MMU
! 1329: */
! 1330:
! 1331: void
! 1332: pmap_activate(struct proc *p)
! 1333: {
! 1334: struct pcb *pcb = &p->p_addr->u_pcb;
! 1335: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
! 1336:
! 1337: pcb->pcb_pmap = pmap;
! 1338: pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
! 1339: pcb->pcb_cr3 = pmap->pm_pdirpa;
! 1340: if (p == curproc)
! 1341: lcr3(pcb->pcb_cr3);
! 1342: if (pcb == curpcb)
! 1343: lldt(pcb->pcb_ldt_sel);
! 1344:
! 1345: /*
! 1346: * mark the pmap in use by this processor.
! 1347: */
! 1348: x86_atomic_setbits_ul(&pmap->pm_cpus, (1U << cpu_number()));
! 1349: }
! 1350:
! 1351: /*
! 1352: * pmap_deactivate: deactivate a process' pmap
! 1353: */
! 1354:
! 1355: void
! 1356: pmap_deactivate(struct proc *p)
! 1357: {
! 1358: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
! 1359:
! 1360: /*
! 1361: * mark the pmap no longer in use by this processor.
! 1362: */
! 1363: x86_atomic_clearbits_ul(&pmap->pm_cpus, (1U << cpu_number()));
! 1364:
! 1365: }
! 1366:
! 1367: /*
! 1368: * end of lifecycle functions
! 1369: */
! 1370:
! 1371: /*
! 1372: * some misc. functions
! 1373: */
! 1374:
! 1375: boolean_t
! 1376: pmap_pdes_valid(vaddr_t va, pd_entry_t **pdes, pd_entry_t *lastpde)
! 1377: {
! 1378: int i;
! 1379: unsigned long index;
! 1380: pd_entry_t pde;
! 1381:
! 1382: for (i = PTP_LEVELS; i > 1; i--) {
! 1383: index = pl_i(va, i);
! 1384: pde = pdes[i - 2][index];
! 1385: if ((pde & PG_V) == 0)
! 1386: return FALSE;
! 1387: }
! 1388: if (lastpde != NULL)
! 1389: *lastpde = pde;
! 1390: return TRUE;
! 1391: }
! 1392:
! 1393: /*
! 1394: * pmap_extract: extract a PA for the given VA
! 1395: */
! 1396:
! 1397: boolean_t
! 1398: pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap)
! 1399: {
! 1400: pt_entry_t *ptes, pte;
! 1401: pd_entry_t pde, **pdes;
! 1402:
! 1403: if (pmap == pmap_kernel() && va >= PMAP_DIRECT_BASE &&
! 1404: va < PMAP_DIRECT_END) {
! 1405: *pap = va - PMAP_DIRECT_BASE;
! 1406: return (TRUE);
! 1407: }
! 1408:
! 1409: if (pmap == pmap_kernel() && va >= PMAP_DIRECT_BASE_NC &&
! 1410: va < PMAP_DIRECT_END_NC) {
! 1411: *pap = va - PMAP_DIRECT_BASE_NC;
! 1412: return (TRUE);
! 1413: }
! 1414:
! 1415: pmap_map_ptes(pmap, &ptes, &pdes);
! 1416: if (pmap_pdes_valid(va, pdes, &pde) == FALSE) {
! 1417: return FALSE;
! 1418: }
! 1419:
! 1420: if (pde & PG_PS) {
! 1421: if (pap != NULL)
! 1422: *pap = (pde & PG_LGFRAME) | (va & 0x1fffff);
! 1423: pmap_unmap_ptes(pmap);
! 1424: return (TRUE);
! 1425: }
! 1426:
! 1427: pte = ptes[pl1_i(va)];
! 1428: pmap_unmap_ptes(pmap);
! 1429:
! 1430: if (__predict_true((pte & PG_V) != 0)) {
! 1431: if (pap != NULL)
! 1432: *pap = (pte & PG_FRAME) | (va & 0xfff);
! 1433: return (TRUE);
! 1434: }
! 1435:
! 1436: return FALSE;
! 1437: }
! 1438:
! 1439: /*
! 1440: * pmap_map: map a range of PAs into kvm
! 1441: *
! 1442: * => used during crash dump
! 1443: * => XXX: pmap_map() should be phased out?
! 1444: */
! 1445:
! 1446: vaddr_t
! 1447: pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot)
! 1448: {
! 1449: while (spa < epa) {
! 1450: pmap_enter(pmap_kernel(), va, spa, prot, 0);
! 1451: va += PAGE_SIZE;
! 1452: spa += PAGE_SIZE;
! 1453: }
! 1454: pmap_update(pmap_kernel());
! 1455: return va;
! 1456: }
! 1457:
! 1458:
! 1459: /*
! 1460: * pmap_zero_page: zero a page
! 1461: */
! 1462:
! 1463: void
! 1464: pmap_zero_page(struct vm_page *pg)
! 1465: {
! 1466: pagezero(pmap_map_direct(pg));
! 1467: }
! 1468:
! 1469: /*
! 1470: * pmap_pagezeroidle: the same, for the idle loop page zero'er.
! 1471: * Returns TRUE if the page was zero'd, FALSE if we aborted for
! 1472: * some reason.
! 1473: */
! 1474:
! 1475: boolean_t
! 1476: pmap_pageidlezero(struct vm_page *pg)
! 1477: {
! 1478: vaddr_t va = pmap_map_direct(pg);
! 1479: boolean_t rv = TRUE;
! 1480: long *ptr;
! 1481: int i;
! 1482:
! 1483: /*
! 1484: * XXX - We'd really like to do this uncached. But at this moment
! 1485: * we're never called, so just pretend that this works.
! 1486: * It shouldn't be too hard to create a second direct map
! 1487: * with uncached mappings.
! 1488: */
! 1489: for (i = 0, ptr = (long *) va; i < PAGE_SIZE / sizeof(long); i++) {
! 1490: if (!sched_is_idle()) {
! 1491:
! 1492: /*
! 1493: * A process has become ready. Abort now,
! 1494: * so we don't keep it waiting while we
! 1495: * do slow memory access to finish this
! 1496: * page.
! 1497: */
! 1498:
! 1499: rv = FALSE;
! 1500: break;
! 1501: }
! 1502: *ptr++ = 0;
! 1503: }
! 1504:
! 1505: return (rv);
! 1506: }
! 1507:
! 1508: /*
! 1509: * pmap_copy_page: copy a page
! 1510: */
! 1511:
! 1512: void
! 1513: pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
! 1514: {
! 1515: vaddr_t srcva = pmap_map_direct(srcpg);
! 1516: vaddr_t dstva = pmap_map_direct(dstpg);
! 1517:
! 1518: memcpy((void *)dstva, (void *)srcva, PAGE_SIZE);
! 1519: }
! 1520:
! 1521: /*
! 1522: * p m a p r e m o v e f u n c t i o n s
! 1523: *
! 1524: * functions that remove mappings
! 1525: */
! 1526:
! 1527: /*
! 1528: * pmap_remove_ptes: remove PTEs from a PTP
! 1529: *
! 1530: * => must have proper locking on pmap_master_lock
! 1531: * => caller must hold pmap's lock
! 1532: * => PTP must be mapped into KVA
! 1533: * => PTP should be null if pmap == pmap_kernel()
! 1534: */
! 1535:
! 1536: void
! 1537: pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
! 1538: vaddr_t startva, vaddr_t endva, int flags)
! 1539: {
! 1540: struct pv_entry *pve;
! 1541: pt_entry_t *pte = (pt_entry_t *) ptpva;
! 1542: struct vm_page *pg;
! 1543: pt_entry_t opte;
! 1544:
! 1545: /*
! 1546: * note that ptpva points to the PTE that maps startva. this may
! 1547: * or may not be the first PTE in the PTP.
! 1548: *
! 1549: * we loop through the PTP while there are still PTEs to look at
! 1550: * and the wire_count is greater than 1 (because we use the wire_count
! 1551: * to keep track of the number of real PTEs in the PTP).
! 1552: */
! 1553:
! 1554: for (/*null*/; startva < endva && (ptp == NULL || ptp->wire_count > 1)
! 1555: ; pte++, startva += PAGE_SIZE) {
! 1556: if (!pmap_valid_entry(*pte))
! 1557: continue; /* VA not mapped */
! 1558: if ((flags & PMAP_REMOVE_SKIPWIRED) && (*pte & PG_W)) {
! 1559: continue;
! 1560: }
! 1561:
! 1562: /* atomically save the old PTE and zap! it */
! 1563: opte = pmap_pte_set(pte, 0);
! 1564:
! 1565: if (opte & PG_W)
! 1566: pmap->pm_stats.wired_count--;
! 1567: pmap->pm_stats.resident_count--;
! 1568:
! 1569: if (ptp)
! 1570: ptp->wire_count--; /* dropping a PTE */
! 1571:
! 1572: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
! 1573:
! 1574: /*
! 1575: * if we are not on a pv list we are done.
! 1576: */
! 1577:
! 1578: if ((opte & PG_PVLIST) == 0) {
! 1579: #ifdef DIAGNOSTIC
! 1580: if (pg != NULL)
! 1581: panic("pmap_remove_ptes: managed page without "
! 1582: "PG_PVLIST for 0x%lx", startva);
! 1583: #endif
! 1584: continue;
! 1585: }
! 1586:
! 1587: #ifdef DIAGNOSTIC
! 1588: if (pg == NULL)
! 1589: panic("pmap_remove_ptes: unmanaged page marked "
! 1590: "PG_PVLIST, va = 0x%lx, pa = 0x%lx",
! 1591: startva, (u_long)(opte & PG_FRAME));
! 1592: #endif
! 1593:
! 1594: /* sync R/M bits */
! 1595: pmap_sync_flags_pte(pg, opte);
! 1596: pve = pmap_remove_pv(pg, pmap, startva);
! 1597:
! 1598: if (pve) {
! 1599: pool_put(&pmap_pv_pool, pve);
! 1600: }
! 1601:
! 1602: /* end of "for" loop: time for next pte */
! 1603: }
! 1604: }
! 1605:
! 1606:
! 1607: /*
! 1608: * pmap_remove_pte: remove a single PTE from a PTP
! 1609: *
! 1610: * => must have proper locking on pmap_master_lock
! 1611: * => caller must hold pmap's lock
! 1612: * => PTP must be mapped into KVA
! 1613: * => PTP should be null if pmap == pmap_kernel()
! 1614: * => returns true if we removed a mapping
! 1615: */
! 1616:
! 1617: boolean_t
! 1618: pmap_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte,
! 1619: vaddr_t va, int flags)
! 1620: {
! 1621: struct pv_entry *pve;
! 1622: struct vm_page *pg;
! 1623: pt_entry_t opte;
! 1624:
! 1625: if (!pmap_valid_entry(*pte))
! 1626: return(FALSE); /* VA not mapped */
! 1627: if ((flags & PMAP_REMOVE_SKIPWIRED) && (*pte & PG_W)) {
! 1628: return(FALSE);
! 1629: }
! 1630:
! 1631: /* atomically save the old PTE and zap! it */
! 1632: opte = pmap_pte_set(pte, 0);
! 1633:
! 1634: if (opte & PG_W)
! 1635: pmap->pm_stats.wired_count--;
! 1636: pmap->pm_stats.resident_count--;
! 1637:
! 1638: if (ptp)
! 1639: ptp->wire_count--; /* dropping a PTE */
! 1640:
! 1641: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
! 1642:
! 1643: /*
! 1644: * if we are not on a pv list we are done.
! 1645: */
! 1646: if ((opte & PG_PVLIST) == 0) {
! 1647: #ifdef DIAGNOSTIC
! 1648: if (pg != NULL)
! 1649: panic("pmap_remove_pte: managed page without "
! 1650: "PG_PVLIST for 0x%lx", va);
! 1651: #endif
! 1652: return(TRUE);
! 1653: }
! 1654:
! 1655: #ifdef DIAGNOSTIC
! 1656: if (pg == NULL)
! 1657: panic("pmap_remove_pte: unmanaged page marked "
! 1658: "PG_PVLIST, va = 0x%lx, pa = 0x%lx", va,
! 1659: (u_long)(opte & PG_FRAME));
! 1660: #endif
! 1661:
! 1662: /* sync R/M bits */
! 1663: pmap_sync_flags_pte(pg, opte);
! 1664: pve = pmap_remove_pv(pg, pmap, va);
! 1665: if (pve)
! 1666: pool_put(&pmap_pv_pool, pve);
! 1667: return(TRUE);
! 1668: }
! 1669:
! 1670: /*
! 1671: * pmap_remove: top level mapping removal function
! 1672: *
! 1673: * => caller should not be holding any pmap locks
! 1674: */
! 1675:
! 1676: void
! 1677: pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
! 1678: {
! 1679: pmap_do_remove(pmap, sva, eva, PMAP_REMOVE_ALL);
! 1680: }
! 1681:
! 1682: /*
! 1683: * pmap_do_remove: mapping removal guts
! 1684: *
! 1685: * => caller should not be holding any pmap locks
! 1686: */
! 1687:
! 1688: void
! 1689: pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags)
! 1690: {
! 1691: pt_entry_t *ptes;
! 1692: pd_entry_t **pdes, pde;
! 1693: boolean_t result;
! 1694: paddr_t ptppa;
! 1695: vaddr_t blkendva;
! 1696: struct vm_page *ptp;
! 1697: vaddr_t va;
! 1698: int shootall = 0;
! 1699: struct pg_to_free empty_ptps;
! 1700:
! 1701: TAILQ_INIT(&empty_ptps);
! 1702:
! 1703: PMAP_MAP_TO_HEAD_LOCK();
! 1704: pmap_map_ptes(pmap, &ptes, &pdes); /* locks pmap */
! 1705:
! 1706: /*
! 1707: * removing one page? take shortcut function.
! 1708: */
! 1709:
! 1710: if (sva + PAGE_SIZE == eva) {
! 1711: if (pmap_pdes_valid(sva, pdes, &pde)) {
! 1712:
! 1713: /* PA of the PTP */
! 1714: ptppa = pde & PG_FRAME;
! 1715:
! 1716: /* get PTP if non-kernel mapping */
! 1717:
! 1718: if (pmap == pmap_kernel()) {
! 1719: /* we never free kernel PTPs */
! 1720: ptp = NULL;
! 1721: } else {
! 1722: ptp = pmap_find_ptp(pmap, sva, ptppa, 1);
! 1723: #ifdef DIAGNOSTIC
! 1724: if (ptp == NULL)
! 1725: panic("pmap_remove: unmanaged "
! 1726: "PTP detected");
! 1727: #endif
! 1728: }
! 1729:
! 1730: /* do it! */
! 1731: result = pmap_remove_pte(pmap, ptp,
! 1732: &ptes[pl1_i(sva)], sva, flags);
! 1733:
! 1734: /*
! 1735: * if mapping removed and the PTP is no longer
! 1736: * being used, free it!
! 1737: */
! 1738:
! 1739: if (result && ptp && ptp->wire_count <= 1)
! 1740: pmap_free_ptp(pmap, ptp, sva, ptes, pdes,
! 1741: &empty_ptps);
! 1742: pmap_tlb_shootpage(pmap, sva);
! 1743: }
! 1744:
! 1745: pmap_tlb_shootwait();
! 1746: pmap_unmap_ptes(pmap); /* unlock pmap */
! 1747: PMAP_MAP_TO_HEAD_UNLOCK();
! 1748:
! 1749: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
! 1750: TAILQ_REMOVE(&empty_ptps, ptp, listq);
! 1751: uvm_pagefree(ptp);
! 1752: }
! 1753:
! 1754: return;
! 1755: }
! 1756:
! 1757: if ((eva - sva > 32 * PAGE_SIZE) && pmap != pmap_kernel())
! 1758: shootall = 1;
! 1759:
! 1760: for (va = sva; va < eva; va = blkendva) {
! 1761: /* determine range of block */
! 1762: blkendva = x86_round_pdr(va + 1);
! 1763: if (blkendva > eva)
! 1764: blkendva = eva;
! 1765:
! 1766: /*
! 1767: * XXXCDC: our PTE mappings should never be removed
! 1768: * with pmap_remove! if we allow this (and why would
! 1769: * we?) then we end up freeing the pmap's page
! 1770: * directory page (PDP) before we are finished using
! 1771: * it when we hit in in the recursive mapping. this
! 1772: * is BAD.
! 1773: *
! 1774: * long term solution is to move the PTEs out of user
! 1775: * address space. and into kernel address space (up
! 1776: * with APTE). then we can set VM_MAXUSER_ADDRESS to
! 1777: * be VM_MAX_ADDRESS.
! 1778: */
! 1779:
! 1780: if (pl_i(va, PTP_LEVELS) == PDIR_SLOT_PTE)
! 1781: /* XXXCDC: ugly hack to avoid freeing PDP here */
! 1782: continue;
! 1783:
! 1784: if (!pmap_pdes_valid(va, pdes, &pde))
! 1785: continue;
! 1786:
! 1787: /* PA of the PTP */
! 1788: ptppa = pde & PG_FRAME;
! 1789:
! 1790: /* get PTP if non-kernel mapping */
! 1791: if (pmap == pmap_kernel()) {
! 1792: /* we never free kernel PTPs */
! 1793: ptp = NULL;
! 1794: } else {
! 1795: ptp = pmap_find_ptp(pmap, va, ptppa, 1);
! 1796: #ifdef DIAGNOSTIC
! 1797: if (ptp == NULL)
! 1798: panic("pmap_remove: unmanaged PTP "
! 1799: "detected");
! 1800: #endif
! 1801: }
! 1802: pmap_remove_ptes(pmap, ptp,
! 1803: (vaddr_t)&ptes[pl1_i(va)], va, blkendva, flags);
! 1804:
! 1805: /* if PTP is no longer being used, free it! */
! 1806: if (ptp && ptp->wire_count <= 1) {
! 1807: pmap_free_ptp(pmap, ptp, va, ptes, pdes, &empty_ptps);
! 1808: }
! 1809: }
! 1810:
! 1811: if (shootall)
! 1812: pmap_tlb_shoottlb();
! 1813: else
! 1814: pmap_tlb_shootrange(pmap, sva, eva);
! 1815:
! 1816: pmap_tlb_shootwait();
! 1817:
! 1818: pmap_unmap_ptes(pmap);
! 1819: PMAP_MAP_TO_HEAD_UNLOCK();
! 1820:
! 1821: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
! 1822: TAILQ_REMOVE(&empty_ptps, ptp, listq);
! 1823: uvm_pagefree(ptp);
! 1824: }
! 1825: }
! 1826:
! 1827: /*
! 1828: * pmap_page_remove: remove a managed vm_page from all pmaps that map it
! 1829: *
! 1830: * => R/M bits are sync'd back to attrs
! 1831: */
! 1832:
! 1833: void
! 1834: pmap_page_remove(struct vm_page *pg)
! 1835: {
! 1836: struct pv_entry *pve;
! 1837: pt_entry_t *ptes, opte;
! 1838: pd_entry_t **pdes;
! 1839: #ifdef DIAGNOSTIC
! 1840: pd_entry_t pde;
! 1841: #endif
! 1842: struct pg_to_free empty_ptps;
! 1843: struct vm_page *ptp;
! 1844:
! 1845: TAILQ_INIT(&empty_ptps);
! 1846:
! 1847: PMAP_HEAD_TO_MAP_LOCK();
! 1848:
! 1849: while ((pve = pg->mdpage.pv_list) != NULL) {
! 1850: pg->mdpage.pv_list = pve->pv_next;
! 1851:
! 1852: pmap_map_ptes(pve->pv_pmap, &ptes, &pdes); /* locks pmap */
! 1853:
! 1854: #ifdef DIAGNOSTIC
! 1855: if (pve->pv_ptp && pmap_pdes_valid(pve->pv_va, pdes, &pde) &&
! 1856: (pde & PG_FRAME) != VM_PAGE_TO_PHYS(pve->pv_ptp)) {
! 1857: printf("pmap_page_remove: pg=%p: va=%lx, pv_ptp=%p\n",
! 1858: pg, pve->pv_va, pve->pv_ptp);
! 1859: printf("pmap_page_remove: PTP's phys addr: "
! 1860: "actual=%lx, recorded=%lx\n",
! 1861: (unsigned long)(pde & PG_FRAME),
! 1862: VM_PAGE_TO_PHYS(pve->pv_ptp));
! 1863: panic("pmap_page_remove: mapped managed page has "
! 1864: "invalid pv_ptp field");
! 1865: }
! 1866: #endif
! 1867:
! 1868: /* atomically save the old PTE and zap it */
! 1869: opte = pmap_pte_set(&ptes[pl1_i(pve->pv_va)], 0);
! 1870:
! 1871: if (opte & PG_W)
! 1872: pve->pv_pmap->pm_stats.wired_count--;
! 1873: pve->pv_pmap->pm_stats.resident_count--;
! 1874:
! 1875: pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va);
! 1876:
! 1877: pmap_sync_flags_pte(pg, opte);
! 1878:
! 1879: /* update the PTP reference count. free if last reference. */
! 1880: if (pve->pv_ptp) {
! 1881: pve->pv_ptp->wire_count--;
! 1882: if (pve->pv_ptp->wire_count <= 1) {
! 1883: pmap_free_ptp(pve->pv_pmap, pve->pv_ptp,
! 1884: pve->pv_va, ptes, pdes, &empty_ptps);
! 1885: }
! 1886: }
! 1887: pmap_unmap_ptes(pve->pv_pmap); /* unlocks pmap */
! 1888: pool_put(&pmap_pv_pool, pve);
! 1889: }
! 1890:
! 1891: PMAP_HEAD_TO_MAP_UNLOCK();
! 1892: pmap_tlb_shootwait();
! 1893:
! 1894: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
! 1895: TAILQ_REMOVE(&empty_ptps, ptp, listq);
! 1896: uvm_pagefree(ptp);
! 1897: }
! 1898: }
! 1899:
! 1900: /*
! 1901: * p m a p a t t r i b u t e f u n c t i o n s
! 1902: * functions that test/change managed page's attributes
! 1903: * since a page can be mapped multiple times we must check each PTE that
! 1904: * maps it by going down the pv lists.
! 1905: */
! 1906:
! 1907: /*
! 1908: * pmap_test_attrs: test a page's attributes
! 1909: */
! 1910:
! 1911: boolean_t
! 1912: pmap_test_attrs(struct vm_page *pg, unsigned int testbits)
! 1913: {
! 1914: struct pv_entry *pve;
! 1915: pt_entry_t *ptes, pte;
! 1916: pd_entry_t **pdes;
! 1917: u_long mybits, testflags;
! 1918:
! 1919: testflags = pmap_pte2flags(testbits);
! 1920:
! 1921: if (pg->pg_flags & testflags)
! 1922: return (TRUE);
! 1923:
! 1924: PMAP_HEAD_TO_MAP_LOCK();
! 1925: mybits = 0;
! 1926: for (pve = pg->mdpage.pv_list; pve != NULL && mybits == 0;
! 1927: pve = pve->pv_next) {
! 1928: pmap_map_ptes(pve->pv_pmap, &ptes, &pdes);
! 1929: pte = ptes[pl1_i(pve->pv_va)];
! 1930: pmap_unmap_ptes(pve->pv_pmap);
! 1931: mybits |= (pte & testbits);
! 1932: }
! 1933: PMAP_HEAD_TO_MAP_UNLOCK();
! 1934:
! 1935: if (mybits == 0)
! 1936: return (FALSE);
! 1937:
! 1938: atomic_setbits_int(&pg->pg_flags, pmap_pte2flags(mybits));
! 1939:
! 1940: return (TRUE);
! 1941: }
! 1942:
! 1943: /*
! 1944: * pmap_clear_attrs: change a page's attributes
! 1945: *
! 1946: * => we return TRUE if we cleared one of the bits we were asked to
! 1947: */
! 1948:
! 1949: boolean_t
! 1950: pmap_clear_attrs(struct vm_page *pg, unsigned long clearbits)
! 1951: {
! 1952: struct pv_entry *pve;
! 1953: pt_entry_t *ptes, opte;
! 1954: pd_entry_t **pdes;
! 1955: u_long clearflags;
! 1956: int result;
! 1957:
! 1958: clearflags = pmap_pte2flags(clearbits);
! 1959:
! 1960: PMAP_HEAD_TO_MAP_LOCK();
! 1961:
! 1962: result = pg->pg_flags & clearflags;
! 1963: if (result)
! 1964: atomic_clearbits_int(&pg->pg_flags, clearflags);
! 1965:
! 1966: for (pve = pg->mdpage.pv_list; pve != NULL; pve = pve->pv_next) {
! 1967: pmap_map_ptes(pve->pv_pmap, &ptes, &pdes); /* locks pmap */
! 1968: #ifdef DIAGNOSTIC
! 1969: if (!pmap_pdes_valid(pve->pv_va, pdes, NULL))
! 1970: panic("pmap_change_attrs: mapping without PTP "
! 1971: "detected");
! 1972: #endif
! 1973:
! 1974: opte = ptes[pl1_i(pve->pv_va)];
! 1975: if (opte & clearbits) {
! 1976: result = 1;
! 1977: pmap_pte_clearbits(&ptes[pl1_i(pve->pv_va)],
! 1978: (opte & clearbits));
! 1979: pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va);
! 1980: }
! 1981: pmap_unmap_ptes(pve->pv_pmap); /* unlocks pmap */
! 1982: }
! 1983:
! 1984: PMAP_HEAD_TO_MAP_UNLOCK();
! 1985:
! 1986: pmap_tlb_shootwait();
! 1987:
! 1988: return (result != 0);
! 1989: }
! 1990:
! 1991: /*
! 1992: * p m a p p r o t e c t i o n f u n c t i o n s
! 1993: */
! 1994:
! 1995: /*
! 1996: * pmap_page_protect: change the protection of all recorded mappings
! 1997: * of a managed page
! 1998: *
! 1999: * => NOTE: this is an inline function in pmap.h
! 2000: */
! 2001:
! 2002: /* see pmap.h */
! 2003:
! 2004: /*
! 2005: * pmap_protect: set the protection in of the pages in a pmap
! 2006: *
! 2007: * => NOTE: this is an inline function in pmap.h
! 2008: */
! 2009:
! 2010: /* see pmap.h */
! 2011:
! 2012: /*
! 2013: * pmap_write_protect: write-protect pages in a pmap
! 2014: */
! 2015:
! 2016: void
! 2017: pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
! 2018: {
! 2019: pt_entry_t nx, opte, *ptes, *spte, *epte;
! 2020: pd_entry_t **pdes;
! 2021: vaddr_t blockend;
! 2022: int shootall = 0;
! 2023: vaddr_t va;
! 2024:
! 2025: pmap_map_ptes(pmap, &ptes, &pdes); /* locks pmap */
! 2026:
! 2027: /* should be ok, but just in case ... */
! 2028: sva &= PG_FRAME;
! 2029: eva &= PG_FRAME;
! 2030:
! 2031: nx = 0;
! 2032: if ((cpu_feature & CPUID_NXE) && !(prot & VM_PROT_EXECUTE))
! 2033: nx = PG_NX;
! 2034:
! 2035: if ((eva - sva > 32 * PAGE_SIZE) && pmap != pmap_kernel())
! 2036: shootall = 1;
! 2037:
! 2038: for (va = sva; va < eva ; va = blockend) {
! 2039: blockend = (va & L2_FRAME) + NBPD_L2;
! 2040: if (blockend > eva)
! 2041: blockend = eva;
! 2042:
! 2043: /*
! 2044: * XXXCDC: our PTE mappings should never be write-protected!
! 2045: *
! 2046: * long term solution is to move the PTEs out of user
! 2047: * address space. and into kernel address space (up
! 2048: * with APTE). then we can set VM_MAXUSER_ADDRESS to
! 2049: * be VM_MAX_ADDRESS.
! 2050: */
! 2051:
! 2052: /* XXXCDC: ugly hack to avoid freeing PDP here */
! 2053: if (pl_i(va, PTP_LEVELS) == PDIR_SLOT_PTE)
! 2054: continue;
! 2055:
! 2056: /* empty block? */
! 2057: if (!pmap_pdes_valid(va, pdes, NULL))
! 2058: continue;
! 2059:
! 2060: #ifdef DIAGNOSTIC
! 2061: if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS)
! 2062: panic("pmap_write_protect: PTE space");
! 2063: #endif
! 2064:
! 2065: spte = &ptes[pl1_i(va)];
! 2066: epte = &ptes[pl1_i(blockend)];
! 2067:
! 2068: for (/*null */; spte < epte ; spte++) {
! 2069: if (!(*spte & PG_V))
! 2070: continue;
! 2071: opte = *spte;
! 2072: pmap_pte_clearbits(spte, PG_RW);
! 2073: pmap_pte_setbits(spte, nx);
! 2074: }
! 2075: }
! 2076:
! 2077: if (shootall)
! 2078: pmap_tlb_shoottlb();
! 2079: else
! 2080: pmap_tlb_shootrange(pmap, sva, eva);
! 2081:
! 2082: pmap_tlb_shootwait();
! 2083:
! 2084: pmap_unmap_ptes(pmap); /* unlocks pmap */
! 2085: }
! 2086:
! 2087: /*
! 2088: * end of protection functions
! 2089: */
! 2090:
! 2091: /*
! 2092: * pmap_unwire: clear the wired bit in the PTE
! 2093: *
! 2094: * => mapping should already be in map
! 2095: */
! 2096:
! 2097: void
! 2098: pmap_unwire(struct pmap *pmap, vaddr_t va)
! 2099: {
! 2100: pt_entry_t *ptes;
! 2101: pd_entry_t **pdes;
! 2102:
! 2103: pmap_map_ptes(pmap, &ptes, &pdes); /* locks pmap */
! 2104:
! 2105: if (pmap_pdes_valid(va, pdes, NULL)) {
! 2106:
! 2107: #ifdef DIAGNOSTIC
! 2108: if (!pmap_valid_entry(ptes[pl1_i(va)]))
! 2109: panic("pmap_unwire: invalid (unmapped) va 0x%lx", va);
! 2110: #endif
! 2111: if ((ptes[pl1_i(va)] & PG_W) != 0) {
! 2112: ptes[pl1_i(va)] &= ~PG_W;
! 2113: pmap->pm_stats.wired_count--;
! 2114: }
! 2115: #ifdef DIAGNOSTIC
! 2116: else {
! 2117: printf("pmap_unwire: wiring for pmap %p va 0x%lx "
! 2118: "didn't change!\n", pmap, va);
! 2119: }
! 2120: #endif
! 2121: pmap_unmap_ptes(pmap); /* unlocks map */
! 2122: }
! 2123: #ifdef DIAGNOSTIC
! 2124: else {
! 2125: panic("pmap_unwire: invalid PDE");
! 2126: }
! 2127: #endif
! 2128: }
! 2129:
! 2130: /*
! 2131: * pmap_collect: free resources held by a pmap
! 2132: *
! 2133: * => optional function.
! 2134: * => called when a process is swapped out to free memory.
! 2135: */
! 2136:
! 2137: void
! 2138: pmap_collect(struct pmap *pmap)
! 2139: {
! 2140: /*
! 2141: * free all of the pt pages by removing the physical mappings
! 2142: * for its entire address space.
! 2143: */
! 2144:
! 2145: /* pmap_do_remove(pmap, VM_MIN_ADDRESS, VM_MAX_ADDRESS,
! 2146: PMAP_REMOVE_SKIPWIRED);
! 2147: */
! 2148: }
! 2149:
! 2150: /*
! 2151: * pmap_copy: copy mappings from one pmap to another
! 2152: *
! 2153: * => optional function
! 2154: * void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
! 2155: */
! 2156:
! 2157: /*
! 2158: * defined as macro in pmap.h
! 2159: */
! 2160:
! 2161: /*
! 2162: * pmap_enter: enter a mapping into a pmap
! 2163: *
! 2164: * => must be done "now" ... no lazy-evaluation
! 2165: */
! 2166:
! 2167: int
! 2168: pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
! 2169: {
! 2170: pt_entry_t *ptes, opte, npte;
! 2171: pd_entry_t **pdes;
! 2172: struct vm_page *ptp, *pg = NULL;
! 2173: struct pv_entry *pve = NULL;
! 2174: int ptpdelta, wireddelta, resdelta;
! 2175: boolean_t wired = (flags & PMAP_WIRED) != 0;
! 2176: int error;
! 2177:
! 2178: #ifdef DIAGNOSTIC
! 2179: /* sanity check: totally out of range? */
! 2180: if (va >= VM_MAX_KERNEL_ADDRESS)
! 2181: panic("pmap_enter: too big");
! 2182:
! 2183: if (va == (vaddr_t) PDP_BASE || va == (vaddr_t) APDP_BASE)
! 2184: panic("pmap_enter: trying to map over PDP/APDP!");
! 2185:
! 2186: /* sanity check: kernel PTPs should already have been pre-allocated */
! 2187: if (va >= VM_MIN_KERNEL_ADDRESS &&
! 2188: !pmap_valid_entry(pmap->pm_pdir[pl_i(va, PTP_LEVELS)]))
! 2189: panic("pmap_enter: missing kernel PTP for va %lx!", va);
! 2190:
! 2191: #endif
! 2192:
! 2193: /* get lock */
! 2194: PMAP_MAP_TO_HEAD_LOCK();
! 2195:
! 2196: /*
! 2197: * map in ptes and get a pointer to our PTP (unless we are the kernel)
! 2198: */
! 2199:
! 2200: pmap_map_ptes(pmap, &ptes, &pdes); /* locks pmap */
! 2201: if (pmap == pmap_kernel()) {
! 2202: ptp = NULL;
! 2203: } else {
! 2204: ptp = pmap_get_ptp(pmap, va, pdes);
! 2205: if (ptp == NULL) {
! 2206: if (flags & PMAP_CANFAIL) {
! 2207: error = ENOMEM;
! 2208: goto out;
! 2209: }
! 2210: panic("pmap_enter: get ptp failed");
! 2211: }
! 2212: }
! 2213: opte = ptes[pl1_i(va)]; /* old PTE */
! 2214:
! 2215: /*
! 2216: * is there currently a valid mapping at our VA?
! 2217: */
! 2218:
! 2219: if (pmap_valid_entry(opte)) {
! 2220: /*
! 2221: * first, calculate pm_stats updates. resident count will not
! 2222: * change since we are replacing/changing a valid mapping.
! 2223: * wired count might change...
! 2224: */
! 2225:
! 2226: resdelta = 0;
! 2227: if (wired && (opte & PG_W) == 0)
! 2228: wireddelta = 1;
! 2229: else if (!wired && (opte & PG_W) != 0)
! 2230: wireddelta = -1;
! 2231: else
! 2232: wireddelta = 0;
! 2233: ptpdelta = 0;
! 2234:
! 2235: /*
! 2236: * is the currently mapped PA the same as the one we
! 2237: * want to map?
! 2238: */
! 2239:
! 2240: if ((opte & PG_FRAME) == pa) {
! 2241:
! 2242: /* if this is on the PVLIST, sync R/M bit */
! 2243: if (opte & PG_PVLIST) {
! 2244: pg = PHYS_TO_VM_PAGE(pa);
! 2245: #ifdef DIAGNOSTIC
! 2246: if (pg == NULL)
! 2247: panic("pmap_enter: same pa PG_PVLIST "
! 2248: "mapping with unmanaged page "
! 2249: "pa = 0x%lx (0x%lx)", pa,
! 2250: atop(pa));
! 2251: #endif
! 2252: pmap_sync_flags_pte(pg, opte);
! 2253: } else {
! 2254: #ifdef DIAGNOSTIC
! 2255: if (PHYS_TO_VM_PAGE(pa) != NULL)
! 2256: panic("pmap_enter: same pa, managed "
! 2257: "page, no PG_VLIST pa: 0x%lx\n",
! 2258: pa);
! 2259: #endif
! 2260: }
! 2261: goto enter_now;
! 2262: }
! 2263:
! 2264: /*
! 2265: * changing PAs: we must remove the old one first
! 2266: */
! 2267:
! 2268: /*
! 2269: * if current mapping is on a pvlist,
! 2270: * remove it (sync R/M bits)
! 2271: */
! 2272:
! 2273: if (opte & PG_PVLIST) {
! 2274: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
! 2275: #ifdef DIAGNOSTIC
! 2276: if (pg == NULL)
! 2277: panic("pmap_enter: PG_PVLIST mapping with "
! 2278: "unmanaged page "
! 2279: "pa = 0x%lx (0x%lx)", pa, atop(pa));
! 2280: #endif
! 2281: pmap_sync_flags_pte(pg, opte);
! 2282: pve = pmap_remove_pv(pg, pmap, va);
! 2283: pg = NULL; /* This is not the page we are looking for */
! 2284: }
! 2285: } else { /* opte not valid */
! 2286: pve = NULL;
! 2287: resdelta = 1;
! 2288: if (wired)
! 2289: wireddelta = 1;
! 2290: else
! 2291: wireddelta = 0;
! 2292: if (ptp)
! 2293: ptpdelta = 1;
! 2294: else
! 2295: ptpdelta = 0;
! 2296: }
! 2297:
! 2298: /*
! 2299: * pve is either NULL or points to a now-free pv_entry structure
! 2300: * (the latter case is if we called pmap_remove_pv above).
! 2301: *
! 2302: * if this entry is to be on a pvlist, enter it now.
! 2303: */
! 2304:
! 2305: if (pmap_initialized)
! 2306: pg = PHYS_TO_VM_PAGE(pa);
! 2307:
! 2308: if (pg != NULL) {
! 2309: if (pve == NULL) {
! 2310: pve = pool_get(&pmap_pv_pool, PR_NOWAIT);
! 2311: if (pve == NULL) {
! 2312: if (flags & PMAP_CANFAIL) {
! 2313: error = ENOMEM;
! 2314: goto out;
! 2315: }
! 2316: panic("pmap_enter: no pv entries available");
! 2317: }
! 2318: }
! 2319: pmap_enter_pv(pg, pve, pmap, va, ptp);
! 2320: } else {
! 2321: /* new mapping is not PG_PVLIST. free pve if we've got one */
! 2322: if (pve)
! 2323: pool_put(&pmap_pv_pool, pve);
! 2324: }
! 2325:
! 2326: enter_now:
! 2327: /*
! 2328: * at this point pg is !NULL if we want the PG_PVLIST bit set
! 2329: */
! 2330:
! 2331: pmap->pm_stats.resident_count += resdelta;
! 2332: pmap->pm_stats.wired_count += wireddelta;
! 2333: if (ptp)
! 2334: ptp->wire_count += ptpdelta;
! 2335:
! 2336: if (pg != PHYS_TO_VM_PAGE(pa))
! 2337: panic("wtf?");
! 2338:
! 2339: npte = pa | protection_codes[prot] | PG_V;
! 2340: if (pg != NULL)
! 2341: npte |= PG_PVLIST;
! 2342: if (wired)
! 2343: npte |= PG_W;
! 2344: if (va < VM_MAXUSER_ADDRESS)
! 2345: npte |= PG_u;
! 2346: else if (va < VM_MAX_ADDRESS)
! 2347: npte |= (PG_u | PG_RW); /* XXXCDC: no longer needed? */
! 2348: if (pmap == pmap_kernel())
! 2349: npte |= pmap_pg_g;
! 2350:
! 2351: ptes[pl1_i(va)] = npte; /* zap! */
! 2352:
! 2353: /*
! 2354: * If we changed anything other than modified/used bits,
! 2355: * flush the TLB. (is this overkill?)
! 2356: */
! 2357: if (opte & PG_V) {
! 2358: pmap_tlb_shootpage(pmap, va);
! 2359: pmap_tlb_shootwait();
! 2360: }
! 2361:
! 2362: error = 0;
! 2363:
! 2364: out:
! 2365: pmap_unmap_ptes(pmap);
! 2366: PMAP_MAP_TO_HEAD_UNLOCK();
! 2367:
! 2368: return error;
! 2369: }
! 2370:
! 2371: boolean_t
! 2372: pmap_get_physpage(vaddr_t va, int level, paddr_t *paddrp)
! 2373: {
! 2374: struct vm_page *ptp;
! 2375: struct pmap *kpm = pmap_kernel();
! 2376:
! 2377: if (uvm.page_init_done == FALSE) {
! 2378: vaddr_t va;
! 2379:
! 2380: /*
! 2381: * we're growing the kernel pmap early (from
! 2382: * uvm_pageboot_alloc()). this case must be
! 2383: * handled a little differently.
! 2384: */
! 2385:
! 2386: va = pmap_steal_memory(PAGE_SIZE, NULL, NULL);
! 2387: *paddrp = PMAP_DIRECT_UNMAP(va);
! 2388: } else {
! 2389: ptp = uvm_pagealloc(&kpm->pm_obj[level - 1],
! 2390: ptp_va2o(va, level), NULL,
! 2391: UVM_PGA_USERESERVE|UVM_PGA_ZERO);
! 2392: if (ptp == NULL)
! 2393: panic("pmap_get_physpage: out of memory");
! 2394: atomic_clearbits_int(&ptp->pg_flags, PG_BUSY);
! 2395: ptp->wire_count = 1;
! 2396: *paddrp = VM_PAGE_TO_PHYS(ptp);
! 2397: }
! 2398: kpm->pm_stats.resident_count++;
! 2399: return TRUE;
! 2400: }
! 2401:
! 2402: /*
! 2403: * Allocate the amount of specified ptps for a ptp level, and populate
! 2404: * all levels below accordingly, mapping virtual addresses starting at
! 2405: * kva.
! 2406: *
! 2407: * Used by pmap_growkernel.
! 2408: */
! 2409: void
! 2410: pmap_alloc_level(pd_entry_t **pdes, vaddr_t kva, int lvl, long *needed_ptps)
! 2411: {
! 2412: unsigned long i;
! 2413: vaddr_t va;
! 2414: paddr_t pa;
! 2415: unsigned long index, endindex;
! 2416: int level;
! 2417: pd_entry_t *pdep;
! 2418:
! 2419: for (level = lvl; level > 1; level--) {
! 2420: if (level == PTP_LEVELS)
! 2421: pdep = pmap_kernel()->pm_pdir;
! 2422: else
! 2423: pdep = pdes[level - 2];
! 2424: va = kva;
! 2425: index = pl_i(kva, level);
! 2426: endindex = index + needed_ptps[level - 1];
! 2427: /*
! 2428: * XXX special case for first time call.
! 2429: */
! 2430: if (nkptp[level - 1] != 0)
! 2431: index++;
! 2432: else
! 2433: endindex--;
! 2434:
! 2435: for (i = index; i <= endindex; i++) {
! 2436: pmap_get_physpage(va, level - 1, &pa);
! 2437: pdep[i] = pa | PG_RW | PG_V;
! 2438: nkptp[level - 1]++;
! 2439: va += nbpd[level - 1];
! 2440: }
! 2441: }
! 2442: }
! 2443:
! 2444: /*
! 2445: * pmap_growkernel: increase usage of KVM space
! 2446: *
! 2447: * => we allocate new PTPs for the kernel and install them in all
! 2448: * the pmaps on the system.
! 2449: */
! 2450:
! 2451: static vaddr_t pmap_maxkvaddr = VM_MIN_KERNEL_ADDRESS;
! 2452:
! 2453: vaddr_t
! 2454: pmap_growkernel(vaddr_t maxkvaddr)
! 2455: {
! 2456: struct pmap *kpm = pmap_kernel(), *pm;
! 2457: int s, i;
! 2458: unsigned newpdes;
! 2459: long needed_kptp[PTP_LEVELS], target_nptp, old;
! 2460:
! 2461: if (maxkvaddr <= pmap_maxkvaddr)
! 2462: return pmap_maxkvaddr;
! 2463:
! 2464: maxkvaddr = x86_round_pdr(maxkvaddr);
! 2465: old = nkptp[PTP_LEVELS - 1];
! 2466: /*
! 2467: * This loop could be optimized more, but pmap_growkernel()
! 2468: * is called infrequently.
! 2469: */
! 2470: for (i = PTP_LEVELS - 1; i >= 1; i--) {
! 2471: target_nptp = pl_i(maxkvaddr, i + 1) -
! 2472: pl_i(VM_MIN_KERNEL_ADDRESS, i + 1);
! 2473: /*
! 2474: * XXX only need to check toplevel.
! 2475: */
! 2476: if (target_nptp > nkptpmax[i])
! 2477: panic("out of KVA space");
! 2478: needed_kptp[i] = target_nptp - nkptp[i] + 1;
! 2479: }
! 2480:
! 2481:
! 2482: s = splhigh(); /* to be safe */
! 2483: simple_lock(&kpm->pm_lock);
! 2484: pmap_alloc_level(normal_pdes, pmap_maxkvaddr, PTP_LEVELS,
! 2485: needed_kptp);
! 2486:
! 2487: /*
! 2488: * If the number of top level entries changed, update all
! 2489: * pmaps.
! 2490: */
! 2491: if (needed_kptp[PTP_LEVELS - 1] != 0) {
! 2492: newpdes = nkptp[PTP_LEVELS - 1] - old;
! 2493: simple_lock(&pmaps_lock);
! 2494: LIST_FOREACH(pm, &pmaps, pm_list) {
! 2495: memcpy(&pm->pm_pdir[PDIR_SLOT_KERN + old],
! 2496: &kpm->pm_pdir[PDIR_SLOT_KERN + old],
! 2497: newpdes * sizeof (pd_entry_t));
! 2498: }
! 2499:
! 2500: /* Invalidate the PDP cache. */
! 2501: pool_cache_invalidate(&pmap_pdp_cache);
! 2502: pmap_pdp_cache_generation++;
! 2503:
! 2504: simple_unlock(&pmaps_lock);
! 2505: }
! 2506: pmap_maxkvaddr = maxkvaddr;
! 2507: simple_unlock(&kpm->pm_lock);
! 2508: splx(s);
! 2509:
! 2510: return maxkvaddr;
! 2511: }
! 2512:
! 2513: vaddr_t
! 2514: pmap_steal_memory(vsize_t size, vaddr_t *start, vaddr_t *end)
! 2515: {
! 2516: int segno;
! 2517: u_int npg;
! 2518: vaddr_t va;
! 2519: paddr_t pa;
! 2520: struct vm_physseg *seg;
! 2521:
! 2522: size = round_page(size);
! 2523: npg = atop(size);
! 2524:
! 2525: for (segno = 0, seg = vm_physmem; segno < vm_nphysseg; segno++, seg++) {
! 2526: if (seg->avail_end - seg->avail_start < npg)
! 2527: continue;
! 2528: /*
! 2529: * We can only steal at an ``unused'' segment boundary,
! 2530: * i.e. either at the start or at the end.
! 2531: */
! 2532: if (seg->avail_start == seg->start ||
! 2533: seg->avail_end == seg->end)
! 2534: break;
! 2535: }
! 2536: if (segno == vm_nphysseg) {
! 2537: panic("pmap_steal_memory: out of memory");
! 2538: } else {
! 2539: if (seg->avail_start == seg->start) {
! 2540: pa = ptoa(seg->avail_start);
! 2541: seg->avail_start += npg;
! 2542: seg->start += npg;
! 2543: } else {
! 2544: pa = ptoa(seg->avail_end) - size;
! 2545: seg->avail_end -= npg;
! 2546: seg->end -= npg;
! 2547: }
! 2548: /*
! 2549: * If all the segment has been consumed now, remove it.
! 2550: * Note that the crash dump code still knows about it
! 2551: * and will dump it correctly.
! 2552: */
! 2553: if (seg->start == seg->end) {
! 2554: if (vm_nphysseg-- == 1)
! 2555: panic("pmap_steal_memory: out of memory");
! 2556: while (segno < vm_nphysseg) {
! 2557: seg[0] = seg[1]; /* struct copy */
! 2558: seg++;
! 2559: segno++;
! 2560: }
! 2561: }
! 2562:
! 2563: va = PMAP_DIRECT_MAP(pa);
! 2564: memset((void *)va, 0, size);
! 2565: }
! 2566:
! 2567: if (start != NULL)
! 2568: *start = virtual_avail;
! 2569: if (end != NULL)
! 2570: *end = VM_MAX_KERNEL_ADDRESS;
! 2571:
! 2572: return (va);
! 2573: }
! 2574:
! 2575: #ifdef DEBUG
! 2576: void pmap_dump(struct pmap *, vaddr_t, vaddr_t);
! 2577:
! 2578: /*
! 2579: * pmap_dump: dump all the mappings from a pmap
! 2580: *
! 2581: * => caller should not be holding any pmap locks
! 2582: */
! 2583:
! 2584: void
! 2585: pmap_dump(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
! 2586: {
! 2587: pt_entry_t *ptes, *pte;
! 2588: pd_entry_t **pdes;
! 2589: vaddr_t blkendva;
! 2590:
! 2591: /*
! 2592: * if end is out of range truncate.
! 2593: * if (end == start) update to max.
! 2594: */
! 2595:
! 2596: if (eva > VM_MAXUSER_ADDRESS || eva <= sva)
! 2597: eva = VM_MAXUSER_ADDRESS;
! 2598:
! 2599:
! 2600: PMAP_MAP_TO_HEAD_LOCK();
! 2601: pmap_map_ptes(pmap, &ptes, &pdes); /* locks pmap */
! 2602:
! 2603: /*
! 2604: * dumping a range of pages: we dump in PTP sized blocks (4MB)
! 2605: */
! 2606:
! 2607: for (/* null */ ; sva < eva ; sva = blkendva) {
! 2608:
! 2609: /* determine range of block */
! 2610: blkendva = x86_round_pdr(sva+1);
! 2611: if (blkendva > eva)
! 2612: blkendva = eva;
! 2613:
! 2614: /* valid block? */
! 2615: if (!pmap_pdes_valid(sva, pdes, NULL))
! 2616: continue;
! 2617:
! 2618: pte = &ptes[pl1_i(sva)];
! 2619: for (/* null */; sva < blkendva ; sva += PAGE_SIZE, pte++) {
! 2620: if (!pmap_valid_entry(*pte))
! 2621: continue;
! 2622: printf("va %#lx -> pa %#lx (pte=%#lx)\n",
! 2623: sva, *pte, *pte & PG_FRAME);
! 2624: }
! 2625: }
! 2626: pmap_unmap_ptes(pmap);
! 2627: PMAP_MAP_TO_HEAD_UNLOCK();
! 2628: }
! 2629: #endif
! 2630:
! 2631: void
! 2632: pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
! 2633: {
! 2634: *vstartp = virtual_avail;
! 2635: *vendp = VM_MAX_KERNEL_ADDRESS;
! 2636: }
! 2637:
! 2638: #ifdef MULTIPROCESSOR
! 2639: /*
! 2640: * Locking for tlb shootdown.
! 2641: *
! 2642: * We lock by setting tlb_shoot_wait to the number of cpus that will
! 2643: * receive our tlb shootdown. After sending the IPIs, we don't need to
! 2644: * worry about locking order or interrupts spinning for the lock because
! 2645: * the call that grabs the "lock" isn't the one that releases it. And
! 2646: * there is nothing that can block the IPI that releases the lock.
! 2647: *
! 2648: * The functions are organized so that we first count the number of
! 2649: * cpus we need to send the IPI to, then we grab the counter, then
! 2650: * we send the IPIs, then we finally do our own shootdown.
! 2651: *
! 2652: * Our shootdown is last to make it parallell with the other cpus
! 2653: * to shorten the spin time.
! 2654: *
! 2655: * Notice that we depend on failures to send IPIs only being able to
! 2656: * happen during boot. If they happen later, the above assumption
! 2657: * doesn't hold since we can end up in situations where noone will
! 2658: * release the lock if we get an interrupt in a bad moment.
! 2659: */
! 2660:
! 2661: volatile long tlb_shoot_wait;
! 2662:
! 2663: volatile vaddr_t tlb_shoot_addr1;
! 2664: volatile vaddr_t tlb_shoot_addr2;
! 2665:
! 2666: /* XXX */
! 2667: #define SPINLOCK_SPIN_HOOK __asm __volatile("pause": : :"memory")
! 2668:
! 2669: void
! 2670: pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
! 2671: {
! 2672: struct cpu_info *ci, *self = curcpu();
! 2673: CPU_INFO_ITERATOR cii;
! 2674: long wait = 0;
! 2675: int mask = 0;
! 2676:
! 2677: CPU_INFO_FOREACH(cii, ci) {
! 2678: if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
! 2679: !(ci->ci_flags & CPUF_RUNNING))
! 2680: continue;
! 2681: mask |= 1 << ci->ci_cpuid;
! 2682: wait++;
! 2683: }
! 2684:
! 2685: if (wait > 0) {
! 2686: int s = splvm();
! 2687:
! 2688: while (x86_atomic_cas_ul(&tlb_shoot_wait, 0, wait) != 0) {
! 2689: while (tlb_shoot_wait != 0)
! 2690: SPINLOCK_SPIN_HOOK;
! 2691: }
! 2692: tlb_shoot_addr1 = va;
! 2693: CPU_INFO_FOREACH(cii, ci) {
! 2694: if ((mask & 1 << ci->ci_cpuid) == 0)
! 2695: continue;
! 2696: if (x86_fast_ipi(ci, LAPIC_IPI_INVLPG) != 0)
! 2697: panic("pmap_tlb_shootpage: ipi failed");
! 2698: }
! 2699: splx(s);
! 2700: }
! 2701:
! 2702: if (pmap_is_curpmap(pm))
! 2703: pmap_update_pg(va);
! 2704: }
! 2705:
! 2706: void
! 2707: pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
! 2708: {
! 2709: struct cpu_info *ci, *self = curcpu();
! 2710: CPU_INFO_ITERATOR cii;
! 2711: long wait = 0;
! 2712: int mask = 0;
! 2713: vaddr_t va;
! 2714:
! 2715: CPU_INFO_FOREACH(cii, ci) {
! 2716: if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
! 2717: !(ci->ci_flags & CPUF_RUNNING))
! 2718: continue;
! 2719: mask |= 1 << ci->ci_cpuid;
! 2720: wait++;
! 2721: }
! 2722:
! 2723: if (wait > 0) {
! 2724: int s = splvm();
! 2725:
! 2726: while (x86_atomic_cas_ul(&tlb_shoot_wait, 0, wait) != 0) {
! 2727: while (tlb_shoot_wait != 0)
! 2728: SPINLOCK_SPIN_HOOK;
! 2729: }
! 2730: tlb_shoot_addr1 = sva;
! 2731: tlb_shoot_addr2 = eva;
! 2732: CPU_INFO_FOREACH(cii, ci) {
! 2733: if ((mask & 1 << ci->ci_cpuid) == 0)
! 2734: continue;
! 2735: if (x86_fast_ipi(ci, LAPIC_IPI_INVLRANGE) != 0)
! 2736: panic("pmap_tlb_shootrange: ipi failed");
! 2737: }
! 2738: splx(s);
! 2739: }
! 2740:
! 2741: if (pmap_is_curpmap(pm))
! 2742: for (va = sva; va < eva; va += PAGE_SIZE)
! 2743: pmap_update_pg(va);
! 2744: }
! 2745:
! 2746: void
! 2747: pmap_tlb_shoottlb(void)
! 2748: {
! 2749: struct cpu_info *ci, *self = curcpu();
! 2750: CPU_INFO_ITERATOR cii;
! 2751: long wait = 0;
! 2752: int mask = 0;
! 2753:
! 2754: CPU_INFO_FOREACH(cii, ci) {
! 2755: if (ci == self || !(ci->ci_flags & CPUF_RUNNING))
! 2756: continue;
! 2757: mask |= 1 << ci->ci_cpuid;
! 2758: wait++;
! 2759: }
! 2760:
! 2761: if (wait) {
! 2762: int s = splvm();
! 2763:
! 2764: while (x86_atomic_cas_ul(&tlb_shoot_wait, 0, wait) != 0) {
! 2765: while (tlb_shoot_wait != 0)
! 2766: SPINLOCK_SPIN_HOOK;
! 2767: }
! 2768:
! 2769: CPU_INFO_FOREACH(cii, ci) {
! 2770: if ((mask & 1 << ci->ci_cpuid) == 0)
! 2771: continue;
! 2772: if (x86_fast_ipi(ci, LAPIC_IPI_INVLTLB) != 0)
! 2773: panic("pmap_tlb_shoottlb: ipi failed");
! 2774: }
! 2775: splx(s);
! 2776: }
! 2777:
! 2778: tlbflush();
! 2779: }
! 2780:
! 2781: void
! 2782: pmap_tlb_shootwait(void)
! 2783: {
! 2784: while (tlb_shoot_wait != 0)
! 2785: SPINLOCK_SPIN_HOOK;
! 2786: }
! 2787:
! 2788: #else
! 2789:
! 2790: void
! 2791: pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
! 2792: {
! 2793: if (pmap_is_curpmap(pm))
! 2794: pmap_update_pg(va);
! 2795:
! 2796: }
! 2797:
! 2798: void
! 2799: pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
! 2800: {
! 2801: vaddr_t va;
! 2802:
! 2803: for (va = sva; va < eva; va += PAGE_SIZE)
! 2804: pmap_update_pg(va);
! 2805:
! 2806: }
! 2807:
! 2808: void
! 2809: pmap_tlb_shoottlb(void)
! 2810: {
! 2811: tlbflush();
! 2812: }
! 2813: #endif /* MULTIPROCESSOR */
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / amd64 / amd64