Annotation of sys/arch/i386/i386/pmap.c, Revision 1.1.1.1
1.1 nbrk 1: /* $OpenBSD: pmap.c,v 1.119 2007/06/27 16:16:53 art Exp $ */
2: /* $NetBSD: pmap.c,v 1.91 2000/06/02 17:46:37 thorpej Exp $ */
3:
4: /*
5: *
6: * Copyright (c) 1997 Charles D. Cranor and Washington University.
7: * All rights reserved.
8: *
9: * Redistribution and use in source and binary forms, with or without
10: * modification, are permitted provided that the following conditions
11: * are met:
12: * 1. Redistributions of source code must retain the above copyright
13: * notice, this list of conditions and the following disclaimer.
14: * 2. Redistributions in binary form must reproduce the above copyright
15: * notice, this list of conditions and the following disclaimer in the
16: * documentation and/or other materials provided with the distribution.
17: * 3. All advertising materials mentioning features or use of this software
18: * must display the following acknowledgement:
19: * This product includes software developed by Charles D. Cranor and
20: * Washington University.
21: * 4. The name of the author may not be used to endorse or promote products
22: * derived from this software without specific prior written permission.
23: *
24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34: */
35:
36: /*
37: * pmap.c: i386 pmap module rewrite
38: * Chuck Cranor <chuck@ccrc.wustl.edu>
39: * 11-Aug-97
40: *
41: * history of this pmap module: in addition to my own input, i used
42: * the following references for this rewrite of the i386 pmap:
43: *
44: * [1] the NetBSD i386 pmap. this pmap appears to be based on the
45: * BSD hp300 pmap done by Mike Hibler at University of Utah.
46: * it was then ported to the i386 by William Jolitz of UUNET
47: * Technologies, Inc. Then Charles M. Hannum of the NetBSD
48: * project fixed some bugs and provided some speed ups.
49: *
50: * [2] the FreeBSD i386 pmap. this pmap seems to be the
51: * Hibler/Jolitz pmap, as modified for FreeBSD by John S. Dyson
52: * and David Greenman.
53: *
54: * [3] the Mach pmap. this pmap, from CMU, seems to have migrated
55: * between several processors. the VAX version was done by
56: * Avadis Tevanian, Jr., and Michael Wayne Young. the i386
57: * version was done by Lance Berc, Mike Kupfer, Bob Baron,
58: * David Golub, and Richard Draves. the alpha version was
59: * done by Alessandro Forin (CMU/Mach) and Chris Demetriou
60: * (NetBSD/alpha).
61: */
62:
63: #include <sys/param.h>
64: #include <sys/systm.h>
65: #include <sys/proc.h>
66: #include <sys/malloc.h>
67: #include <sys/pool.h>
68: #include <sys/user.h>
69: #include <sys/kernel.h>
70: #include <sys/mutex.h>
71:
72: #include <uvm/uvm.h>
73:
74: #include <machine/atomic.h>
75: #include <machine/cpu.h>
76: #include <machine/specialreg.h>
77: #include <machine/gdt.h>
78:
79: #include <dev/isa/isareg.h>
80: #include <sys/msgbuf.h>
81: #include <stand/boot/bootarg.h>
82:
83: /*
84: * general info:
85: *
86: * - for an explanation of how the i386 MMU hardware works see
87: * the comments in <machine/pte.h>.
88: *
89: * - for an explanation of the general memory structure used by
90: * this pmap (including the recursive mapping), see the comments
91: * in <machine/pmap.h>.
92: *
93: * this file contains the code for the "pmap module." the module's
94: * job is to manage the hardware's virtual to physical address mappings.
95: * note that there are two levels of mapping in the VM system:
96: *
97: * [1] the upper layer of the VM system uses vm_map's and vm_map_entry's
98: * to map ranges of virtual address space to objects/files. for
99: * example, the vm_map may say: "map VA 0x1000 to 0x22000 read-only
100: * to the file /bin/ls starting at offset zero." note that
101: * the upper layer mapping is not concerned with how individual
102: * vm_pages are mapped.
103: *
104: * [2] the lower layer of the VM system (the pmap) maintains the mappings
105: * from virtual addresses. it is concerned with which vm_page is
106: * mapped where. for example, when you run /bin/ls and start
107: * at page 0x1000 the fault routine may lookup the correct page
108: * of the /bin/ls file and then ask the pmap layer to establish
109: * a mapping for it.
110: *
111: * note that information in the lower layer of the VM system can be
112: * thrown away since it can easily be reconstructed from the info
113: * in the upper layer.
114: *
115: * data structures we use include:
116: *
117: * - struct pmap: describes the address space of one thread
118: * - struct pv_entry: describes one <PMAP,VA> mapping of a PA
119: * - struct pv_head: there is one pv_head per managed page of
120: * physical memory. the pv_head points to a list of pv_entry
121: * structures which describe all the <PMAP,VA> pairs that this
122: * page is mapped in. this is critical for page based operations
123: * such as pmap_page_protect() [change protection on _all_ mappings
124: * of a page]
125: * - pv_page/pv_page_info: pv_entry's are allocated out of pv_page's.
126: * if we run out of pv_entry's we allocate a new pv_page and free
127: * its pv_entrys.
128: */
129: /*
130: * memory allocation
131: *
132: * - there are three data structures that we must dynamically allocate:
133: *
134: * [A] new process' page directory page (PDP)
135: * - plan 1: done at pmap_create() we use
136: * uvm_km_alloc(kernel_map, PAGE_SIZE) [fka kmem_alloc] to do this
137: * allocation.
138: *
139: * if we are low in free physical memory then we sleep in
140: * uvm_km_alloc -- in this case this is ok since we are creating
141: * a new pmap and should not be holding any locks.
142: *
143: * if the kernel is totally out of virtual space
144: * (i.e. uvm_km_alloc returns NULL), then we panic.
145: *
146: * XXX: the fork code currently has no way to return an "out of
147: * memory, try again" error code since uvm_fork [fka vm_fork]
148: * is a void function.
149: *
150: * [B] new page tables pages (PTP)
151: * call uvm_pagealloc()
152: * => success: zero page, add to pm_pdir
153: * => failure: we are out of free vm_pages, let pmap_enter()
154: * tell UVM about it.
155: *
156: * note: for kernel PTPs, we start with NKPTP of them. as we map
157: * kernel memory (at uvm_map time) we check to see if we've grown
158: * the kernel pmap. if so, we call the optional function
159: * pmap_growkernel() to grow the kernel PTPs in advance.
160: *
161: * [C] pv_entry structures
162: * - plan 1: try to allocate one off the free list
163: * => success: done!
164: * => failure: no more free pv_entrys on the list
165: * - plan 2: try to allocate a new pv_page to add a chunk of
166: * pv_entrys to the free list
167: * [a] obtain a free, unmapped, VA in kmem_map. either
168: * we have one saved from a previous call, or we allocate
169: * one now using a "vm_map_lock_try" in uvm_map
170: * => success: we have an unmapped VA, continue to [b]
171: * => failure: unable to lock kmem_map or out of VA in it.
172: * move on to plan 3.
173: * [b] allocate a page for the VA
174: * => success: map it in, free the pv_entry's, DONE!
175: * => failure: no free vm_pages, etc.
176: * save VA for later call to [a], go to plan 3.
177: * If we fail, we simply let pmap_enter() tell UVM about it.
178: */
179: /*
180: * locking
181: *
182: * we have the following locks that we must contend with:
183: *
184: * "simple" locks:
185: *
186: * - pmap lock (per pmap, part of uvm_object)
187: * this lock protects the fields in the pmap structure including
188: * the non-kernel PDEs in the PDP, and the PTEs. it also locks
189: * in the alternate PTE space (since that is determined by the
190: * entry in the PDP).
191: *
192: * - pvalloc_lock
193: * this lock protects the data structures which are used to manage
194: * the free list of pv_entry structures.
195: *
196: * - pmaps_lock
197: * this lock protects the list of active pmaps (headed by "pmaps").
198: * we lock it when adding or removing pmaps from this list.
199: *
200: */
201:
202: /*
203: * locking data structures
204: */
205:
206: struct simplelock pvalloc_lock;
207: struct simplelock pmaps_lock;
208:
209: #define PMAP_MAP_TO_HEAD_LOCK() /* null */
210: #define PMAP_MAP_TO_HEAD_UNLOCK() /* null */
211:
212: #define PMAP_HEAD_TO_MAP_LOCK() /* null */
213: #define PMAP_HEAD_TO_MAP_UNLOCK() /* null */
214:
215: /*
216: * global data structures
217: */
218:
219: struct pmap kernel_pmap_store; /* the kernel's pmap (proc0) */
220:
221: /*
222: * nkpde is the number of kernel PTPs allocated for the kernel at
223: * boot time (NKPTP is a compile time override). this number can
224: * grow dynamically as needed (but once allocated, we never free
225: * kernel PTPs).
226: */
227:
228: int nkpde = NKPTP;
229: #ifdef NKPDE
230: #error "obsolete NKPDE: use NKPTP"
231: #endif
232:
233: /*
234: * pmap_pg_g: if our processor supports PG_G in the PTE then we
235: * set pmap_pg_g to PG_G (otherwise it is zero).
236: */
237:
238: int pmap_pg_g = 0;
239:
240: /*
241: * i386 physical memory comes in a big contig chunk with a small
242: * hole toward the front of it... the following 4 paddr_t's
243: * (shared with machdep.c) describe the physical address space
244: * of this machine.
245: */
246: paddr_t avail_start; /* PA of first available physical page */
247: paddr_t hole_start; /* PA of start of "hole" */
248: paddr_t hole_end; /* PA of end of "hole" */
249:
250: /*
251: * other data structures
252: */
253:
254: static pt_entry_t protection_codes[8]; /* maps MI prot to i386 prot code */
255: static boolean_t pmap_initialized = FALSE; /* pmap_init done yet? */
256:
257: /*
258: * the following two vaddr_t's are used during system startup
259: * to keep track of how much of the kernel's VM space we have used.
260: * once the system is started, the management of the remaining kernel
261: * VM space is turned over to the kernel_map vm_map.
262: */
263:
264: static vaddr_t virtual_avail; /* VA of first free KVA */
265: static vaddr_t virtual_end; /* VA of last free KVA */
266:
267: /*
268: * pv_page management structures: locked by pvalloc_lock
269: */
270:
271: TAILQ_HEAD(pv_pagelist, pv_page);
272: static struct pv_pagelist pv_freepages; /* list of pv_pages with free entries */
273: static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */
274: static int pv_nfpvents; /* # of free pv entries */
275: static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */
276: static vaddr_t pv_cachedva; /* cached VA for later use */
277:
278: #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */
279: #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2))
280: /* high water mark */
281:
282: /*
283: * linked list of all non-kernel pmaps
284: */
285:
286: struct pmap_head pmaps;
287:
288: /*
289: * pool that pmap structures are allocated from
290: */
291:
292: struct pool pmap_pmap_pool;
293:
294: /*
295: * MULTIPROCESSOR: special VA's/ PTE's are actually allocated inside a
296: * I386_MAXPROCS*NPTECL array of PTE's, to avoid cache line thrashing
297: * due to false sharing.
298: */
299:
300: #ifdef MULTIPROCESSOR
301: #define PTESLEW(pte, id) ((pte)+(id)*NPTECL)
302: #define VASLEW(va,id) ((va)+(id)*NPTECL*NBPG)
303: #else
304: #define PTESLEW(pte, id) (pte)
305: #define VASLEW(va,id) (va)
306: #endif
307:
308: /*
309: * special VAs and the PTEs that map them
310: */
311:
312: static pt_entry_t *csrc_pte, *cdst_pte, *zero_pte, *ptp_pte;
313: static caddr_t csrcp, cdstp, zerop, ptpp;
314: caddr_t vmmap; /* XXX: used by mem.c... it should really uvm_map_reserve it */
315:
316: #if defined(I586_CPU)
317: /* stuff to fix the pentium f00f bug */
318: extern vaddr_t pentium_idt_vaddr;
319: #endif
320:
321:
322: /*
323: * local prototypes
324: */
325:
326: struct pv_entry *pmap_add_pvpage(struct pv_page *, boolean_t);
327: struct vm_page *pmap_alloc_ptp(struct pmap *, int, boolean_t, pt_entry_t);
328: struct pv_entry *pmap_alloc_pv(struct pmap *, int); /* see codes below */
329: #define ALLOCPV_NEED 0 /* need PV now */
330: #define ALLOCPV_TRY 1 /* just try to allocate */
331: #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */
332: struct pv_entry *pmap_alloc_pvpage(struct pmap *, int);
333: void pmap_enter_pv(struct vm_page *, struct pv_entry *,
334: struct pmap *, vaddr_t, struct vm_page *);
335: void pmap_free_pv(struct pmap *, struct pv_entry *);
336: void pmap_free_pvs(struct pmap *, struct pv_entry *);
337: void pmap_free_pv_doit(struct pv_entry *);
338: void pmap_free_pvpage(void);
339: struct vm_page *pmap_get_ptp(struct pmap *, int, boolean_t);
340: boolean_t pmap_is_curpmap(struct pmap *);
341: boolean_t pmap_is_active(struct pmap *, int);
342: void pmap_sync_flags_pte(struct vm_page *, u_long);
343: pt_entry_t *pmap_map_ptes(struct pmap *);
344: struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t);
345: void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int);
346: boolean_t pmap_remove_pte(struct pmap *, struct vm_page *, pt_entry_t *,
347: vaddr_t, int);
348: void pmap_remove_ptes(struct pmap *, struct vm_page *, vaddr_t,
349: vaddr_t, vaddr_t, int);
350:
351: #define PMAP_REMOVE_ALL 0
352: #define PMAP_REMOVE_SKIPWIRED 1
353:
354: vaddr_t pmap_tmpmap_pa(paddr_t);
355: pt_entry_t *pmap_tmpmap_pvepte(struct pv_entry *);
356: void pmap_tmpunmap_pa(void);
357: void pmap_tmpunmap_pvepte(struct pv_entry *);
358: void pmap_apte_flush(struct pmap *);
359: void pmap_unmap_ptes(struct pmap *);
360: void pmap_exec_account(struct pmap *, vaddr_t, pt_entry_t,
361: pt_entry_t);
362:
363: void pmap_pinit(pmap_t);
364: void pmap_release(pmap_t);
365:
366: void pmap_zero_phys(paddr_t);
367:
368: void setcslimit(struct pmap *, struct trapframe *, struct pcb *, vaddr_t);
369:
370: /*
371: * p m a p i n l i n e h e l p e r f u n c t i o n s
372: */
373:
374: /*
375: * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
376: * of course the kernel is always loaded
377: */
378:
379: boolean_t
380: pmap_is_curpmap(pmap)
381: struct pmap *pmap;
382: {
383: return((pmap == pmap_kernel()) ||
384: (pmap->pm_pdirpa == (paddr_t) rcr3()));
385: }
386:
387: /*
388: * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
389: */
390:
391: boolean_t
392: pmap_is_active(pmap, cpu_id)
393: struct pmap *pmap;
394: int cpu_id;
395: {
396:
397: return (pmap == pmap_kernel() ||
398: (pmap->pm_cpus & (1U << cpu_id)) != 0);
399: }
400:
401: static __inline u_int
402: pmap_pte2flags(u_long pte)
403: {
404: return (((pte & PG_U) ? PG_PMAP_REF : 0) |
405: ((pte & PG_M) ? PG_PMAP_MOD : 0));
406: }
407:
408: static __inline u_int
409: pmap_flags2pte(u_long pte)
410: {
411: return (((pte & PG_PMAP_REF) ? PG_U : 0) |
412: ((pte & PG_PMAP_MOD) ? PG_M : 0));
413: }
414:
415: void
416: pmap_sync_flags_pte(struct vm_page *pg, u_long pte)
417: {
418: if (pte & (PG_U|PG_M)) {
419: atomic_setbits_int(&pg->pg_flags, pmap_pte2flags(pte));
420: }
421: }
422:
423: /*
424: * pmap_tmpmap_pa: map a page in for tmp usage
425: */
426:
427: vaddr_t
428: pmap_tmpmap_pa(paddr_t pa)
429: {
430: #ifdef MULTIPROCESSOR
431: int id = cpu_number();
432: #endif
433: pt_entry_t *ptpte = PTESLEW(ptp_pte, id);
434: caddr_t ptpva = VASLEW(ptpp, id);
435: #if defined(DIAGNOSTIC)
436: if (*ptpte)
437: panic("pmap_tmpmap_pa: ptp_pte in use?");
438: #endif
439: *ptpte = PG_V | PG_RW | pa; /* always a new mapping */
440: return((vaddr_t)ptpva);
441: }
442:
443: /*
444: * pmap_tmpunmap_pa: unmap a tmp use page (undoes pmap_tmpmap_pa)
445: */
446:
447: void
448: pmap_tmpunmap_pa()
449: {
450: #ifdef MULTIPROCESSOR
451: int id = cpu_number();
452: #endif
453: pt_entry_t *ptpte = PTESLEW(ptp_pte, id);
454: caddr_t ptpva = VASLEW(ptpp, id);
455: #if defined(DIAGNOSTIC)
456: if (!pmap_valid_entry(*ptpte))
457: panic("pmap_tmpunmap_pa: our pte invalid?");
458: #endif
459: *ptpte = 0; /* zap! */
460: pmap_update_pg((vaddr_t)ptpva);
461: #ifdef MULTIPROCESSOR
462: /*
463: * No need for tlb shootdown here, since ptp_pte is per-CPU.
464: */
465: #endif
466: }
467:
468: /*
469: * pmap_tmpmap_pvepte: get a quick mapping of a PTE for a pv_entry
470: *
471: * => do NOT use this on kernel mappings [why? because pv_ptp may be NULL]
472: */
473:
474: pt_entry_t *
475: pmap_tmpmap_pvepte(struct pv_entry *pve)
476: {
477: #ifdef DIAGNOSTIC
478: if (pve->pv_pmap == pmap_kernel())
479: panic("pmap_tmpmap_pvepte: attempt to map kernel");
480: #endif
481:
482: /* is it current pmap? use direct mapping... */
483: if (pmap_is_curpmap(pve->pv_pmap))
484: return(vtopte(pve->pv_va));
485:
486: return(((pt_entry_t *)pmap_tmpmap_pa(VM_PAGE_TO_PHYS(pve->pv_ptp)))
487: + ptei((unsigned)pve->pv_va));
488: }
489:
490: /*
491: * pmap_tmpunmap_pvepte: release a mapping obtained with pmap_tmpmap_pvepte
492: */
493:
494: void
495: pmap_tmpunmap_pvepte(struct pv_entry *pve)
496: {
497: /* was it current pmap? if so, return */
498: if (pmap_is_curpmap(pve->pv_pmap))
499: return;
500:
501: pmap_tmpunmap_pa();
502: }
503:
504: void
505: pmap_apte_flush(struct pmap *pmap)
506: {
507: pmap_tlb_shoottlb();
508: pmap_tlb_shootwait();
509: }
510:
511: /*
512: * pmap_map_ptes: map a pmap's PTEs into KVM and lock them in
513: *
514: * => we lock enough pmaps to keep things locked in
515: * => must be undone with pmap_unmap_ptes before returning
516: */
517:
518: pt_entry_t *
519: pmap_map_ptes(struct pmap *pmap)
520: {
521: pd_entry_t opde;
522:
523: /* the kernel's pmap is always accessible */
524: if (pmap == pmap_kernel()) {
525: return(PTE_BASE);
526: }
527:
528: /* if curpmap then we are always mapped */
529: if (pmap_is_curpmap(pmap)) {
530: simple_lock(&pmap->pm_obj.vmobjlock);
531: return(PTE_BASE);
532: }
533:
534: /* need to lock both curpmap and pmap: use ordered locking */
535: if ((unsigned) pmap < (unsigned) curpcb->pcb_pmap) {
536: simple_lock(&pmap->pm_obj.vmobjlock);
537: simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);
538: } else {
539: simple_lock(&curpcb->pcb_pmap->pm_obj.vmobjlock);
540: simple_lock(&pmap->pm_obj.vmobjlock);
541: }
542:
543: /* need to load a new alternate pt space into curpmap? */
544: opde = *APDP_PDE;
545: if (!pmap_valid_entry(opde) || (opde & PG_FRAME) != pmap->pm_pdirpa) {
546: *APDP_PDE = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V |
547: PG_U | PG_M);
548: if (pmap_valid_entry(opde))
549: pmap_apte_flush(curpcb->pcb_pmap);
550: }
551: return(APTE_BASE);
552: }
553:
554: /*
555: * pmap_unmap_ptes: unlock the PTE mapping of "pmap"
556: */
557:
558: void
559: pmap_unmap_ptes(struct pmap *pmap)
560: {
561: if (pmap == pmap_kernel())
562: return;
563:
564: if (pmap_is_curpmap(pmap)) {
565: simple_unlock(&pmap->pm_obj.vmobjlock);
566: } else {
567: #if defined(MULTIPROCESSOR)
568: *APDP_PDE = 0;
569: pmap_apte_flush(curpcb->pcb_pmap);
570: #endif
571: simple_unlock(&pmap->pm_obj.vmobjlock);
572: simple_unlock(&curpcb->pcb_pmap->pm_obj.vmobjlock);
573: }
574: }
575:
576: void
577: pmap_exec_account(struct pmap *pm, vaddr_t va,
578: pt_entry_t opte, pt_entry_t npte)
579: {
580: if (pm == pmap_kernel())
581: return;
582:
583: if (curproc == NULL || curproc->p_vmspace == NULL ||
584: pm != vm_map_pmap(&curproc->p_vmspace->vm_map))
585: return;
586:
587: if ((opte ^ npte) & PG_X)
588: pmap_tlb_shootpage(pm, va);
589:
590: /*
591: * Executability was removed on the last executable change.
592: * Reset the code segment to something conservative and
593: * let the trap handler deal with setting the right limit.
594: * We can't do that because of locking constraints on the vm map.
595: *
596: * XXX - floating cs - set this _really_ low.
597: */
598: if ((opte & PG_X) && (npte & PG_X) == 0 && va == pm->pm_hiexec) {
599: struct trapframe *tf = curproc->p_md.md_regs;
600: struct pcb *pcb = &curproc->p_addr->u_pcb;
601:
602: pm->pm_hiexec = I386_MAX_EXE_ADDR;
603: setcslimit(pm, tf, pcb, I386_MAX_EXE_ADDR);
604: }
605: }
606:
607: /*
608: * Fixup the code segment to cover all potential executable mappings.
609: * Called by kernel SEGV trap handler.
610: * returns 0 if no changes to the code segment were made.
611: */
612: int
613: pmap_exec_fixup(struct vm_map *map, struct trapframe *tf, struct pcb *pcb)
614: {
615: struct vm_map_entry *ent;
616: struct pmap *pm = vm_map_pmap(map);
617: vaddr_t va = 0;
618:
619: vm_map_lock(map);
620: for (ent = (&map->header)->next; ent != &map->header; ent = ent->next) {
621: /*
622: * This entry has greater va than the entries before.
623: * We need to make it point to the last page, not past it.
624: */
625: if (ent->protection & VM_PROT_EXECUTE)
626: va = trunc_page(ent->end - 1);
627: }
628: vm_map_unlock(map);
629:
630: if (va <= pm->pm_hiexec) {
631: return (0);
632: }
633:
634: pm->pm_hiexec = va;
635:
636: /*
637: * We have a new 'highest executable' va, so we need to update
638: * the value for the code segment limit, which is stored in the
639: * PCB.
640: */
641: setcslimit(pm, tf, pcb, va);
642:
643: return (1);
644: }
645:
646: void
647: setcslimit(struct pmap *pm, struct trapframe *tf, struct pcb *pcb,
648: vaddr_t limit)
649: {
650: /*
651: * Called when we have a new 'highest executable' va, so we need
652: * to update the value for the code segment limit, which is stored
653: * in the PCB.
654: *
655: * There are no caching issues to be concerned with: the
656: * processor reads the whole descriptor from the GDT when the
657: * appropriate selector is loaded into a segment register, and
658: * this only happens on the return to userland.
659: *
660: * This also works in the MP case, since whichever CPU gets to
661: * run the process will pick up the right descriptor value from
662: * the PCB.
663: */
664: limit = min(limit, VM_MAXUSER_ADDRESS - 1);
665:
666: setsegment(&pm->pm_codeseg, 0, atop(limit),
667: SDT_MEMERA, SEL_UPL, 1, 1);
668:
669: /* And update the GDT and LDT since we may be called by the
670: * trap handler (cpu_switch won't get a chance).
671: */
672: curcpu()->ci_gdt[GUCODE_SEL].sd = pcb->pcb_ldt[LUCODE_SEL].sd =
673: pm->pm_codeseg;
674:
675: pcb->pcb_cs = tf->tf_cs = GSEL(GUCODE_SEL, SEL_UPL);
676: }
677:
678: /*
679: * p m a p k e n t e r f u n c t i o n s
680: *
681: * functions to quickly enter/remove pages from the kernel address
682: * space. pmap_kremove is exported to MI kernel. we make use of
683: * the recursive PTE mappings.
684: */
685:
686: /*
687: * pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking
688: *
689: * => no need to lock anything, assume va is already allocated
690: * => should be faster than normal pmap enter function
691: */
692:
693: void
694: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
695: {
696: pt_entry_t *pte, opte, npte;
697:
698: pte = vtopte(va);
699: npte = pa | ((prot & VM_PROT_WRITE)? PG_RW : PG_RO) | PG_V |
700: pmap_pg_g | PG_U | PG_M;
701: opte = i386_atomic_testset_ul(pte, npte); /* zap! */
702: if (pmap_valid_entry(opte)) {
703: /* NB. - this should not happen. */
704: pmap_tlb_shootpage(pmap_kernel(), va);
705: pmap_tlb_shootwait();
706: }
707: }
708:
709: /*
710: * pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking
711: *
712: * => no need to lock anything
713: * => caller must dispose of any vm_page mapped in the va range
714: * => note: not an inline function
715: * => we assume the va is page aligned and the len is a multiple of PAGE_SIZE
716: */
717:
718: void
719: pmap_kremove(vaddr_t sva, vsize_t len)
720: {
721: pt_entry_t *pte, opte;
722: vaddr_t va, eva;
723:
724: eva = sva + len;
725:
726: for (va = sva; va != eva; va += PAGE_SIZE) {
727: pte = kvtopte(va);
728: opte = i386_atomic_testset_ul(pte, 0);
729: #ifdef DIAGNOSTIC
730: if (opte & PG_PVLIST)
731: panic("pmap_kremove: PG_PVLIST mapping for 0x%lx", va);
732: #endif
733: }
734: pmap_tlb_shootrange(pmap_kernel(), sva, eva);
735: pmap_tlb_shootwait();
736: }
737:
738: /*
739: * p m a p i n i t f u n c t i o n s
740: *
741: * pmap_bootstrap and pmap_init are called during system startup
742: * to init the pmap module. pmap_bootstrap() does a low level
743: * init just to get things rolling. pmap_init() finishes the job.
744: */
745:
746: /*
747: * pmap_bootstrap: get the system in a state where it can run with VM
748: * properly enabled (called before main()). the VM system is
749: * fully init'd later...
750: *
751: * => on i386, locore.s has already enabled the MMU by allocating
752: * a PDP for the kernel, and nkpde PTP's for the kernel.
753: * => kva_start is the first free virtual address in kernel space
754: */
755:
756: void
757: pmap_bootstrap(vaddr_t kva_start)
758: {
759: extern paddr_t avail_end;
760: struct pmap *kpm;
761: vaddr_t kva;
762: pt_entry_t *pte;
763:
764: /*
765: * set the page size (default value is 4K which is ok)
766: */
767:
768: uvm_setpagesize();
769:
770: /*
771: * a quick sanity check
772: */
773:
774: if (PAGE_SIZE != NBPG)
775: panic("pmap_bootstrap: PAGE_SIZE != NBPG");
776:
777: /*
778: * use the very last page of physical memory for the message buffer
779: */
780:
781: avail_end -= round_page(MSGBUFSIZE);
782: /*
783: * The arguments passed in from /boot needs space too.
784: */
785: avail_end -= round_page(bootargc);
786:
787: /*
788: * set up our local static global vars that keep track of the
789: * usage of KVM before kernel_map is set up
790: */
791:
792: virtual_avail = kva_start; /* first free KVA */
793: virtual_end = VM_MAX_KERNEL_ADDRESS; /* last KVA */
794:
795: /*
796: * set up protection_codes: we need to be able to convert from
797: * a MI protection code (some combo of VM_PROT...) to something
798: * we can jam into a i386 PTE.
799: */
800:
801: protection_codes[UVM_PROT_NONE] = 0; /* --- */
802: protection_codes[UVM_PROT_EXEC] = PG_X; /* --x */
803: protection_codes[UVM_PROT_READ] = PG_RO; /* -r- */
804: protection_codes[UVM_PROT_RX] = PG_X; /* -rx */
805: protection_codes[UVM_PROT_WRITE] = PG_RW; /* w-- */
806: protection_codes[UVM_PROT_WX] = PG_RW|PG_X; /* w-x */
807: protection_codes[UVM_PROT_RW] = PG_RW; /* wr- */
808: protection_codes[UVM_PROT_RWX] = PG_RW|PG_X; /* wrx */
809:
810: /*
811: * now we init the kernel's pmap
812: *
813: * the kernel pmap's pm_obj is not used for much. however, in
814: * user pmaps the pm_obj contains the list of active PTPs.
815: * the pm_obj currently does not have a pager. it might be possible
816: * to add a pager that would allow a process to read-only mmap its
817: * own page tables (fast user level vtophys?). this may or may not
818: * be useful.
819: */
820:
821: kpm = pmap_kernel();
822: simple_lock_init(&kpm->pm_obj.vmobjlock);
823: kpm->pm_obj.pgops = NULL;
824: TAILQ_INIT(&kpm->pm_obj.memq);
825: kpm->pm_obj.uo_npages = 0;
826: kpm->pm_obj.uo_refs = 1;
827: bzero(&kpm->pm_list, sizeof(kpm->pm_list)); /* pm_list not used */
828: kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE);
829: kpm->pm_pdirpa = (u_int32_t) proc0.p_addr->u_pcb.pcb_cr3;
830: kpm->pm_stats.wired_count = kpm->pm_stats.resident_count =
831: atop(kva_start - VM_MIN_KERNEL_ADDRESS);
832:
833: /*
834: * the above is just a rough estimate and not critical to the proper
835: * operation of the system.
836: */
837:
838: /*
839: * enable global TLB entries if they are supported
840: */
841:
842: if (cpu_feature & CPUID_PGE) {
843: lcr4(rcr4() | CR4_PGE); /* enable hardware (via %cr4) */
844: pmap_pg_g = PG_G; /* enable software */
845:
846: /* add PG_G attribute to already mapped kernel pages */
847: for (kva = VM_MIN_KERNEL_ADDRESS ; kva < virtual_avail ;
848: kva += PAGE_SIZE)
849: if (pmap_valid_entry(PTE_BASE[atop(kva)]))
850: PTE_BASE[atop(kva)] |= PG_G;
851: }
852:
853: /*
854: * now we allocate the "special" VAs which are used for tmp mappings
855: * by the pmap (and other modules). we allocate the VAs by advancing
856: * virtual_avail (note that there are no pages mapped at these VAs).
857: * we find the PTE that maps the allocated VA via the linear PTE
858: * mapping.
859: */
860:
861: pte = PTE_BASE + atop(virtual_avail);
862:
863: #ifdef MULTIPROCESSOR
864: /*
865: * Waste some VA space to avoid false sharing of cache lines
866: * for page table pages: Give each possible CPU a cache line
867: * of PTE's (8) to play with, though we only need 4. We could
868: * recycle some of this waste by putting the idle stacks here
869: * as well; we could waste less space if we knew the largest
870: * CPU ID beforehand.
871: */
872: csrcp = (caddr_t) virtual_avail; csrc_pte = pte;
873:
874: cdstp = (caddr_t) virtual_avail+PAGE_SIZE; cdst_pte = pte+1;
875:
876: zerop = (caddr_t) virtual_avail+PAGE_SIZE*2; zero_pte = pte+2;
877:
878: ptpp = (caddr_t) virtual_avail+PAGE_SIZE*3; ptp_pte = pte+3;
879:
880: virtual_avail += PAGE_SIZE * I386_MAXPROCS * NPTECL;
881: pte += I386_MAXPROCS * NPTECL;
882: #else
883: csrcp = (caddr_t) virtual_avail; csrc_pte = pte; /* allocate */
884: virtual_avail += PAGE_SIZE; pte++; /* advance */
885:
886: cdstp = (caddr_t) virtual_avail; cdst_pte = pte;
887: virtual_avail += PAGE_SIZE; pte++;
888:
889: zerop = (caddr_t) virtual_avail; zero_pte = pte;
890: virtual_avail += PAGE_SIZE; pte++;
891:
892: ptpp = (caddr_t) virtual_avail; ptp_pte = pte;
893: virtual_avail += PAGE_SIZE; pte++;
894: #endif
895:
896: /* XXX: vmmap used by mem.c... should be uvm_map_reserve */
897: vmmap = (char *)virtual_avail; /* don't need pte */
898: virtual_avail += PAGE_SIZE;
899:
900: msgbufp = (struct msgbuf *)virtual_avail; /* don't need pte */
901: virtual_avail += round_page(MSGBUFSIZE); pte++;
902:
903: bootargp = (bootarg_t *)virtual_avail;
904: virtual_avail += round_page(bootargc); pte++;
905:
906: /*
907: * now we reserve some VM for mapping pages when doing a crash dump
908: */
909:
910: virtual_avail = reserve_dumppages(virtual_avail);
911:
912: /*
913: * init the static-global locks and global lists.
914: */
915:
916: simple_lock_init(&pvalloc_lock);
917: simple_lock_init(&pmaps_lock);
918: LIST_INIT(&pmaps);
919: TAILQ_INIT(&pv_freepages);
920: TAILQ_INIT(&pv_unusedpgs);
921:
922: /*
923: * initialize the pmap pool.
924: */
925:
926: pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
927: &pool_allocator_nointr);
928:
929: /*
930: * ensure the TLB is sync'd with reality by flushing it...
931: */
932:
933: tlbflush();
934: }
935:
936: /*
937: * pmap_init: called from uvm_init, our job is to get the pmap
938: * system ready to manage mappings... this mainly means initing
939: * the pv_entry stuff.
940: */
941:
942: void
943: pmap_init(void)
944: {
945: /*
946: * now we need to free enough pv_entry structures to allow us to get
947: * the kmem_map allocated and inited (done after this function is
948: * finished). to do this we allocate one bootstrap page out of
949: * kernel_map and use it to provide an initial pool of pv_entry
950: * structures. we never free this page.
951: */
952:
953: pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE);
954: if (pv_initpage == NULL)
955: panic("pmap_init: pv_initpage");
956: pv_cachedva = 0; /* a VA we have allocated but not used yet */
957: pv_nfpvents = 0;
958: (void) pmap_add_pvpage(pv_initpage, FALSE);
959:
960: /*
961: * done: pmap module is up (and ready for business)
962: */
963:
964: pmap_initialized = TRUE;
965: }
966:
967: /*
968: * p v _ e n t r y f u n c t i o n s
969: */
970:
971: /*
972: * pv_entry allocation functions:
973: * the main pv_entry allocation functions are:
974: * pmap_alloc_pv: allocate a pv_entry structure
975: * pmap_free_pv: free one pv_entry
976: * pmap_free_pvs: free a list of pv_entrys
977: *
978: * the rest are helper functions
979: */
980:
981: /*
982: * pmap_alloc_pv: inline function to allocate a pv_entry structure
983: * => we lock pvalloc_lock
984: * => if we fail, we call out to pmap_alloc_pvpage
985: * => 3 modes:
986: * ALLOCPV_NEED = we really need a pv_entry
987: * ALLOCPV_TRY = we want a pv_entry
988: * ALLOCPV_NONEED = we are trying to grow our free list, don't really need
989: * one now
990: *
991: * "try" is for optional functions like pmap_copy().
992: */
993:
994: struct pv_entry *
995: pmap_alloc_pv(struct pmap *pmap, int mode)
996: {
997: struct pv_page *pvpage;
998: struct pv_entry *pv;
999:
1000: simple_lock(&pvalloc_lock);
1001:
1002: if (!TAILQ_EMPTY(&pv_freepages)) {
1003: pvpage = TAILQ_FIRST(&pv_freepages);
1004: pvpage->pvinfo.pvpi_nfree--;
1005: if (pvpage->pvinfo.pvpi_nfree == 0) {
1006: /* nothing left in this one? */
1007: TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list);
1008: }
1009: pv = pvpage->pvinfo.pvpi_pvfree;
1010: #ifdef DIAGNOSTIC
1011: if (pv == NULL)
1012: panic("pmap_alloc_pv: pvpi_nfree off");
1013: #endif
1014: pvpage->pvinfo.pvpi_pvfree = pv->pv_next;
1015: pv_nfpvents--; /* took one from pool */
1016: } else {
1017: pv = NULL; /* need more of them */
1018: }
1019:
1020: /*
1021: * if below low water mark or we didn't get a pv_entry we try and
1022: * create more pv_entrys ...
1023: */
1024:
1025: if (pv_nfpvents < PVE_LOWAT || pv == NULL) {
1026: if (pv == NULL)
1027: pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ?
1028: mode : ALLOCPV_NEED);
1029: else
1030: (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED);
1031: }
1032:
1033: simple_unlock(&pvalloc_lock);
1034: return(pv);
1035: }
1036:
1037: /*
1038: * pmap_alloc_pvpage: maybe allocate a new pvpage
1039: *
1040: * if need_entry is false: try and allocate a new pv_page
1041: * if need_entry is true: try and allocate a new pv_page and return a
1042: * new pv_entry from it.
1043: *
1044: * => we assume that the caller holds pvalloc_lock
1045: */
1046:
1047: struct pv_entry *
1048: pmap_alloc_pvpage(struct pmap *pmap, int mode)
1049: {
1050: struct vm_page *pg;
1051: struct pv_page *pvpage;
1052: struct pv_entry *pv;
1053: int s;
1054:
1055: /*
1056: * if we need_entry and we've got unused pv_pages, allocate from there
1057: */
1058:
1059: if (mode != ALLOCPV_NONEED && !TAILQ_EMPTY(&pv_unusedpgs)) {
1060:
1061: /* move it to pv_freepages list */
1062: pvpage = TAILQ_FIRST(&pv_unusedpgs);
1063: TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list);
1064: TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list);
1065:
1066: /* allocate a pv_entry */
1067: pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */
1068: pv = pvpage->pvinfo.pvpi_pvfree;
1069: #ifdef DIAGNOSTIC
1070: if (pv == NULL)
1071: panic("pmap_alloc_pvpage: pvpi_nfree off");
1072: #endif
1073: pvpage->pvinfo.pvpi_pvfree = pv->pv_next;
1074:
1075: pv_nfpvents--; /* took one from pool */
1076: return(pv);
1077: }
1078:
1079: /*
1080: * see if we've got a cached unmapped VA that we can map a page in.
1081: * if not, try to allocate one.
1082: */
1083:
1084: s = splvm(); /* must protect kmem_map with splvm! */
1085: if (pv_cachedva == 0) {
1086: pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL,
1087: NBPG, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC);
1088: }
1089: splx(s);
1090: if (pv_cachedva == 0)
1091: return (NULL);
1092:
1093: pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
1094: if (pg == NULL)
1095: return (NULL);
1096:
1097: atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
1098:
1099: /*
1100: * add a mapping for our new pv_page and free its entries (save one!)
1101: *
1102: * NOTE: If we are allocating a PV page for the kernel pmap, the
1103: * pmap is already locked! (...but entering the mapping is safe...)
1104: */
1105:
1106: pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg),
1107: VM_PROT_READ|VM_PROT_WRITE);
1108: pvpage = (struct pv_page *) pv_cachedva;
1109: pv_cachedva = 0;
1110: return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED));
1111: }
1112:
1113: /*
1114: * pmap_add_pvpage: add a pv_page's pv_entrys to the free list
1115: *
1116: * => caller must hold pvalloc_lock
1117: * => if need_entry is true, we allocate and return one pv_entry
1118: */
1119:
1120: struct pv_entry *
1121: pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry)
1122: {
1123: int tofree, lcv;
1124:
1125: /* do we need to return one? */
1126: tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE;
1127:
1128: pvp->pvinfo.pvpi_pvfree = NULL;
1129: pvp->pvinfo.pvpi_nfree = tofree;
1130: for (lcv = 0 ; lcv < tofree ; lcv++) {
1131: pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree;
1132: pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv];
1133: }
1134: if (need_entry)
1135: TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list);
1136: else
1137: TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list);
1138: pv_nfpvents += tofree;
1139: return((need_entry) ? &pvp->pvents[lcv] : NULL);
1140: }
1141:
1142: /*
1143: * pmap_free_pv_doit: actually free a pv_entry
1144: *
1145: * => do not call this directly! instead use either
1146: * 1. pmap_free_pv ==> free a single pv_entry
1147: * 2. pmap_free_pvs => free a list of pv_entrys
1148: * => we must be holding pvalloc_lock
1149: */
1150:
1151: void
1152: pmap_free_pv_doit(struct pv_entry *pv)
1153: {
1154: struct pv_page *pvp;
1155:
1156: pvp = (struct pv_page*)trunc_page((vaddr_t)pv);
1157: pv_nfpvents++;
1158: pvp->pvinfo.pvpi_nfree++;
1159:
1160: /* nfree == 1 => fully allocated page just became partly allocated */
1161: if (pvp->pvinfo.pvpi_nfree == 1) {
1162: TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list);
1163: }
1164:
1165: /* free it */
1166: pv->pv_next = pvp->pvinfo.pvpi_pvfree;
1167: pvp->pvinfo.pvpi_pvfree = pv;
1168:
1169: /*
1170: * are all pv_page's pv_entry's free? move it to unused queue.
1171: */
1172:
1173: if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) {
1174: TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list);
1175: TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list);
1176: }
1177: }
1178:
1179: /*
1180: * pmap_free_pv: free a single pv_entry
1181: *
1182: * => we gain the pvalloc_lock
1183: */
1184:
1185: void
1186: pmap_free_pv(struct pmap *pmap, struct pv_entry *pv)
1187: {
1188: simple_lock(&pvalloc_lock);
1189: pmap_free_pv_doit(pv);
1190:
1191: /*
1192: * Can't free the PV page if the PV entries were associated with
1193: * the kernel pmap; the pmap is already locked.
1194: */
1195: if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL &&
1196: pmap != pmap_kernel())
1197: pmap_free_pvpage();
1198:
1199: simple_unlock(&pvalloc_lock);
1200: }
1201:
1202: /*
1203: * pmap_free_pvs: free a list of pv_entrys
1204: *
1205: * => we gain the pvalloc_lock
1206: */
1207:
1208: void
1209: pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs)
1210: {
1211: struct pv_entry *nextpv;
1212:
1213: simple_lock(&pvalloc_lock);
1214:
1215: for ( /* null */ ; pvs != NULL ; pvs = nextpv) {
1216: nextpv = pvs->pv_next;
1217: pmap_free_pv_doit(pvs);
1218: }
1219:
1220: /*
1221: * Can't free the PV page if the PV entries were associated with
1222: * the kernel pmap; the pmap is already locked.
1223: */
1224: if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL &&
1225: pmap != pmap_kernel())
1226: pmap_free_pvpage();
1227:
1228: simple_unlock(&pvalloc_lock);
1229: }
1230:
1231:
1232: /*
1233: * pmap_free_pvpage: try and free an unused pv_page structure
1234: *
1235: * => assume caller is holding the pvalloc_lock and that
1236: * there is a page on the pv_unusedpgs list
1237: * => if we can't get a lock on the kmem_map we try again later
1238: */
1239:
1240: void
1241: pmap_free_pvpage(void)
1242: {
1243: int s;
1244: struct vm_map *map;
1245: struct vm_map_entry *dead_entries;
1246: struct pv_page *pvp;
1247:
1248: s = splvm(); /* protect kmem_map */
1249: pvp = TAILQ_FIRST(&pv_unusedpgs);
1250:
1251: /*
1252: * note: watch out for pv_initpage which is allocated out of
1253: * kernel_map rather than kmem_map.
1254: */
1255:
1256: if (pvp == pv_initpage)
1257: map = kernel_map;
1258: else
1259: map = kmem_map;
1260: if (vm_map_lock_try(map)) {
1261:
1262: /* remove pvp from pv_unusedpgs */
1263: TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list);
1264:
1265: /* unmap the page */
1266: dead_entries = NULL;
1267: uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE,
1268: &dead_entries, NULL);
1269: vm_map_unlock(map);
1270:
1271: if (dead_entries != NULL)
1272: uvm_unmap_detach(dead_entries, 0);
1273:
1274: pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */
1275: }
1276:
1277: if (pvp == pv_initpage)
1278: /* no more initpage, we've freed it */
1279: pv_initpage = NULL;
1280:
1281: splx(s);
1282: }
1283:
1284: /*
1285: * main pv_entry manipulation functions:
1286: * pmap_enter_pv: enter a mapping onto a pv list
1287: * pmap_remove_pv: remove a mappiing from a pv list
1288: */
1289:
1290: /*
1291: * pmap_enter_pv: enter a mapping onto a pv list
1292: *
1293: * => caller should have pmap locked
1294: * => we will gain the lock on the pv and allocate the new pv_entry
1295: * => caller should adjust ptp's wire_count before calling
1296: *
1297: * pve: preallocated pve for us to use
1298: * ptp: PTP in pmap that maps this VA
1299: */
1300:
1301: void
1302: pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap,
1303: vaddr_t va, struct vm_page *ptp)
1304: {
1305: pve->pv_pmap = pmap;
1306: pve->pv_va = va;
1307: pve->pv_ptp = ptp; /* NULL for kernel pmap */
1308: pve->pv_next = pg->mdpage.pv_list; /* add to ... */
1309: pg->mdpage.pv_list = pve; /* ... locked list */
1310: }
1311:
1312: /*
1313: * pmap_remove_pv: try to remove a mapping from a pv_list
1314: *
1315: * => pmap should be locked
1316: * => caller should hold lock on pv [so that attrs can be adjusted]
1317: * => caller should adjust ptp's wire_count and free PTP if needed
1318: * => we return the removed pve
1319: */
1320:
1321: struct pv_entry *
1322: pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va)
1323: {
1324: struct pv_entry *pve, **prevptr;
1325:
1326: prevptr = &pg->mdpage.pv_list; /* previous pv_entry pointer */
1327: while ((pve = *prevptr) != NULL) {
1328: if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */
1329: *prevptr = pve->pv_next; /* remove it! */
1330: break;
1331: }
1332: prevptr = &pve->pv_next; /* previous pointer */
1333: }
1334: return(pve); /* return removed pve */
1335: }
1336:
1337: /*
1338: * p t p f u n c t i o n s
1339: */
1340:
1341: /*
1342: * pmap_alloc_ptp: allocate a PTP for a PMAP
1343: *
1344: * => pmap should already be locked by caller
1345: * => we use the ptp's wire_count to count the number of active mappings
1346: * in the PTP (we start it at one to prevent any chance this PTP
1347: * will ever leak onto the active/inactive queues)
1348: * => we may need to lock pv lists if we have to steal a PTP
1349: * => just_try: true if we want a PTP, but not enough to steal one
1350: * from another pmap (e.g. during optional functions like pmap_copy)
1351: */
1352:
1353: struct vm_page *
1354: pmap_alloc_ptp(struct pmap *pmap, int pde_index, boolean_t just_try,
1355: pt_entry_t pde_flags)
1356: {
1357: struct vm_page *ptp;
1358:
1359: ptp = uvm_pagealloc(&pmap->pm_obj, ptp_i2o(pde_index), NULL,
1360: UVM_PGA_USERESERVE|UVM_PGA_ZERO);
1361: if (ptp == NULL)
1362: return (NULL);
1363:
1364: /* got one! */
1365: atomic_clearbits_int(&ptp->pg_flags, PG_BUSY);
1366: ptp->wire_count = 1; /* no mappings yet */
1367: pmap->pm_pdir[pde_index] = (pd_entry_t)(VM_PAGE_TO_PHYS(ptp) |
1368: PG_RW | PG_V | PG_M | PG_U | pde_flags);
1369: pmap->pm_stats.resident_count++; /* count PTP as resident */
1370: pmap->pm_ptphint = ptp;
1371: return(ptp);
1372: }
1373:
1374: /*
1375: * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one)
1376: *
1377: * => pmap should NOT be pmap_kernel()
1378: * => pmap should be locked
1379: */
1380:
1381: struct vm_page *
1382: pmap_get_ptp(struct pmap *pmap, int pde_index, boolean_t just_try)
1383: {
1384: struct vm_page *ptp;
1385:
1386: if (pmap_valid_entry(pmap->pm_pdir[pde_index])) {
1387:
1388: /* valid... check hint (saves us a PA->PG lookup) */
1389: if (pmap->pm_ptphint &&
1390: (pmap->pm_pdir[pde_index] & PG_FRAME) ==
1391: VM_PAGE_TO_PHYS(pmap->pm_ptphint))
1392: return(pmap->pm_ptphint);
1393:
1394: ptp = uvm_pagelookup(&pmap->pm_obj, ptp_i2o(pde_index));
1395: #ifdef DIAGNOSTIC
1396: if (ptp == NULL)
1397: panic("pmap_get_ptp: unmanaged user PTP");
1398: #endif
1399: pmap->pm_ptphint = ptp;
1400: return(ptp);
1401: }
1402:
1403: /* allocate a new PTP (updates ptphint) */
1404: return (pmap_alloc_ptp(pmap, pde_index, just_try, PG_u));
1405: }
1406:
1407: /*
1408: * p m a p l i f e c y c l e f u n c t i o n s
1409: */
1410:
1411: /*
1412: * pmap_create: create a pmap
1413: *
1414: * => note: old pmap interface took a "size" args which allowed for
1415: * the creation of "software only" pmaps (not in bsd).
1416: */
1417:
1418: struct pmap *
1419: pmap_create(void)
1420: {
1421: struct pmap *pmap;
1422:
1423: pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
1424: pmap_pinit(pmap);
1425: return(pmap);
1426: }
1427:
1428: /*
1429: * pmap_pinit: given a zero'd pmap structure, init it.
1430: */
1431:
1432: void
1433: pmap_pinit(struct pmap *pmap)
1434: {
1435: /* init uvm_object */
1436: simple_lock_init(&pmap->pm_obj.vmobjlock);
1437: pmap->pm_obj.pgops = NULL; /* currently not a mappable object */
1438: TAILQ_INIT(&pmap->pm_obj.memq);
1439: pmap->pm_obj.uo_npages = 0;
1440: pmap->pm_obj.uo_refs = 1;
1441: pmap->pm_stats.wired_count = 0;
1442: pmap->pm_stats.resident_count = 1; /* count the PDP allocd below */
1443: pmap->pm_ptphint = NULL;
1444: pmap->pm_hiexec = 0;
1445: pmap->pm_flags = 0;
1446: pmap->pm_cpus = 0;
1447:
1448: setsegment(&pmap->pm_codeseg, 0, atop(I386_MAX_EXE_ADDR) - 1,
1449: SDT_MEMERA, SEL_UPL, 1, 1);
1450:
1451: /* allocate PDP */
1452: pmap->pm_pdir = (pd_entry_t *) uvm_km_alloc(kernel_map, NBPG);
1453: if (pmap->pm_pdir == NULL)
1454: panic("pmap_pinit: kernel_map out of virtual space!");
1455: (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir,
1456: (paddr_t *)&pmap->pm_pdirpa);
1457:
1458: /* init PDP */
1459: /* zero init area */
1460: bzero(pmap->pm_pdir, PDSLOT_PTE * sizeof(pd_entry_t));
1461: /* put in recursive PDE to map the PTEs */
1462: pmap->pm_pdir[PDSLOT_PTE] = pmap->pm_pdirpa | PG_V | PG_KW | PG_U |
1463: PG_M;
1464:
1465: /* init the LDT */
1466: pmap->pm_ldt = NULL;
1467: pmap->pm_ldt_len = 0;
1468: pmap->pm_ldt_sel = GSEL(GLDT_SEL, SEL_KPL);
1469:
1470: /*
1471: * we need to lock pmaps_lock to prevent nkpde from changing on
1472: * us. note that there is no need to splvm to protect us from
1473: * malloc since malloc allocates out of a submap and we should have
1474: * already allocated kernel PTPs to cover the range...
1475: */
1476: simple_lock(&pmaps_lock);
1477: /* put in kernel VM PDEs */
1478: bcopy(&PDP_BASE[PDSLOT_KERN], &pmap->pm_pdir[PDSLOT_KERN],
1479: nkpde * sizeof(pd_entry_t));
1480: /* zero the rest */
1481: bzero(&pmap->pm_pdir[PDSLOT_KERN + nkpde],
1482: NBPG - ((PDSLOT_KERN + nkpde) * sizeof(pd_entry_t)));
1483: LIST_INSERT_HEAD(&pmaps, pmap, pm_list);
1484: simple_unlock(&pmaps_lock);
1485: }
1486:
1487: /*
1488: * pmap_destroy: drop reference count on pmap. free pmap if
1489: * reference count goes to zero.
1490: */
1491:
1492: void
1493: pmap_destroy(struct pmap *pmap)
1494: {
1495: int refs;
1496:
1497: /*
1498: * drop reference count
1499: */
1500:
1501: simple_lock(&pmap->pm_obj.vmobjlock);
1502: refs = --pmap->pm_obj.uo_refs;
1503: simple_unlock(&pmap->pm_obj.vmobjlock);
1504: if (refs > 0)
1505: return;
1506:
1507: /*
1508: * reference count is zero, free pmap resources and then free pmap.
1509: */
1510:
1511: pmap_release(pmap);
1512: pool_put(&pmap_pmap_pool, pmap);
1513: }
1514:
1515: /*
1516: * pmap_release: release all resources held by a pmap
1517: *
1518: * => if pmap is still referenced it should be locked
1519: * => XXX: we currently don't expect any busy PTPs because we don't
1520: * allow anything to map them (except for the kernel's private
1521: * recursive mapping) or make them busy.
1522: */
1523:
1524: void
1525: pmap_release(struct pmap *pmap)
1526: {
1527: struct vm_page *pg;
1528:
1529: /*
1530: * remove it from global list of pmaps
1531: */
1532:
1533: simple_lock(&pmaps_lock);
1534: LIST_REMOVE(pmap, pm_list);
1535: simple_unlock(&pmaps_lock);
1536:
1537: /*
1538: * Before we free the pmap just make sure it's not cached anywhere.
1539: */
1540: tlbflushg();
1541:
1542: /*
1543: * free any remaining PTPs
1544: */
1545:
1546: while (!TAILQ_EMPTY(&pmap->pm_obj.memq)) {
1547: pg = TAILQ_FIRST(&pmap->pm_obj.memq);
1548: #ifdef DIAGNOSTIC
1549: if (pg->pg_flags & PG_BUSY)
1550: panic("pmap_release: busy page table page");
1551: #endif
1552: /* pmap_page_protect? currently no need for it. */
1553:
1554: pg->wire_count = 0;
1555: uvm_pagefree(pg);
1556: }
1557:
1558: /*
1559: * MULTIPROCESSOR -- no need to flush out of other processors'
1560: * APTE space because we do that in pmap_unmap_ptes().
1561: */
1562: uvm_km_free(kernel_map, (vaddr_t)pmap->pm_pdir, NBPG);
1563:
1564: #ifdef USER_LDT
1565: if (pmap->pm_flags & PMF_USER_LDT) {
1566: /*
1567: * no need to switch the LDT; this address space is gone,
1568: * nothing is using it.
1569: *
1570: * No need to lock the pmap for ldt_free (or anything else),
1571: * we're the last one to use it.
1572: */
1573: ldt_free(pmap);
1574: uvm_km_free(kernel_map, (vaddr_t)pmap->pm_ldt,
1575: pmap->pm_ldt_len * sizeof(union descriptor));
1576: }
1577: #endif
1578: }
1579:
1580: /*
1581: * Add a reference to the specified pmap.
1582: */
1583:
1584: void
1585: pmap_reference(struct pmap *pmap)
1586: {
1587: simple_lock(&pmap->pm_obj.vmobjlock);
1588: pmap->pm_obj.uo_refs++;
1589: simple_unlock(&pmap->pm_obj.vmobjlock);
1590: }
1591:
1592: #if defined(PMAP_FORK)
1593: /*
1594: * pmap_fork: perform any necessary data structure manipulation when
1595: * a VM space is forked.
1596: */
1597:
1598: void
1599: pmap_fork(struct pmap *pmap1, struct pmap *pmap2)
1600: {
1601: simple_lock(&pmap1->pm_obj.vmobjlock);
1602: simple_lock(&pmap2->pm_obj.vmobjlock);
1603:
1604: #ifdef USER_LDT
1605: /* Copy the LDT, if necessary. */
1606: if (pmap1->pm_flags & PMF_USER_LDT) {
1607: union descriptor *new_ldt;
1608: size_t len;
1609:
1610: len = pmap1->pm_ldt_len * sizeof(union descriptor);
1611: new_ldt = (union descriptor *)uvm_km_alloc(kernel_map, len);
1612: bcopy(pmap1->pm_ldt, new_ldt, len);
1613: pmap2->pm_ldt = new_ldt;
1614: pmap2->pm_ldt_len = pmap1->pm_ldt_len;
1615: pmap2->pm_flags |= PMF_USER_LDT;
1616: ldt_alloc(pmap2, new_ldt, len);
1617: }
1618: #endif /* USER_LDT */
1619:
1620: simple_unlock(&pmap2->pm_obj.vmobjlock);
1621: simple_unlock(&pmap1->pm_obj.vmobjlock);
1622: }
1623: #endif /* PMAP_FORK */
1624:
1625: #ifdef USER_LDT
1626: /*
1627: * pmap_ldt_cleanup: if the pmap has a local LDT, deallocate it, and
1628: * restore the default.
1629: */
1630:
1631: void
1632: pmap_ldt_cleanup(struct proc *p)
1633: {
1634: struct pcb *pcb = &p->p_addr->u_pcb;
1635: pmap_t pmap = p->p_vmspace->vm_map.pmap;
1636: union descriptor *old_ldt = NULL;
1637: size_t len = 0;
1638:
1639: simple_lock(&pmap->pm_obj.vmobjlock);
1640:
1641: if (pmap->pm_flags & PMF_USER_LDT) {
1642: ldt_free(pmap);
1643: pmap->pm_ldt_sel = GSEL(GLDT_SEL, SEL_KPL);
1644: pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
1645: /* Reset the cached address of the LDT that this process uses */
1646: #ifdef MULTIPROCESSOR
1647: pcb->pcb_ldt = curcpu()->ci_ldt;
1648: #else
1649: pcb->pcb_ldt = ldt;
1650: #endif
1651: if (pcb == curpcb)
1652: lldt(pcb->pcb_ldt_sel);
1653: old_ldt = pmap->pm_ldt;
1654: len = pmap->pm_ldt_len * sizeof(union descriptor);
1655: pmap->pm_ldt = NULL;
1656: pmap->pm_ldt_len = 0;
1657: pmap->pm_flags &= ~PMF_USER_LDT;
1658: }
1659:
1660: simple_unlock(&pmap->pm_obj.vmobjlock);
1661:
1662: if (old_ldt != NULL)
1663: uvm_km_free(kernel_map, (vaddr_t)old_ldt, len);
1664: }
1665: #endif /* USER_LDT */
1666:
1667: /*
1668: * pmap_activate: activate a process' pmap (fill in %cr3 and LDT info)
1669: *
1670: * => called from cpu_switch()
1671: * => if proc is the curproc, then load it into the MMU
1672: */
1673:
1674: void
1675: pmap_activate(struct proc *p)
1676: {
1677: struct pcb *pcb = &p->p_addr->u_pcb;
1678: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
1679: struct cpu_info *self = curcpu();
1680:
1681: pcb->pcb_pmap = pmap;
1682: /* Get the LDT that this process will actually use */
1683: #ifdef MULTIPROCESSOR
1684: pcb->pcb_ldt = pmap->pm_ldt == NULL ? self->ci_ldt : pmap->pm_ldt;
1685: #else
1686: pcb->pcb_ldt = pmap->pm_ldt == NULL ? ldt : pmap->pm_ldt;
1687: #endif
1688: pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
1689: pcb->pcb_cr3 = pmap->pm_pdirpa;
1690: if (p == curproc) {
1691: /*
1692: * Set the correct descriptor value (i.e. with the
1693: * correct code segment X limit) in the GDT and the LDT.
1694: */
1695: self->ci_gdt[GUCODE_SEL].sd = pcb->pcb_ldt[LUCODE_SEL].sd =
1696: pmap->pm_codeseg;
1697:
1698: lcr3(pcb->pcb_cr3);
1699: lldt(pcb->pcb_ldt_sel);
1700:
1701: /*
1702: * mark the pmap in use by this processor.
1703: */
1704: i386_atomic_setbits_l(&pmap->pm_cpus, (1U << cpu_number()));
1705: }
1706: }
1707:
1708: /*
1709: * pmap_deactivate: deactivate a process' pmap
1710: */
1711:
1712: void
1713: pmap_deactivate(struct proc *p)
1714: {
1715: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
1716:
1717: /*
1718: * mark the pmap no longer in use by this processor.
1719: */
1720: i386_atomic_clearbits_l(&pmap->pm_cpus, (1U << cpu_number()));
1721: }
1722:
1723: /*
1724: * end of lifecycle functions
1725: */
1726:
1727: /*
1728: * some misc. functions
1729: */
1730:
1731: /*
1732: * pmap_extract: extract a PA for the given VA
1733: */
1734:
1735: boolean_t
1736: pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap)
1737: {
1738: pt_entry_t *ptes, pte;
1739:
1740: if (pmap_valid_entry(pmap->pm_pdir[pdei(va)])) {
1741: ptes = pmap_map_ptes(pmap);
1742: pte = ptes[atop(va)];
1743: pmap_unmap_ptes(pmap);
1744: if (!pmap_valid_entry(pte))
1745: return (FALSE);
1746: if (pap != NULL)
1747: *pap = (pte & PG_FRAME) | (va & ~PG_FRAME);
1748: return (TRUE);
1749: }
1750: return (FALSE);
1751: }
1752:
1753: /*
1754: * pmap_virtual_space: used during bootup [pmap_steal_memory] to
1755: * determine the bounds of the kernel virtual address space.
1756: */
1757:
1758: void
1759: pmap_virtual_space(vaddr_t *startp, vaddr_t *endp)
1760: {
1761: *startp = virtual_avail;
1762: *endp = virtual_end;
1763: }
1764:
1765: /*
1766: * pmap_zero_page: zero a page
1767: */
1768: void (*pagezero)(void *, size_t) = bzero;
1769:
1770: void
1771: pmap_zero_page(struct vm_page *pg)
1772: {
1773: pmap_zero_phys(VM_PAGE_TO_PHYS(pg));
1774: }
1775:
1776: /*
1777: * pmap_zero_phys: same as pmap_zero_page, but for use before vm_pages are
1778: * initialized.
1779: */
1780: void
1781: pmap_zero_phys(paddr_t pa)
1782: {
1783: #ifdef MULTIPROCESSOR
1784: int id = cpu_number();
1785: #endif
1786: pt_entry_t *zpte = PTESLEW(zero_pte, id);
1787: caddr_t zerova = VASLEW(zerop, id);
1788:
1789: #ifdef DIAGNOSTIC
1790: if (*zpte)
1791: panic("pmap_zero_phys: lock botch");
1792: #endif
1793:
1794: *zpte = (pa & PG_FRAME) | PG_V | PG_RW; /* map in */
1795: pmap_update_pg((vaddr_t)zerova); /* flush TLB */
1796: pagezero(zerova, PAGE_SIZE); /* zero */
1797: *zpte = 0; /* zap! */
1798: }
1799:
1800: /*
1801: * pmap_zero_page_uncached: the same, except uncached.
1802: */
1803:
1804: boolean_t
1805: pmap_zero_page_uncached(paddr_t pa)
1806: {
1807: #ifdef MULTIPROCESSOR
1808: int id = cpu_number();
1809: #endif
1810: pt_entry_t *zpte = PTESLEW(zero_pte, id);
1811: caddr_t zerova = VASLEW(zerop, id);
1812:
1813: #ifdef DIAGNOSTIC
1814: if (*zpte)
1815: panic("pmap_zero_page_uncached: lock botch");
1816: #endif
1817:
1818: *zpte = (pa & PG_FRAME) | PG_V | PG_RW | PG_N; /* map in */
1819: pmap_update_pg((vaddr_t)zerova); /* flush TLB */
1820: pagezero(zerova, PAGE_SIZE); /* zero */
1821: *zpte = 0; /* zap! */
1822:
1823: return (TRUE);
1824: }
1825:
1826: /*
1827: * pmap_copy_page: copy a page
1828: */
1829:
1830: void
1831: pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
1832: {
1833: paddr_t srcpa = VM_PAGE_TO_PHYS(srcpg);
1834: paddr_t dstpa = VM_PAGE_TO_PHYS(dstpg);
1835: #ifdef MULTIPROCESSOR
1836: int id = cpu_number();
1837: #endif
1838: pt_entry_t *spte = PTESLEW(csrc_pte, id);
1839: pt_entry_t *dpte = PTESLEW(cdst_pte, id);
1840: caddr_t csrcva = VASLEW(csrcp, id);
1841: caddr_t cdstva = VASLEW(cdstp, id);
1842:
1843: #ifdef DIAGNOSTIC
1844: if (*spte || *dpte)
1845: panic("pmap_copy_page: lock botch");
1846: #endif
1847:
1848: *spte = (srcpa & PG_FRAME) | PG_V | PG_RW;
1849: *dpte = (dstpa & PG_FRAME) | PG_V | PG_RW;
1850: pmap_update_2pg((vaddr_t)csrcva, (vaddr_t)cdstva);
1851: bcopy(csrcva, cdstva, PAGE_SIZE);
1852: *spte = *dpte = 0; /* zap! */
1853: pmap_update_2pg((vaddr_t)csrcva, (vaddr_t)cdstva);
1854: }
1855:
1856: /*
1857: * p m a p r e m o v e f u n c t i o n s
1858: *
1859: * functions that remove mappings
1860: */
1861:
1862: /*
1863: * pmap_remove_ptes: remove PTEs from a PTP
1864: *
1865: * => must have proper locking on pmap_master_lock
1866: * => caller must hold pmap's lock
1867: * => PTP must be mapped into KVA
1868: * => PTP should be null if pmap == pmap_kernel()
1869: */
1870:
1871: void
1872: pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
1873: vaddr_t startva, vaddr_t endva, int flags)
1874: {
1875: struct pv_entry *pv_tofree = NULL; /* list of pv_entrys to free */
1876: struct pv_entry *pve;
1877: pt_entry_t *pte = (pt_entry_t *) ptpva;
1878: struct vm_page *pg;
1879: pt_entry_t opte;
1880:
1881: /*
1882: * note that ptpva points to the PTE that maps startva. this may
1883: * or may not be the first PTE in the PTP.
1884: *
1885: * we loop through the PTP while there are still PTEs to look at
1886: * and the wire_count is greater than 1 (because we use the wire_count
1887: * to keep track of the number of real PTEs in the PTP).
1888: */
1889:
1890: for (/*null*/; startva < endva && (ptp == NULL || ptp->wire_count > 1)
1891: ; pte++, startva += NBPG) {
1892: if (!pmap_valid_entry(*pte))
1893: continue; /* VA not mapped */
1894:
1895: if ((flags & PMAP_REMOVE_SKIPWIRED) && (*pte & PG_W))
1896: continue;
1897:
1898: /* atomically save the old PTE and zap! it */
1899: opte = i386_atomic_testset_ul(pte, 0);
1900:
1901: if (opte & PG_W)
1902: pmap->pm_stats.wired_count--;
1903: pmap->pm_stats.resident_count--;
1904:
1905: if (ptp)
1906: ptp->wire_count--; /* dropping a PTE */
1907:
1908: /*
1909: * Unnecessary work if not PG_VLIST.
1910: */
1911: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
1912:
1913: /*
1914: * if we are not on a pv list we are done.
1915: */
1916: if ((opte & PG_PVLIST) == 0) {
1917: #ifdef DIAGNOSTIC
1918: if (pg != NULL)
1919: panic("pmap_remove_ptes: managed page without "
1920: "PG_PVLIST for 0x%lx", startva);
1921: #endif
1922: continue;
1923: }
1924:
1925: #ifdef DIAGNOSTIC
1926: if (pg == NULL)
1927: panic("pmap_remove_ptes: unmanaged page marked "
1928: "PG_PVLIST, va = 0x%lx, pa = 0x%lx",
1929: startva, (u_long)(opte & PG_FRAME));
1930: #endif
1931:
1932: /* sync R/M bits */
1933: pmap_sync_flags_pte(pg, opte);
1934: pve = pmap_remove_pv(pg, pmap, startva);
1935: if (pve) {
1936: pve->pv_next = pv_tofree;
1937: pv_tofree = pve;
1938: }
1939:
1940: /* end of "for" loop: time for next pte */
1941: }
1942: if (pv_tofree)
1943: pmap_free_pvs(pmap, pv_tofree);
1944: }
1945:
1946:
1947: /*
1948: * pmap_remove_pte: remove a single PTE from a PTP
1949: *
1950: * => must have proper locking on pmap_master_lock
1951: * => caller must hold pmap's lock
1952: * => PTP must be mapped into KVA
1953: * => PTP should be null if pmap == pmap_kernel()
1954: * => returns true if we removed a mapping
1955: */
1956:
1957: boolean_t
1958: pmap_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte,
1959: vaddr_t va, int flags)
1960: {
1961: struct pv_entry *pve;
1962: struct vm_page *pg;
1963: pt_entry_t opte;
1964:
1965: if (!pmap_valid_entry(*pte))
1966: return (FALSE); /* VA not mapped */
1967:
1968: if ((flags & PMAP_REMOVE_SKIPWIRED) && (*pte & PG_W))
1969: return (FALSE);
1970:
1971: opte = *pte; /* save the old PTE */
1972: *pte = 0; /* zap! */
1973:
1974: pmap_exec_account(pmap, va, opte, 0);
1975:
1976: if (opte & PG_W)
1977: pmap->pm_stats.wired_count--;
1978: pmap->pm_stats.resident_count--;
1979:
1980: if (ptp)
1981: ptp->wire_count--; /* dropping a PTE */
1982:
1983: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
1984:
1985: /*
1986: * if we are not on a pv list we are done.
1987: */
1988: if ((opte & PG_PVLIST) == 0) {
1989: #ifdef DIAGNOSTIC
1990: if (pg != NULL)
1991: panic("pmap_remove_pte: managed page without "
1992: "PG_PVLIST for 0x%lx", va);
1993: #endif
1994: return(TRUE);
1995: }
1996:
1997: #ifdef DIAGNOSTIC
1998: if (pg == NULL)
1999: panic("pmap_remove_pte: unmanaged page marked "
2000: "PG_PVLIST, va = 0x%lx, pa = 0x%lx", va,
2001: (u_long)(opte & PG_FRAME));
2002: #endif
2003:
2004: pmap_sync_flags_pte(pg, opte);
2005: pve = pmap_remove_pv(pg, pmap, va);
2006: if (pve)
2007: pmap_free_pv(pmap, pve);
2008: return(TRUE);
2009: }
2010:
2011: /*
2012: * pmap_remove: top level mapping removal function
2013: *
2014: * => caller should not be holding any pmap locks
2015: */
2016:
2017: void
2018: pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
2019: {
2020: pmap_do_remove(pmap, sva, eva, PMAP_REMOVE_ALL);
2021: }
2022:
2023: void
2024: pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags)
2025: {
2026: pt_entry_t *ptes, opte;
2027: boolean_t result;
2028: paddr_t ptppa;
2029: vaddr_t blkendva;
2030: struct vm_page *ptp;
2031: TAILQ_HEAD(, vm_page) empty_ptps;
2032: int shootall;
2033: vaddr_t va;
2034:
2035: TAILQ_INIT(&empty_ptps);
2036:
2037: PMAP_MAP_TO_HEAD_LOCK();
2038: ptes = pmap_map_ptes(pmap); /* locks pmap */
2039:
2040: /*
2041: * removing one page? take shortcut function.
2042: */
2043:
2044: if (sva + PAGE_SIZE == eva) {
2045:
2046: if (pmap_valid_entry(pmap->pm_pdir[pdei(sva)])) {
2047:
2048: /* PA of the PTP */
2049: ptppa = pmap->pm_pdir[pdei(sva)] & PG_FRAME;
2050:
2051: /* get PTP if non-kernel mapping */
2052:
2053: if (pmap == pmap_kernel()) {
2054: /* we never free kernel PTPs */
2055: ptp = NULL;
2056: } else {
2057: if (pmap->pm_ptphint &&
2058: VM_PAGE_TO_PHYS(pmap->pm_ptphint) ==
2059: ptppa) {
2060: ptp = pmap->pm_ptphint;
2061: } else {
2062: ptp = PHYS_TO_VM_PAGE(ptppa);
2063: #ifdef DIAGNOSTIC
2064: if (ptp == NULL)
2065: panic("pmap_remove: unmanaged "
2066: "PTP detected");
2067: #endif
2068: }
2069: }
2070:
2071: /* do it! */
2072: result = pmap_remove_pte(pmap, ptp, &ptes[atop(sva)],
2073: sva, flags);
2074:
2075: /*
2076: * if mapping removed and the PTP is no longer
2077: * being used, free it!
2078: */
2079:
2080: if (result && ptp && ptp->wire_count <= 1) {
2081: opte = i386_atomic_testset_ul(
2082: &pmap->pm_pdir[pdei(sva)], 0);
2083: #ifdef MULTIPROCESSOR
2084: /*
2085: * XXXthorpej Redundant shootdown can happen
2086: * here if we're using APTE space.
2087: */
2088: #endif
2089: pmap_tlb_shootpage(curpcb->pcb_pmap,
2090: ((vaddr_t)ptes) + ptp->offset);
2091: #ifdef MULTIPROCESSOR
2092: /*
2093: * Always shoot down the pmap's self-mapping
2094: * of the PTP.
2095: * XXXthorpej Redundant shootdown can happen
2096: * here if pmap == curpcb->pcb_pmap (not APTE
2097: * space).
2098: */
2099: pmap_tlb_shootpage(pmap,
2100: ((vaddr_t)PTE_BASE) + ptp->offset);
2101: #endif
2102: pmap->pm_stats.resident_count--;
2103: if (pmap->pm_ptphint == ptp)
2104: pmap->pm_ptphint =
2105: TAILQ_FIRST(&pmap->pm_obj.memq);
2106: ptp->wire_count = 0;
2107: /* Postpone free to after shootdown. */
2108: uvm_pagerealloc(ptp, NULL, 0);
2109: TAILQ_INSERT_TAIL(&empty_ptps, ptp, listq);
2110: }
2111: /*
2112: * Shoot the tlb after any updates to the PDE.
2113: */
2114: pmap_tlb_shootpage(pmap, sva);
2115: }
2116: pmap_tlb_shootwait();
2117: pmap_unmap_ptes(pmap); /* unlock pmap */
2118: PMAP_MAP_TO_HEAD_UNLOCK();
2119: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
2120: TAILQ_REMOVE(&empty_ptps, ptp, listq);
2121: uvm_pagefree(ptp);
2122: }
2123: return;
2124: }
2125:
2126: /*
2127: * Decide if we want to shoot the whole tlb or just the range.
2128: * Right now, we simply shoot everything when we remove more
2129: * than 32 pages, but never in the kernel pmap. XXX - tune.
2130: */
2131: if ((eva - sva > 32 * PAGE_SIZE) && pmap != pmap_kernel())
2132: shootall = 1;
2133: else
2134: shootall = 0;
2135:
2136: for (va = sva ; va < eva ; va = blkendva) {
2137: /* determine range of block */
2138: blkendva = i386_round_pdr(va + 1);
2139: if (blkendva > eva)
2140: blkendva = eva;
2141:
2142: /*
2143: * XXXCDC: our PTE mappings should never be removed
2144: * with pmap_remove! if we allow this (and why would
2145: * we?) then we end up freeing the pmap's page
2146: * directory page (PDP) before we are finished using
2147: * it when we hit in in the recursive mapping. this
2148: * is BAD.
2149: *
2150: * long term solution is to move the PTEs out of user
2151: * address space. and into kernel address space (up
2152: * with APTE). then we can set VM_MAXUSER_ADDRESS to
2153: * be VM_MAX_ADDRESS.
2154: */
2155:
2156: if (pdei(va) == PDSLOT_PTE)
2157: /* XXXCDC: ugly hack to avoid freeing PDP here */
2158: continue;
2159:
2160: if (!pmap_valid_entry(pmap->pm_pdir[pdei(va)]))
2161: /* valid block? */
2162: continue;
2163:
2164: /* PA of the PTP */
2165: ptppa = (pmap->pm_pdir[pdei(va)] & PG_FRAME);
2166:
2167: /* get PTP if non-kernel mapping */
2168: if (pmap == pmap_kernel()) {
2169: /* we never free kernel PTPs */
2170: ptp = NULL;
2171: } else {
2172: if (pmap->pm_ptphint &&
2173: VM_PAGE_TO_PHYS(pmap->pm_ptphint) == ptppa) {
2174: ptp = pmap->pm_ptphint;
2175: } else {
2176: ptp = PHYS_TO_VM_PAGE(ptppa);
2177: #ifdef DIAGNOSTIC
2178: if (ptp == NULL)
2179: panic("pmap_remove: unmanaged PTP "
2180: "detected");
2181: #endif
2182: }
2183: }
2184: pmap_remove_ptes(pmap, ptp, (vaddr_t)&ptes[atop(va)],
2185: va, blkendva, flags);
2186:
2187: /* if PTP is no longer being used, free it! */
2188: if (ptp && ptp->wire_count <= 1) {
2189: opte = i386_atomic_testset_ul(
2190: &pmap->pm_pdir[pdei(va)], 0);
2191: #if defined(MULTIPROCESSOR)
2192: /*
2193: * XXXthorpej Redundant shootdown can happen here
2194: * if we're using APTE space.
2195: */
2196: #endif
2197: pmap_tlb_shootpage(curpcb->pcb_pmap,
2198: ((vaddr_t)ptes) + ptp->offset);
2199: #if defined(MULTIPROCESSOR)
2200: /*
2201: * Always shoot down the pmap's self-mapping
2202: * of the PTP.
2203: * XXXthorpej Redundant shootdown can happen here
2204: * if pmap == curpcb->pcb_pmap (not APTE space).
2205: */
2206: pmap_tlb_shootpage(pmap,
2207: ((vaddr_t)PTE_BASE) + ptp->offset);
2208: #endif
2209: pmap->pm_stats.resident_count--;
2210: if (pmap->pm_ptphint == ptp) /* update hint? */
2211: pmap->pm_ptphint =
2212: TAILQ_FIRST(&pmap->pm_obj.memq);
2213: ptp->wire_count = 0;
2214: /* Postpone free to after shootdown. */
2215: uvm_pagerealloc(ptp, NULL, 0);
2216: TAILQ_INSERT_TAIL(&empty_ptps, ptp, listq);
2217: }
2218: }
2219: if (!shootall)
2220: pmap_tlb_shootrange(pmap, sva, eva);
2221: else
2222: pmap_tlb_shoottlb();
2223:
2224: pmap_tlb_shootwait();
2225: pmap_unmap_ptes(pmap);
2226: PMAP_MAP_TO_HEAD_UNLOCK();
2227: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
2228: TAILQ_REMOVE(&empty_ptps, ptp, listq);
2229: uvm_pagefree(ptp);
2230: }
2231: }
2232:
2233: /*
2234: * pmap_page_remove: remove a managed vm_page from all pmaps that map it
2235: *
2236: * => R/M bits are sync'd back to attrs
2237: */
2238:
2239: void
2240: pmap_page_remove(struct vm_page *pg)
2241: {
2242: struct pv_entry *pve;
2243: pt_entry_t *ptes, opte;
2244: TAILQ_HEAD(, vm_page) empty_ptps;
2245: struct vm_page *ptp;
2246:
2247: if (pg->mdpage.pv_list == NULL)
2248: return;
2249:
2250: TAILQ_INIT(&empty_ptps);
2251:
2252: PMAP_HEAD_TO_MAP_LOCK();
2253:
2254: for (pve = pg->mdpage.pv_list ; pve != NULL ; pve = pve->pv_next) {
2255: ptes = pmap_map_ptes(pve->pv_pmap); /* locks pmap */
2256:
2257: #ifdef DIAGNOSTIC
2258: if (pve->pv_va >= uvm.pager_sva && pve->pv_va < uvm.pager_eva)
2259: printf("pmap_page_remove: found pager VA on pv_list\n");
2260: if (pve->pv_ptp && (pve->pv_pmap->pm_pdir[pdei(pve->pv_va)] &
2261: PG_FRAME)
2262: != VM_PAGE_TO_PHYS(pve->pv_ptp)) {
2263: printf("pmap_page_remove: pg=%p: va=%lx, pv_ptp=%p\n",
2264: pg, pve->pv_va, pve->pv_ptp);
2265: printf("pmap_page_remove: PTP's phys addr: "
2266: "actual=%x, recorded=%lx\n",
2267: (pve->pv_pmap->pm_pdir[pdei(pve->pv_va)] &
2268: PG_FRAME), VM_PAGE_TO_PHYS(pve->pv_ptp));
2269: panic("pmap_page_remove: mapped managed page has "
2270: "invalid pv_ptp field");
2271: }
2272: #endif
2273:
2274: opte = i386_atomic_testset_ul(&ptes[atop(pve->pv_va)], 0);
2275:
2276: if (opte & PG_W)
2277: pve->pv_pmap->pm_stats.wired_count--;
2278: pve->pv_pmap->pm_stats.resident_count--;
2279:
2280: /* sync R/M bits */
2281: pmap_sync_flags_pte(pg, opte);
2282:
2283: /* update the PTP reference count. free if last reference. */
2284: if (pve->pv_ptp) {
2285: pve->pv_ptp->wire_count--;
2286: if (pve->pv_ptp->wire_count <= 1) {
2287: opte = i386_atomic_testset_ul(
2288: &pve->pv_pmap->pm_pdir[pdei(pve->pv_va)],
2289: 0);
2290: pmap_tlb_shootpage(curpcb->pcb_pmap,
2291: ((vaddr_t)ptes) + pve->pv_ptp->offset);
2292: #if defined(MULTIPROCESSOR)
2293: /*
2294: * Always shoot down the other pmap's
2295: * self-mapping of the PTP.
2296: */
2297: pmap_tlb_shootpage(pve->pv_pmap,
2298: ((vaddr_t)PTE_BASE) + pve->pv_ptp->offset);
2299: #endif
2300: pve->pv_pmap->pm_stats.resident_count--;
2301: /* update hint? */
2302: if (pve->pv_pmap->pm_ptphint == pve->pv_ptp)
2303: pve->pv_pmap->pm_ptphint =
2304: TAILQ_FIRST(&pve->pv_pmap->pm_obj.memq);
2305: pve->pv_ptp->wire_count = 0;
2306: /* Postpone free to after shootdown. */
2307: uvm_pagerealloc(pve->pv_ptp, NULL, 0);
2308: TAILQ_INSERT_TAIL(&empty_ptps, pve->pv_ptp,
2309: listq);
2310: }
2311: }
2312:
2313: pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va);
2314:
2315: pmap_unmap_ptes(pve->pv_pmap); /* unlocks pmap */
2316: }
2317: pmap_free_pvs(NULL, pg->mdpage.pv_list);
2318: pg->mdpage.pv_list = NULL;
2319: PMAP_HEAD_TO_MAP_UNLOCK();
2320: pmap_tlb_shootwait();
2321:
2322: while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) {
2323: TAILQ_REMOVE(&empty_ptps, ptp, listq);
2324: uvm_pagefree(ptp);
2325: }
2326: }
2327:
2328: /*
2329: * p m a p a t t r i b u t e f u n c t i o n s
2330: * functions that test/change managed page's attributes
2331: * since a page can be mapped multiple times we must check each PTE that
2332: * maps it by going down the pv lists.
2333: */
2334:
2335: /*
2336: * pmap_test_attrs: test a page's attributes
2337: */
2338:
2339: boolean_t
2340: pmap_test_attrs(struct vm_page *pg, int testbits)
2341: {
2342: struct pv_entry *pve;
2343: pt_entry_t *ptes, pte;
2344: u_long mybits, testflags;
2345:
2346: testflags = pmap_pte2flags(testbits);
2347:
2348: if (pg->pg_flags & testflags)
2349: return (TRUE);
2350:
2351: PMAP_HEAD_TO_MAP_LOCK();
2352: mybits = 0;
2353: for (pve = pg->mdpage.pv_list; pve != NULL && mybits == 0;
2354: pve = pve->pv_next) {
2355: ptes = pmap_map_ptes(pve->pv_pmap);
2356: pte = ptes[atop(pve->pv_va)];
2357: pmap_unmap_ptes(pve->pv_pmap);
2358: mybits |= (pte & testbits);
2359: }
2360: PMAP_HEAD_TO_MAP_UNLOCK();
2361:
2362: if (mybits == 0)
2363: return (FALSE);
2364:
2365: atomic_setbits_int(&pg->pg_flags, pmap_pte2flags(mybits));
2366:
2367: return (TRUE);
2368: }
2369:
2370: /*
2371: * pmap_clear_attrs: change a page's attributes
2372: *
2373: * => we return TRUE if we cleared one of the bits we were asked to
2374: */
2375:
2376: boolean_t
2377: pmap_clear_attrs(struct vm_page *pg, int clearbits)
2378: {
2379: struct pv_entry *pve;
2380: pt_entry_t *ptes, npte, opte;
2381: u_long clearflags;
2382: int result;
2383:
2384: clearflags = pmap_pte2flags(clearbits);
2385:
2386: PMAP_HEAD_TO_MAP_LOCK();
2387:
2388: result = pg->pg_flags & clearflags;
2389: if (result)
2390: atomic_clearbits_int(&pg->pg_flags, clearflags);
2391:
2392: for (pve = pg->mdpage.pv_list; pve != NULL; pve = pve->pv_next) {
2393: #ifdef DIAGNOSTIC
2394: if (!pmap_valid_entry(pve->pv_pmap->pm_pdir[pdei(pve->pv_va)]))
2395: panic("pmap_change_attrs: mapping without PTP "
2396: "detected");
2397: #endif
2398:
2399: ptes = pmap_map_ptes(pve->pv_pmap); /* locks pmap */
2400: npte = ptes[atop(pve->pv_va)];
2401: if (npte & clearbits) {
2402: result = TRUE;
2403: npte &= ~clearbits;
2404: opte = i386_atomic_testset_ul(
2405: &ptes[atop(pve->pv_va)], npte);
2406: pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va);
2407: }
2408: pmap_unmap_ptes(pve->pv_pmap); /* unlocks pmap */
2409: }
2410:
2411: PMAP_HEAD_TO_MAP_UNLOCK();
2412: pmap_tlb_shootwait();
2413:
2414: return (result != 0);
2415: }
2416:
2417: /*
2418: * p m a p p r o t e c t i o n f u n c t i o n s
2419: */
2420:
2421: /*
2422: * pmap_page_protect: change the protection of all recorded mappings
2423: * of a managed page
2424: *
2425: * => NOTE: this is an inline function in pmap.h
2426: */
2427:
2428: /* see pmap.h */
2429:
2430: /*
2431: * pmap_protect: set the protection in of the pages in a pmap
2432: *
2433: * => NOTE: this is an inline function in pmap.h
2434: */
2435:
2436: /* see pmap.h */
2437:
2438: /*
2439: * pmap_write_protect: write-protect pages in a pmap
2440: */
2441:
2442: void
2443: pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva,
2444: vm_prot_t prot)
2445: {
2446: pt_entry_t *ptes, *spte, *epte, npte;
2447: vaddr_t blockend;
2448: u_int32_t md_prot;
2449: vaddr_t va;
2450: int shootall = 0;
2451:
2452: ptes = pmap_map_ptes(pmap); /* locks pmap */
2453:
2454: /* should be ok, but just in case ... */
2455: sva &= PG_FRAME;
2456: eva &= PG_FRAME;
2457:
2458: if ((eva - sva > 32 * PAGE_SIZE) && pmap != pmap_kernel())
2459: shootall = 1;
2460:
2461: for (va = sva; va < eva; va = blockend) {
2462: blockend = (va & PD_MASK) + NBPD;
2463: if (blockend > eva)
2464: blockend = eva;
2465:
2466: /*
2467: * XXXCDC: our PTE mappings should never be write-protected!
2468: *
2469: * long term solution is to move the PTEs out of user
2470: * address space. and into kernel address space (up
2471: * with APTE). then we can set VM_MAXUSER_ADDRESS to
2472: * be VM_MAX_ADDRESS.
2473: */
2474:
2475: /* XXXCDC: ugly hack to avoid freeing PDP here */
2476: if (pdei(va) == PDSLOT_PTE)
2477: continue;
2478:
2479: /* empty block? */
2480: if (!pmap_valid_entry(pmap->pm_pdir[pdei(va)]))
2481: continue;
2482:
2483: md_prot = protection_codes[prot];
2484: if (va < VM_MAXUSER_ADDRESS)
2485: md_prot |= PG_u;
2486: else if (va < VM_MAX_ADDRESS)
2487: /* XXX: write-prot our PTES? never! */
2488: md_prot |= (PG_u | PG_RW);
2489:
2490: spte = &ptes[atop(va)];
2491: epte = &ptes[atop(blockend)];
2492:
2493: for (/*null */; spte < epte ; spte++, va += PAGE_SIZE) {
2494:
2495: if (!pmap_valid_entry(*spte)) /* no mapping? */
2496: continue;
2497:
2498: npte = (*spte & ~PG_PROT) | md_prot;
2499:
2500: if (npte != *spte) {
2501: pmap_exec_account(pmap, va, *spte, npte);
2502: i386_atomic_testset_ul(spte, npte);
2503: }
2504: }
2505: }
2506: if (shootall)
2507: pmap_tlb_shoottlb();
2508: else
2509: pmap_tlb_shootrange(pmap, sva, eva);
2510:
2511: pmap_tlb_shootwait();
2512: pmap_unmap_ptes(pmap); /* unlocks pmap */
2513: }
2514:
2515: /*
2516: * end of protection functions
2517: */
2518:
2519: /*
2520: * pmap_unwire: clear the wired bit in the PTE
2521: *
2522: * => mapping should already be in map
2523: */
2524:
2525: void
2526: pmap_unwire(struct pmap *pmap, vaddr_t va)
2527: {
2528: pt_entry_t *ptes;
2529:
2530: if (pmap_valid_entry(pmap->pm_pdir[pdei(va)])) {
2531: ptes = pmap_map_ptes(pmap); /* locks pmap */
2532:
2533: #ifdef DIAGNOSTIC
2534: if (!pmap_valid_entry(ptes[atop(va)]))
2535: panic("pmap_unwire: invalid (unmapped) va 0x%lx", va);
2536: #endif
2537: if ((ptes[atop(va)] & PG_W) != 0) {
2538: ptes[atop(va)] &= ~PG_W;
2539: pmap->pm_stats.wired_count--;
2540: }
2541: #ifdef DIAGNOSTIC
2542: else {
2543: printf("pmap_unwire: wiring for pmap %p va 0x%lx "
2544: "didn't change!\n", pmap, va);
2545: }
2546: #endif
2547: pmap_unmap_ptes(pmap); /* unlocks map */
2548: }
2549: #ifdef DIAGNOSTIC
2550: else {
2551: panic("pmap_unwire: invalid PDE");
2552: }
2553: #endif
2554: }
2555:
2556: /*
2557: * pmap_collect: free resources held by a pmap
2558: *
2559: * => optional function.
2560: * => called when a process is swapped out to free memory.
2561: */
2562:
2563: void
2564: pmap_collect(struct pmap *pmap)
2565: {
2566: /*
2567: * free all of the pt pages by removing the physical mappings
2568: * for its entire address space.
2569: */
2570:
2571: pmap_do_remove(pmap, VM_MIN_ADDRESS, VM_MAX_ADDRESS,
2572: PMAP_REMOVE_SKIPWIRED);
2573: }
2574:
2575: /*
2576: * pmap_copy: copy mappings from one pmap to another
2577: *
2578: * => optional function
2579: * void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2580: */
2581:
2582: /*
2583: * defined as macro in pmap.h
2584: */
2585:
2586: /*
2587: * pmap_enter: enter a mapping into a pmap
2588: *
2589: * => must be done "now" ... no lazy-evaluation
2590: */
2591:
2592: int
2593: pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa,
2594: vm_prot_t prot, int flags)
2595: {
2596: pt_entry_t *ptes, opte, npte;
2597: struct vm_page *ptp;
2598: struct pv_entry *pve = NULL;
2599: boolean_t wired = (flags & PMAP_WIRED) != 0;
2600: struct vm_page *pg = NULL;
2601: int error;
2602:
2603: #ifdef DIAGNOSTIC
2604: /* sanity check: totally out of range? */
2605: if (va >= VM_MAX_KERNEL_ADDRESS)
2606: panic("pmap_enter: too big");
2607:
2608: if (va == (vaddr_t) PDP_BASE || va == (vaddr_t) APDP_BASE)
2609: panic("pmap_enter: trying to map over PDP/APDP!");
2610:
2611: /* sanity check: kernel PTPs should already have been pre-allocated */
2612: if (va >= VM_MIN_KERNEL_ADDRESS &&
2613: !pmap_valid_entry(pmap->pm_pdir[pdei(va)]))
2614: panic("pmap_enter: missing kernel PTP!");
2615: #endif
2616:
2617: /* get lock */
2618: PMAP_MAP_TO_HEAD_LOCK();
2619:
2620: /*
2621: * map in ptes and get a pointer to our PTP (unless we are the kernel)
2622: */
2623:
2624: ptes = pmap_map_ptes(pmap); /* locks pmap */
2625: if (pmap == pmap_kernel()) {
2626: ptp = NULL;
2627: } else {
2628: ptp = pmap_get_ptp(pmap, pdei(va), FALSE);
2629: if (ptp == NULL) {
2630: if (flags & PMAP_CANFAIL) {
2631: error = ENOMEM;
2632: goto out;
2633: }
2634: panic("pmap_enter: get ptp failed");
2635: }
2636: }
2637: opte = ptes[atop(va)]; /* old PTE */
2638:
2639: /*
2640: * is there currently a valid mapping at our VA?
2641: */
2642:
2643: if (pmap_valid_entry(opte)) {
2644:
2645: /*
2646: * first, update pm_stats. resident count will not
2647: * change since we are replacing/changing a valid
2648: * mapping. wired count might change...
2649: */
2650:
2651: if (wired && (opte & PG_W) == 0)
2652: pmap->pm_stats.wired_count++;
2653: else if (!wired && (opte & PG_W) != 0)
2654: pmap->pm_stats.wired_count--;
2655:
2656: /*
2657: * is the currently mapped PA the same as the one we
2658: * want to map?
2659: */
2660:
2661: if ((opte & PG_FRAME) == pa) {
2662:
2663: /* if this is on the PVLIST, sync R/M bit */
2664: if (opte & PG_PVLIST) {
2665: pg = PHYS_TO_VM_PAGE(pa);
2666: #ifdef DIAGNOSTIC
2667: if (pg == NULL)
2668: panic("pmap_enter: same pa PG_PVLIST "
2669: "mapping with unmanaged page "
2670: "pa = 0x%lx (0x%lx)", pa,
2671: atop(pa));
2672: #endif
2673: pmap_sync_flags_pte(pg, opte);
2674: }
2675: goto enter_now;
2676: }
2677:
2678: /*
2679: * changing PAs: we must remove the old one first
2680: */
2681:
2682: /*
2683: * if current mapping is on a pvlist,
2684: * remove it (sync R/M bits)
2685: */
2686:
2687: if (opte & PG_PVLIST) {
2688: pg = PHYS_TO_VM_PAGE(opte & PG_FRAME);
2689: #ifdef DIAGNOSTIC
2690: if (pg == NULL)
2691: panic("pmap_enter: PG_PVLIST mapping with "
2692: "unmanaged page "
2693: "pa = 0x%lx (0x%lx)", pa, atop(pa));
2694: #endif
2695: pmap_sync_flags_pte(pg, opte);
2696: pve = pmap_remove_pv(pg, pmap, va);
2697: pg = NULL; /* This is not page we are looking for */
2698: }
2699: } else { /* opte not valid */
2700: pmap->pm_stats.resident_count++;
2701: if (wired)
2702: pmap->pm_stats.wired_count++;
2703: if (ptp)
2704: ptp->wire_count++; /* count # of valid entries */
2705: }
2706:
2707: /*
2708: * at this point pm_stats has been updated. pve is either NULL
2709: * or points to a now-free pv_entry structure (the latter case is
2710: * if we called pmap_remove_pv above).
2711: *
2712: * if this entry is to be on a pvlist, enter it now.
2713: */
2714:
2715: if (pmap_initialized && pg == NULL)
2716: pg = PHYS_TO_VM_PAGE(pa);
2717:
2718: if (pg != NULL) {
2719: if (pve == NULL) {
2720: pve = pmap_alloc_pv(pmap, ALLOCPV_NEED);
2721: if (pve == NULL) {
2722: if (flags & PMAP_CANFAIL) {
2723: /*
2724: * XXX - Back out stats changes!
2725: */
2726: error = ENOMEM;
2727: goto out;
2728: }
2729: panic("pmap_enter: no pv entries available");
2730: }
2731: }
2732: /* lock pvh when adding */
2733: pmap_enter_pv(pg, pve, pmap, va, ptp);
2734: } else {
2735:
2736: /* new mapping is not PG_PVLIST. free pve if we've got one */
2737: if (pve)
2738: pmap_free_pv(pmap, pve);
2739: }
2740:
2741: enter_now:
2742: /*
2743: * at this point pvh is !NULL if we want the PG_PVLIST bit set
2744: */
2745:
2746: npte = pa | protection_codes[prot] | PG_V;
2747: pmap_exec_account(pmap, va, opte, npte);
2748: if (wired)
2749: npte |= PG_W;
2750: if (va < VM_MAXUSER_ADDRESS)
2751: npte |= PG_u;
2752: else if (va < VM_MAX_ADDRESS)
2753: npte |= (PG_u | PG_RW); /* XXXCDC: no longer needed? */
2754: if (pmap == pmap_kernel())
2755: npte |= pmap_pg_g;
2756: if (flags & VM_PROT_READ)
2757: npte |= PG_U;
2758: if (flags & VM_PROT_WRITE)
2759: npte |= PG_M;
2760: if (pg) {
2761: npte |= PG_PVLIST;
2762: pmap_sync_flags_pte(pg, npte);
2763: }
2764:
2765: opte = i386_atomic_testset_ul(&ptes[atop(va)], npte);
2766:
2767: if (opte & PG_V) {
2768: pmap_tlb_shootpage(pmap, va);
2769: pmap_tlb_shootwait();
2770: }
2771:
2772: error = 0;
2773:
2774: out:
2775: pmap_unmap_ptes(pmap);
2776: PMAP_MAP_TO_HEAD_UNLOCK();
2777:
2778: return error;
2779: }
2780:
2781: /*
2782: * pmap_growkernel: increase usage of KVM space
2783: *
2784: * => we allocate new PTPs for the kernel and install them in all
2785: * the pmaps on the system.
2786: */
2787:
2788: vaddr_t
2789: pmap_growkernel(vaddr_t maxkvaddr)
2790: {
2791: struct pmap *kpm = pmap_kernel(), *pm;
2792: int needed_kpde; /* needed number of kernel PTPs */
2793: int s;
2794: paddr_t ptaddr;
2795:
2796: needed_kpde = (int)(maxkvaddr - VM_MIN_KERNEL_ADDRESS + (NBPD-1))
2797: / NBPD;
2798: if (needed_kpde <= nkpde)
2799: goto out; /* we are OK */
2800:
2801: /*
2802: * whoops! we need to add kernel PTPs
2803: */
2804:
2805: s = splhigh(); /* to be safe */
2806: simple_lock(&kpm->pm_obj.vmobjlock);
2807:
2808: for (/*null*/ ; nkpde < needed_kpde ; nkpde++) {
2809:
2810: if (uvm.page_init_done == FALSE) {
2811:
2812: /*
2813: * we're growing the kernel pmap early (from
2814: * uvm_pageboot_alloc()). this case must be
2815: * handled a little differently.
2816: */
2817:
2818: if (uvm_page_physget(&ptaddr) == FALSE)
2819: panic("pmap_growkernel: out of memory");
2820: pmap_zero_phys(ptaddr);
2821:
2822: kpm->pm_pdir[PDSLOT_KERN + nkpde] =
2823: ptaddr | PG_RW | PG_V | PG_U | PG_M;
2824:
2825: /* count PTP as resident */
2826: kpm->pm_stats.resident_count++;
2827: continue;
2828: }
2829:
2830: /*
2831: * THIS *MUST* BE CODED SO AS TO WORK IN THE
2832: * pmap_initialized == FALSE CASE! WE MAY BE
2833: * INVOKED WHILE pmap_init() IS RUNNING!
2834: */
2835:
2836: while (!pmap_alloc_ptp(kpm, PDSLOT_KERN + nkpde, FALSE, 0))
2837: uvm_wait("pmap_growkernel");
2838:
2839: /* distribute new kernel PTP to all active pmaps */
2840: simple_lock(&pmaps_lock);
2841: LIST_FOREACH(pm, &pmaps, pm_list) {
2842: pm->pm_pdir[PDSLOT_KERN + nkpde] =
2843: kpm->pm_pdir[PDSLOT_KERN + nkpde];
2844: }
2845: simple_unlock(&pmaps_lock);
2846: }
2847:
2848: simple_unlock(&kpm->pm_obj.vmobjlock);
2849: splx(s);
2850:
2851: out:
2852: return (VM_MIN_KERNEL_ADDRESS + (nkpde * NBPD));
2853: }
2854:
2855: #ifdef DEBUG
2856: void pmap_dump(struct pmap *, vaddr_t, vaddr_t);
2857:
2858: /*
2859: * pmap_dump: dump all the mappings from a pmap
2860: *
2861: * => caller should not be holding any pmap locks
2862: */
2863:
2864: void
2865: pmap_dump(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
2866: {
2867: pt_entry_t *ptes, *pte;
2868: vaddr_t blkendva;
2869:
2870: /*
2871: * if end is out of range truncate.
2872: * if (end == start) update to max.
2873: */
2874:
2875: if (eva > VM_MAXUSER_ADDRESS || eva <= sva)
2876: eva = VM_MAXUSER_ADDRESS;
2877:
2878: PMAP_MAP_TO_HEAD_LOCK();
2879: ptes = pmap_map_ptes(pmap); /* locks pmap */
2880:
2881: /*
2882: * dumping a range of pages: we dump in PTP sized blocks (4MB)
2883: */
2884:
2885: for (/* null */ ; sva < eva ; sva = blkendva) {
2886:
2887: /* determine range of block */
2888: blkendva = i386_round_pdr(sva+1);
2889: if (blkendva > eva)
2890: blkendva = eva;
2891:
2892: /* valid block? */
2893: if (!pmap_valid_entry(pmap->pm_pdir[pdei(sva)]))
2894: continue;
2895:
2896: pte = &ptes[atop(sva)];
2897: for (/* null */; sva < blkendva ; sva += NBPG, pte++) {
2898: if (!pmap_valid_entry(*pte))
2899: continue;
2900: printf("va %#lx -> pa %#x (pte=%#x)\n",
2901: sva, *pte, *pte & PG_FRAME);
2902: }
2903: }
2904: pmap_unmap_ptes(pmap);
2905: PMAP_MAP_TO_HEAD_UNLOCK();
2906: }
2907: #endif
2908:
2909: #ifdef MULTIPROCESSOR
2910: /*
2911: * Locking for tlb shootdown.
2912: *
2913: * We lock by setting tlb_shoot_wait to the number of cpus that will
2914: * receive our tlb shootdown. After sending the IPIs, we don't need to
2915: * worry about locking order or interrupts spinning for the lock because
2916: * the call that grabs the "lock" isn't the one that releases it. And
2917: * there is nothing that can block the IPI that releases the lock.
2918: *
2919: * The functions are organized so that we first count the number of
2920: * cpus we need to send the IPI to, then we grab the counter, then
2921: * we send the IPIs, then we finally do our own shootdown.
2922: *
2923: * Our shootdown is last to make it parallell with the other cpus
2924: * to shorten the spin time.
2925: *
2926: * Notice that we depend on failures to send IPIs only being able to
2927: * happen during boot. If they happen later, the above assumption
2928: * doesn't hold since we can end up in situations where noone will
2929: * release the lock if we get an interrupt in a bad moment.
2930: */
2931:
2932: volatile int tlb_shoot_wait;
2933:
2934: volatile vaddr_t tlb_shoot_addr1;
2935: volatile vaddr_t tlb_shoot_addr2;
2936:
2937: void
2938: pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
2939: {
2940: struct cpu_info *ci, *self = curcpu();
2941: CPU_INFO_ITERATOR cii;
2942: int wait = 0;
2943: int mask = 0;
2944:
2945: CPU_INFO_FOREACH(cii, ci) {
2946: if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
2947: !(ci->ci_flags & CPUF_RUNNING))
2948: continue;
2949: mask |= 1 << ci->ci_cpuid;
2950: wait++;
2951: }
2952:
2953: if (wait > 0) {
2954: int s = splvm();
2955:
2956: while (i486_atomic_cas_int(&tlb_shoot_wait, 0, wait) != 0) {
2957: while (tlb_shoot_wait != 0)
2958: SPINLOCK_SPIN_HOOK;
2959: }
2960: tlb_shoot_addr1 = va;
2961: CPU_INFO_FOREACH(cii, ci) {
2962: if ((mask & 1 << ci->ci_cpuid) == 0)
2963: continue;
2964: if (i386_fast_ipi(ci, LAPIC_IPI_INVLPG) != 0)
2965: panic("pmap_tlb_shootpage: ipi failed");
2966: }
2967: splx(s);
2968: }
2969:
2970: if (pmap_is_curpmap(pm))
2971: pmap_update_pg(va);
2972: }
2973:
2974: void
2975: pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
2976: {
2977: struct cpu_info *ci, *self = curcpu();
2978: CPU_INFO_ITERATOR cii;
2979: int wait = 0;
2980: int mask = 0;
2981: vaddr_t va;
2982:
2983: CPU_INFO_FOREACH(cii, ci) {
2984: if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
2985: !(ci->ci_flags & CPUF_RUNNING))
2986: continue;
2987: mask |= 1 << ci->ci_cpuid;
2988: wait++;
2989: }
2990:
2991: if (wait > 0) {
2992: int s = splvm();
2993:
2994: while (i486_atomic_cas_int(&tlb_shoot_wait, 0, wait) != 0) {
2995: while (tlb_shoot_wait != 0)
2996: SPINLOCK_SPIN_HOOK;
2997: }
2998: tlb_shoot_addr1 = sva;
2999: tlb_shoot_addr2 = eva;
3000: CPU_INFO_FOREACH(cii, ci) {
3001: if ((mask & 1 << ci->ci_cpuid) == 0)
3002: continue;
3003: if (i386_fast_ipi(ci, LAPIC_IPI_INVLRANGE) != 0)
3004: panic("pmap_tlb_shootrange: ipi failed");
3005: }
3006: splx(s);
3007: }
3008:
3009: if (pmap_is_curpmap(pm))
3010: for (va = sva; va < eva; va += PAGE_SIZE)
3011: pmap_update_pg(va);
3012: }
3013:
3014: void
3015: pmap_tlb_shoottlb(void)
3016: {
3017: struct cpu_info *ci, *self = curcpu();
3018: CPU_INFO_ITERATOR cii;
3019: int wait = 0;
3020: int mask = 0;
3021:
3022: CPU_INFO_FOREACH(cii, ci) {
3023: if (ci == self || !(ci->ci_flags & CPUF_RUNNING))
3024: continue;
3025: mask |= 1 << ci->ci_cpuid;
3026: wait++;
3027: }
3028:
3029: if (wait) {
3030: int s = splvm();
3031:
3032: while (i486_atomic_cas_int(&tlb_shoot_wait, 0, wait) != 0) {
3033: while (tlb_shoot_wait != 0)
3034: SPINLOCK_SPIN_HOOK;
3035: }
3036:
3037: CPU_INFO_FOREACH(cii, ci) {
3038: if ((mask & 1 << ci->ci_cpuid) == 0)
3039: continue;
3040: if (i386_fast_ipi(ci, LAPIC_IPI_INVLTLB) != 0)
3041: panic("pmap_tlb_shoottlb: ipi failed");
3042: }
3043: splx(s);
3044: }
3045:
3046: tlbflush();
3047: }
3048:
3049: void
3050: pmap_tlb_shootwait(void)
3051: {
3052: while (tlb_shoot_wait != 0)
3053: SPINLOCK_SPIN_HOOK;
3054: }
3055:
3056: #else
3057:
3058: void
3059: pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
3060: {
3061: if (pmap_is_curpmap(pm))
3062: pmap_update_pg(va);
3063:
3064: }
3065:
3066: void
3067: pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
3068: {
3069: vaddr_t va;
3070:
3071: for (va = sva; va < eva; va += PAGE_SIZE)
3072: pmap_update_pg(va);
3073: }
3074:
3075: void
3076: pmap_tlb_shoottlb(void)
3077: {
3078: tlbflush();
3079: }
3080: #endif /* MULTIPROCESSOR */
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / i386 / i386