Annotation of sys/arch/alpha/alpha/pmap.c, Revision 1.1.1.1
1.1 nbrk 1: /* $OpenBSD: pmap.c,v 1.52 2007/05/26 20:26:50 pedro Exp $ */
2: /* $NetBSD: pmap.c,v 1.154 2000/12/07 22:18:55 thorpej Exp $ */
3:
4: /*-
5: * Copyright (c) 1998, 1999, 2000 The NetBSD Foundation, Inc.
6: * All rights reserved.
7: *
8: * This code is derived from software contributed to The NetBSD Foundation
9: * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
10: * NASA Ames Research Center and by Chris G. Demetriou.
11: *
12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: * 3. All advertising materials mentioning features or use of this software
21: * must display the following acknowledgement:
22: * This product includes software developed by the NetBSD
23: * Foundation, Inc. and its contributors.
24: * 4. Neither the name of The NetBSD Foundation nor the names of its
25: * contributors may be used to endorse or promote products derived
26: * from this software without specific prior written permission.
27: *
28: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
29: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
30: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
31: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
32: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38: * POSSIBILITY OF SUCH DAMAGE.
39: */
40:
41: /*
42: * Copyright (c) 1991, 1993
43: * The Regents of the University of California. All rights reserved.
44: *
45: * This code is derived from software contributed to Berkeley by
46: * the Systems Programming Group of the University of Utah Computer
47: * Science Department.
48: *
49: * Redistribution and use in source and binary forms, with or without
50: * modification, are permitted provided that the following conditions
51: * are met:
52: * 1. Redistributions of source code must retain the above copyright
53: * notice, this list of conditions and the following disclaimer.
54: * 2. Redistributions in binary form must reproduce the above copyright
55: * notice, this list of conditions and the following disclaimer in the
56: * documentation and/or other materials provided with the distribution.
57: * 3. Neither the name of the University nor the names of its contributors
58: * may be used to endorse or promote products derived from this software
59: * without specific prior written permission.
60: *
61: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
62: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
63: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
64: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
65: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
66: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
67: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
68: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
69: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
70: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
71: * SUCH DAMAGE.
72: *
73: * @(#)pmap.c 8.6 (Berkeley) 5/27/94
74: */
75:
76: /*
77: * DEC Alpha physical map management code.
78: *
79: * History:
80: *
81: * This pmap started life as a Motorola 68851/68030 pmap,
82: * written by Mike Hibler at the University of Utah.
83: *
84: * It was modified for the DEC Alpha by Chris Demetriou
85: * at Carnegie Mellon University.
86: *
87: * Support for non-contiguous physical memory was added by
88: * Jason R. Thorpe of the Numerical Aerospace Simulation
89: * Facility, NASA Ames Research Center and Chris Demetriou.
90: *
91: * Page table management and a major cleanup were undertaken
92: * by Jason R. Thorpe, with lots of help from Ross Harvey of
93: * Avalon Computer Systems and from Chris Demetriou.
94: *
95: * Support for the new UVM pmap interface was written by
96: * Jason R. Thorpe.
97: *
98: * Support for ASNs was written by Jason R. Thorpe, again
99: * with help from Chris Demetriou and Ross Harvey.
100: *
101: * The locking protocol was written by Jason R. Thorpe,
102: * using Chuck Cranor's i386 pmap for UVM as a model.
103: *
104: * TLB shootdown code was written by Jason R. Thorpe.
105: *
106: * Notes:
107: *
108: * All page table access is done via K0SEG. The one exception
109: * to this is for kernel mappings. Since all kernel page
110: * tables are pre-allocated, we can use the Virtual Page Table
111: * to access PTEs that map K1SEG addresses.
112: *
113: * Kernel page table pages are statically allocated in
114: * pmap_bootstrap(), and are never freed. In the future,
115: * support for dynamically adding additional kernel page
116: * table pages may be added. User page table pages are
117: * dynamically allocated and freed.
118: *
119: * Bugs/misfeatures:
120: *
121: * - Some things could be optimized.
122: */
123:
124: /*
125: * Manages physical address maps.
126: *
127: * Since the information managed by this module is
128: * also stored by the logical address mapping module,
129: * this module may throw away valid virtual-to-physical
130: * mappings at almost any time. However, invalidations
131: * of virtual-to-physical mappings must be done as
132: * requested.
133: *
134: * In order to cope with hardware architectures which
135: * make virtual-to-physical map invalidates expensive,
136: * this module may delay invalidate or reduced protection
137: * operations until such time as they are actually
138: * necessary. This module is given full information as
139: * to which processors are currently using which maps,
140: * and to when physical maps must be made correct.
141: */
142:
143: #include <sys/param.h>
144: #include <sys/systm.h>
145: #include <sys/proc.h>
146: #include <sys/malloc.h>
147: #include <sys/pool.h>
148: #include <sys/user.h>
149: #include <sys/buf.h>
150: #ifdef SYSVSHM
151: #include <sys/shm.h>
152: #endif
153:
154: #include <uvm/uvm.h>
155:
156: #include <machine/atomic.h>
157: #include <machine/cpu.h>
158: #if defined(_PMAP_MAY_USE_PROM_CONSOLE) || defined(MULTIPROCESSOR)
159: #include <machine/rpb.h>
160: #endif
161:
162: #ifdef DEBUG
163: #define PDB_FOLLOW 0x0001
164: #define PDB_INIT 0x0002
165: #define PDB_ENTER 0x0004
166: #define PDB_REMOVE 0x0008
167: #define PDB_CREATE 0x0010
168: #define PDB_PTPAGE 0x0020
169: #define PDB_ASN 0x0040
170: #define PDB_BITS 0x0080
171: #define PDB_COLLECT 0x0100
172: #define PDB_PROTECT 0x0200
173: #define PDB_BOOTSTRAP 0x1000
174: #define PDB_PARANOIA 0x2000
175: #define PDB_WIRING 0x4000
176: #define PDB_PVDUMP 0x8000
177:
178: int debugmap = 0;
179: int pmapdebug = PDB_PARANOIA|PDB_FOLLOW|PDB_ENTER;
180: #endif
181:
182: /*
183: * Given a map and a machine independent protection code,
184: * convert to an alpha protection code.
185: */
186: #define pte_prot(m, p) (protection_codes[m == pmap_kernel() ? 0 : 1][p])
187: int protection_codes[2][8];
188:
189: /*
190: * kernel_lev1map:
191: *
192: * Kernel level 1 page table. This maps all kernel level 2
193: * page table pages, and is used as a template for all user
194: * pmap level 1 page tables. When a new user level 1 page
195: * table is allocated, all kernel_lev1map PTEs for kernel
196: * addresses are copied to the new map.
197: *
198: * The kernel also has an initial set of kernel level 2 page
199: * table pages. These map the kernel level 3 page table pages.
200: * As kernel level 3 page table pages are added, more level 2
201: * page table pages may be added to map them. These pages are
202: * never freed.
203: *
204: * Finally, the kernel also has an initial set of kernel level
205: * 3 page table pages. These map pages in K1SEG. More level
206: * 3 page table pages may be added at run-time if additional
207: * K1SEG address space is required. These pages are never freed.
208: *
209: * NOTE: When mappings are inserted into the kernel pmap, all
210: * level 2 and level 3 page table pages must already be allocated
211: * and mapped into the parent page table.
212: */
213: pt_entry_t *kernel_lev1map;
214:
215: /*
216: * Virtual Page Table.
217: */
218: pt_entry_t *VPT;
219:
220: struct pmap kernel_pmap_store;
221: u_int kernel_pmap_asn_store[ALPHA_MAXPROCS];
222: u_long kernel_pmap_asngen_store[ALPHA_MAXPROCS];
223:
224: paddr_t avail_start; /* PA of first available physical page */
225: paddr_t avail_end; /* PA of last available physical page */
226: vaddr_t virtual_avail; /* VA of first avail page (after kernel bss)*/
227: vaddr_t virtual_end; /* VA of last avail page (end of kernel AS) */
228:
229: boolean_t pmap_initialized; /* Has pmap_init completed? */
230:
231: u_long pmap_pages_stolen; /* instrumentation */
232:
233: /*
234: * This variable contains the number of CPU IDs we need to allocate
235: * space for when allocating the pmap structure. It is used to
236: * size a per-CPU array of ASN and ASN Generation number.
237: */
238: u_long pmap_ncpuids;
239:
240: /*
241: * Storage for physical->virtual entries and page attributes.
242: */
243: struct pv_head *pv_table;
244: int pv_table_npages;
245:
246: #ifndef PMAP_PV_LOWAT
247: #define PMAP_PV_LOWAT 16
248: #endif
249: int pmap_pv_lowat = PMAP_PV_LOWAT;
250:
251: /*
252: * List of all pmaps, used to update them when e.g. additional kernel
253: * page tables are allocated. This list is kept LRU-ordered by
254: * pmap_activate().
255: */
256: TAILQ_HEAD(, pmap) pmap_all_pmaps;
257:
258: /*
259: * The pools from which pmap structures and sub-structures are allocated.
260: */
261: struct pool pmap_pmap_pool;
262: struct pool pmap_l1pt_pool;
263: struct pool_cache pmap_l1pt_cache;
264: struct pool pmap_asn_pool;
265: struct pool pmap_asngen_pool;
266: struct pool pmap_pv_pool;
267:
268: /*
269: * Canonical names for PGU_* constants.
270: */
271: const char *pmap_pgu_strings[] = PGU_STRINGS;
272:
273: /*
274: * Address Space Numbers.
275: *
276: * On many implementations of the Alpha architecture, the TLB entries and
277: * I-cache blocks are tagged with a unique number within an implementation-
278: * specified range. When a process context becomes active, the ASN is used
279: * to match TLB entries; if a TLB entry for a particular VA does not match
280: * the current ASN, it is ignored (one could think of the processor as
281: * having a collection of <max ASN> separate TLBs). This allows operating
282: * system software to skip the TLB flush that would otherwise be necessary
283: * at context switch time.
284: *
285: * Alpha PTEs have a bit in them (PG_ASM - Address Space Match) that
286: * causes TLB entries to match any ASN. The PALcode also provides
287: * a TBI (Translation Buffer Invalidate) operation that flushes all
288: * TLB entries that _do not_ have PG_ASM. We use this bit for kernel
289: * mappings, so that invalidation of all user mappings does not invalidate
290: * kernel mappings (which are consistent across all processes).
291: *
292: * pmap_next_asn always indicates to the next ASN to use. When
293: * pmap_next_asn exceeds pmap_max_asn, we start a new ASN generation.
294: *
295: * When a new ASN generation is created, the per-process (i.e. non-PG_ASM)
296: * TLB entries and the I-cache are flushed, the generation number is bumped,
297: * and pmap_next_asn is changed to indicate the first non-reserved ASN.
298: *
299: * We reserve ASN #0 for pmaps that use the global kernel_lev1map. This
300: * prevents the following scenario:
301: *
302: * * New ASN generation starts, and process A is given ASN #0.
303: *
304: * * A new process B (and thus new pmap) is created. The ASN,
305: * for lack of a better value, is initialized to 0.
306: *
307: * * Process B runs. It is now using the TLB entries tagged
308: * by process A. *poof*
309: *
310: * In the scenario above, in addition to the processor using using incorrect
311: * TLB entries, the PALcode might use incorrect information to service a
312: * TLB miss. (The PALcode uses the recursively mapped Virtual Page Table
313: * to locate the PTE for a faulting address, and tagged TLB entries exist
314: * for the Virtual Page Table addresses in order to speed up this procedure,
315: * as well.)
316: *
317: * By reserving an ASN for kernel_lev1map users, we are guaranteeing that
318: * new pmaps will initially run with no TLB entries for user addresses
319: * or VPT mappings that map user page tables. Since kernel_lev1map only
320: * contains mappings for kernel addresses, and since those mappings
321: * are always made with PG_ASM, sharing an ASN for kernel_lev1map users is
322: * safe (since PG_ASM mappings match any ASN).
323: *
324: * On processors that do not support ASNs, the PALcode invalidates
325: * the TLB and I-cache automatically on swpctx. We still still go
326: * through the motions of assigning an ASN (really, just refreshing
327: * the ASN generation in this particular case) to keep the logic sane
328: * in other parts of the code.
329: */
330: u_int pmap_max_asn; /* max ASN supported by the system */
331: u_int pmap_next_asn[ALPHA_MAXPROCS]; /* next free ASN to use */
332: u_long pmap_asn_generation[ALPHA_MAXPROCS]; /* current ASN generation */
333:
334: /*
335: * Locking:
336: *
337: * This pmap module uses two types of locks: `normal' (sleep)
338: * locks and `simple' (spin) locks. They are used as follows:
339: *
340: * SIMPLE LOCKS
341: * ------------
342: *
343: * * pm_slock (per-pmap) - This lock protects all of the members
344: * of the pmap structure itself. This lock will be asserted
345: * in pmap_activate() and pmap_deactivate() from a critical
346: * section of cpu_switch(), and must never sleep. Note that
347: * in the case of the kernel pmap, interrupts which cause
348: * memory allocation *must* be blocked while this lock is
349: * asserted.
350: *
351: * * pvh_slock (per-pv_head) - This lock protects the PV list
352: * for a specified managed page.
353: *
354: * * pmap_all_pmaps_slock - This lock protects the global list of
355: * all pmaps. Note that a pm_slock must never be held while this
356: * lock is held.
357: *
358: * * pmap_growkernel_slock - This lock protects pmap_growkernel()
359: * and the virtual_end variable.
360: *
361: * Address space number management (global ASN counters and per-pmap
362: * ASN state) are not locked; they use arrays of values indexed
363: * per-processor.
364: *
365: * All internal functions which operate on a pmap are called
366: * with the pmap already locked by the caller (which will be
367: * an interface function).
368: */
369: struct simplelock pmap_all_pmaps_slock;
370: struct simplelock pmap_growkernel_slock;
371:
372: #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */
373: #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */
374: #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */
375: #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */
376:
377: #if defined(MULTIPROCESSOR)
378: /*
379: * TLB Shootdown:
380: *
381: * When a mapping is changed in a pmap, the TLB entry corresponding to
382: * the virtual address must be invalidated on all processors. In order
383: * to accomplish this on systems with multiple processors, messages are
384: * sent from the processor which performs the mapping change to all
385: * processors on which the pmap is active. For other processors, the
386: * ASN generation numbers for that processor is invalidated, so that
387: * the next time the pmap is activated on that processor, a new ASN
388: * will be allocated (which implicitly invalidates all TLB entries).
389: *
390: * Note, we can use the pool allocator to allocate job entries
391: * since pool pages are mapped with K0SEG, not with the TLB.
392: */
393: struct pmap_tlb_shootdown_job {
394: TAILQ_ENTRY(pmap_tlb_shootdown_job) pj_list;
395: vaddr_t pj_va; /* virtual address */
396: pmap_t pj_pmap; /* the pmap which maps the address */
397: pt_entry_t pj_pte; /* the PTE bits */
398: };
399:
400: struct pmap_tlb_shootdown_q {
401: TAILQ_HEAD(, pmap_tlb_shootdown_job) pq_head;
402: int pq_pte; /* aggregate PTE bits */
403: int pq_count; /* number of pending requests */
404: struct simplelock pq_slock; /* spin lock on queue */
405: } pmap_tlb_shootdown_q[ALPHA_MAXPROCS];
406:
407: #define PSJQ_LOCK(pq, s) \
408: do { \
409: s = splvm(); \
410: simple_lock(&(pq)->pq_slock); \
411: } while (0)
412:
413: #define PSJQ_UNLOCK(pq, s) \
414: do { \
415: simple_unlock(&(pq)->pq_slock); \
416: splx(s); \
417: } while (0)
418:
419: /* If we have more pending jobs than this, we just nail the whole TLB. */
420: #define PMAP_TLB_SHOOTDOWN_MAXJOBS 6
421:
422: struct pool pmap_tlb_shootdown_job_pool;
423:
424: struct pmap_tlb_shootdown_job *pmap_tlb_shootdown_job_get
425: (struct pmap_tlb_shootdown_q *);
426: void pmap_tlb_shootdown_job_put(struct pmap_tlb_shootdown_q *,
427: struct pmap_tlb_shootdown_job *);
428: #endif /* MULTIPROCESSOR */
429:
430: #define PAGE_IS_MANAGED(pa) (vm_physseg_find(atop(pa), NULL) != -1)
431:
432: static __inline struct pv_head *
433: pa_to_pvh(paddr_t pa)
434: {
435: int bank, pg;
436:
437: bank = vm_physseg_find(atop(pa), &pg);
438: return (&vm_physmem[bank].pmseg.pvhead[pg]);
439: }
440:
441: /*
442: * Optional argument passed to pmap_remove_mapping() for stealing mapping
443: * resources.
444: */
445: struct prm_thief {
446: int prmt_flags; /* flags; what to steal */
447: struct pv_entry *prmt_pv; /* the stolen PV entry */
448: pt_entry_t *prmt_ptp; /* the stolen PT page */
449: };
450:
451: #define PRMT_PV 0x0001 /* steal the PV entry */
452: #define PRMT_PTP 0x0002 /* steal the PT page */
453:
454: /*
455: * Internal routines
456: */
457: void alpha_protection_init(void);
458: void pmap_do_remove(pmap_t, vaddr_t, vaddr_t, boolean_t);
459: boolean_t pmap_remove_mapping(pmap_t, vaddr_t, pt_entry_t *,
460: boolean_t, cpuid_t, struct prm_thief *);
461: void pmap_changebit(paddr_t, pt_entry_t, pt_entry_t, cpuid_t);
462:
463: /*
464: * PT page management functions.
465: */
466: int pmap_lev1map_create(pmap_t, cpuid_t);
467: void pmap_lev1map_destroy(pmap_t, cpuid_t);
468: int pmap_ptpage_alloc(pmap_t, pt_entry_t *, int);
469: boolean_t pmap_ptpage_steal(pmap_t, int, paddr_t *);
470: void pmap_ptpage_free(pmap_t, pt_entry_t *, pt_entry_t **);
471: void pmap_l3pt_delref(pmap_t, vaddr_t, pt_entry_t *, cpuid_t,
472: pt_entry_t **);
473: void pmap_l2pt_delref(pmap_t, pt_entry_t *, pt_entry_t *, cpuid_t);
474: void pmap_l1pt_delref(pmap_t, pt_entry_t *, cpuid_t);
475:
476: void *pmap_l1pt_alloc(struct pool *, int);
477: void pmap_l1pt_free(struct pool *, void *);
478:
479: struct pool_allocator pmap_l1pt_allocator = {
480: pmap_l1pt_alloc, pmap_l1pt_free, 0,
481: };
482:
483: int pmap_l1pt_ctor(void *, void *, int);
484:
485: /*
486: * PV table management functions.
487: */
488: int pmap_pv_enter(pmap_t, paddr_t, vaddr_t, pt_entry_t *, boolean_t);
489: void pmap_pv_remove(pmap_t, paddr_t, vaddr_t, boolean_t,
490: struct pv_entry **);
491: struct pv_entry *pmap_pv_alloc(void);
492: void pmap_pv_free(struct pv_entry *);
493: void *pmap_pv_page_alloc(struct pool *, int);
494: void pmap_pv_page_free(struct pool *, void *);
495: struct pool_allocator pmap_pv_allocator = {
496: pmap_pv_page_alloc, pmap_pv_page_free, 0,
497: };
498: #ifdef DEBUG
499: void pmap_pv_dump(paddr_t);
500: #endif
501:
502: /*
503: * ASN management functions.
504: */
505: void pmap_asn_alloc(pmap_t, cpuid_t);
506:
507: /*
508: * Misc. functions.
509: */
510: boolean_t pmap_physpage_alloc(int, paddr_t *);
511: void pmap_physpage_free(paddr_t);
512: int pmap_physpage_addref(void *);
513: int pmap_physpage_delref(void *);
514:
515: /*
516: * PMAP_ISACTIVE{,_TEST}:
517: *
518: * Check to see if a pmap is active on the current processor.
519: */
520: #define PMAP_ISACTIVE_TEST(pm, cpu_id) \
521: (((pm)->pm_cpus & (1UL << (cpu_id))) != 0)
522:
523: #if defined(DEBUG) && !defined(MULTIPROCESSOR)
524: #define PMAP_ISACTIVE(pm, cpu_id) \
525: ({ \
526: /* \
527: * XXX This test is not MP-safe. \
528: */ \
529: int isactive_ = PMAP_ISACTIVE_TEST(pm, cpu_id); \
530: \
531: if (curproc != NULL && curproc->p_vmspace != NULL && \
532: (pm) != pmap_kernel() && \
533: (isactive_ ^ ((pm) == curproc->p_vmspace->vm_map.pmap))) \
534: panic("PMAP_ISACTIVE, isa: %d pm: %p curpm:%p", \
535: isactive_, (pm), curproc->p_vmspace->vm_map.pmap); \
536: (isactive_); \
537: })
538: #else
539: #define PMAP_ISACTIVE(pm, cpu_id) PMAP_ISACTIVE_TEST(pm, cpu_id)
540: #endif /* DEBUG && !MULTIPROCESSOR */
541:
542: /*
543: * PMAP_ACTIVATE_ASN_SANITY:
544: *
545: * DEBUG sanity checks for ASNs within PMAP_ACTIVATE.
546: */
547: #ifdef DEBUG
548: #define PMAP_ACTIVATE_ASN_SANITY(pmap, cpu_id) \
549: do { \
550: if ((pmap)->pm_lev1map == kernel_lev1map) { \
551: /* \
552: * This pmap implementation also ensures that pmaps \
553: * referencing kernel_lev1map use a reserved ASN \
554: * ASN to prevent the PALcode from servicing a TLB \
555: * miss with the wrong PTE. \
556: */ \
557: if ((pmap)->pm_asn[(cpu_id)] != PMAP_ASN_RESERVED) { \
558: printf("kernel_lev1map with non-reserved ASN " \
559: "(line %d)\n", __LINE__); \
560: panic("PMAP_ACTIVATE_ASN_SANITY"); \
561: } \
562: } else { \
563: if ((pmap)->pm_asngen[(cpu_id)] != \
564: pmap_asn_generation[(cpu_id)]) { \
565: /* \
566: * ASN generation number isn't valid! \
567: */ \
568: printf("pmap asngen %lu, current %lu " \
569: "(line %d)\n", \
570: (pmap)->pm_asngen[(cpu_id)], \
571: pmap_asn_generation[(cpu_id)], \
572: __LINE__); \
573: panic("PMAP_ACTIVATE_ASN_SANITY"); \
574: } \
575: if ((pmap)->pm_asn[(cpu_id)] == PMAP_ASN_RESERVED) { \
576: /* \
577: * DANGER WILL ROBINSON! We're going to \
578: * pollute the VPT TLB entries! \
579: */ \
580: printf("Using reserved ASN! (line %d)\n", \
581: __LINE__); \
582: panic("PMAP_ACTIVATE_ASN_SANITY"); \
583: } \
584: } \
585: } while (0)
586: #else
587: #define PMAP_ACTIVATE_ASN_SANITY(pmap, cpu_id) /* nothing */
588: #endif
589:
590: /*
591: * PMAP_ACTIVATE:
592: *
593: * This is essentially the guts of pmap_activate(), without
594: * ASN allocation. This is used by pmap_activate(),
595: * pmap_lev1map_create(), and pmap_lev1map_destroy().
596: *
597: * This is called only when it is known that a pmap is "active"
598: * on the current processor; the ASN must already be valid.
599: */
600: #define PMAP_ACTIVATE(pmap, p, cpu_id) \
601: do { \
602: PMAP_ACTIVATE_ASN_SANITY(pmap, cpu_id); \
603: \
604: (p)->p_addr->u_pcb.pcb_hw.apcb_ptbr = \
605: ALPHA_K0SEG_TO_PHYS((vaddr_t)(pmap)->pm_lev1map) >> PGSHIFT; \
606: (p)->p_addr->u_pcb.pcb_hw.apcb_asn = (pmap)->pm_asn[(cpu_id)]; \
607: \
608: if ((p) == curproc) { \
609: /* \
610: * Page table base register has changed; switch to \
611: * our own context again so that it will take effect. \
612: */ \
613: (void) alpha_pal_swpctx((u_long)p->p_md.md_pcbpaddr); \
614: } \
615: } while (0)
616:
617: /*
618: * PMAP_SET_NEEDISYNC:
619: *
620: * Mark that a user pmap needs an I-stream synch on its
621: * way back out to userspace.
622: */
623: #define PMAP_SET_NEEDISYNC(pmap) (pmap)->pm_needisync = ~0UL
624:
625: /*
626: * PMAP_SYNC_ISTREAM:
627: *
628: * Synchronize the I-stream for the specified pmap. For user
629: * pmaps, this is deferred until a process using the pmap returns
630: * to userspace.
631: */
632: #if defined(MULTIPROCESSOR)
633: #define PMAP_SYNC_ISTREAM_KERNEL() \
634: do { \
635: alpha_pal_imb(); \
636: alpha_broadcast_ipi(ALPHA_IPI_IMB); \
637: } while (0)
638:
639: #define PMAP_SYNC_ISTREAM_USER(pmap) \
640: do { \
641: alpha_multicast_ipi((pmap)->pm_cpus, ALPHA_IPI_AST); \
642: /* for curcpu, will happen in userret() */ \
643: } while (0)
644: #else
645: #define PMAP_SYNC_ISTREAM_KERNEL() alpha_pal_imb()
646: #define PMAP_SYNC_ISTREAM_USER(pmap) /* will happen in userret() */
647: #endif /* MULTIPROCESSOR */
648:
649: #define PMAP_SYNC_ISTREAM(pmap) \
650: do { \
651: if ((pmap) == pmap_kernel()) \
652: PMAP_SYNC_ISTREAM_KERNEL(); \
653: else \
654: PMAP_SYNC_ISTREAM_USER(pmap); \
655: } while (0)
656:
657: /*
658: * PMAP_INVALIDATE_ASN:
659: *
660: * Invalidate the specified pmap's ASN, so as to force allocation
661: * of a new one the next time pmap_asn_alloc() is called.
662: *
663: * NOTE: THIS MUST ONLY BE CALLED IF AT LEAST ONE OF THE FOLLOWING
664: * CONDITIONS ARE TRUE:
665: *
666: * (1) The pmap references the global kernel_lev1map.
667: *
668: * (2) The pmap is not active on the current processor.
669: */
670: #define PMAP_INVALIDATE_ASN(pmap, cpu_id) \
671: do { \
672: (pmap)->pm_asn[(cpu_id)] = PMAP_ASN_RESERVED; \
673: } while (0)
674:
675: /*
676: * PMAP_INVALIDATE_TLB:
677: *
678: * Invalidate the TLB entry for the pmap/va pair.
679: */
680: #define PMAP_INVALIDATE_TLB(pmap, va, hadasm, isactive, cpu_id) \
681: do { \
682: if ((hadasm) || (isactive)) { \
683: /* \
684: * Simply invalidating the TLB entry and I-cache \
685: * works in this case. \
686: */ \
687: ALPHA_TBIS((va)); \
688: } else if ((pmap)->pm_asngen[(cpu_id)] == \
689: pmap_asn_generation[(cpu_id)]) { \
690: /* \
691: * We can't directly invalidate the TLB entry \
692: * in this case, so we have to force allocation \
693: * of a new ASN the next time this pmap becomes \
694: * active. \
695: */ \
696: PMAP_INVALIDATE_ASN((pmap), (cpu_id)); \
697: } \
698: /* \
699: * Nothing to do in this case; the next time the \
700: * pmap becomes active on this processor, a new \
701: * ASN will be allocated anyway. \
702: */ \
703: } while (0)
704:
705: /*
706: * PMAP_KERNEL_PTE:
707: *
708: * Get a kernel PTE.
709: *
710: * If debugging, do a table walk. If not debugging, just use
711: * the Virtual Page Table, since all kernel page tables are
712: * pre-allocated and mapped in.
713: */
714: #ifdef DEBUG
715: #define PMAP_KERNEL_PTE(va) \
716: ({ \
717: pt_entry_t *l1pte_, *l2pte_; \
718: \
719: l1pte_ = pmap_l1pte(pmap_kernel(), va); \
720: if (pmap_pte_v(l1pte_) == 0) { \
721: printf("kernel level 1 PTE not valid, va 0x%lx " \
722: "(line %d)\n", (va), __LINE__); \
723: panic("PMAP_KERNEL_PTE"); \
724: } \
725: l2pte_ = pmap_l2pte(pmap_kernel(), va, l1pte_); \
726: if (pmap_pte_v(l2pte_) == 0) { \
727: printf("kernel level 2 PTE not valid, va 0x%lx " \
728: "(line %d)\n", (va), __LINE__); \
729: panic("PMAP_KERNEL_PTE"); \
730: } \
731: pmap_l3pte(pmap_kernel(), va, l2pte_); \
732: })
733: #else
734: #define PMAP_KERNEL_PTE(va) (&VPT[VPT_INDEX((va))])
735: #endif
736:
737: /*
738: * PMAP_SET_PTE:
739: *
740: * Set a PTE to a specified value.
741: */
742: #define PMAP_SET_PTE(ptep, val) *(ptep) = (val)
743:
744: /*
745: * PMAP_STAT_{INCR,DECR}:
746: *
747: * Increment or decrement a pmap statistic.
748: */
749: #define PMAP_STAT_INCR(s, v) atomic_add_ulong((unsigned long *)(&(s)), (v))
750: #define PMAP_STAT_DECR(s, v) atomic_sub_ulong((unsigned long *)(&(s)), (v))
751:
752: /*
753: * pmap_bootstrap:
754: *
755: * Bootstrap the system to run with virtual memory.
756: *
757: * Note: no locking is necessary in this function.
758: */
759: void
760: pmap_bootstrap(paddr_t ptaddr, u_int maxasn, u_long ncpuids)
761: {
762: vsize_t lev2mapsize, lev3mapsize;
763: pt_entry_t *lev2map, *lev3map;
764: pt_entry_t pte;
765: int i;
766:
767: #ifdef DEBUG
768: if (pmapdebug & (PDB_FOLLOW|PDB_BOOTSTRAP))
769: printf("pmap_bootstrap(0x%lx, %u)\n", ptaddr, maxasn);
770: #endif
771:
772: /*
773: * Compute the number of pages kmem_map will have.
774: */
775: kmeminit_nkmempages();
776:
777: /*
778: * Figure out how many PTE's are necessary to map the kernel.
779: */
780: lev3mapsize = (VM_PHYS_SIZE + 16 * NCARGS + PAGER_MAP_SIZE) /
781: PAGE_SIZE + (maxproc * UPAGES) + nkmempages;
782:
783: #ifdef SYSVSHM
784: lev3mapsize += shminfo.shmall;
785: #endif
786: lev3mapsize = roundup(lev3mapsize, NPTEPG);
787:
788: /*
789: * Allocate a level 1 PTE table for the kernel.
790: * This is always one page long.
791: * IF THIS IS NOT A MULTIPLE OF PAGE_SIZE, ALL WILL GO TO HELL.
792: */
793: kernel_lev1map = (pt_entry_t *)
794: pmap_steal_memory(sizeof(pt_entry_t) * NPTEPG, NULL, NULL);
795:
796: /*
797: * Allocate a level 2 PTE table for the kernel.
798: * These must map all of the level3 PTEs.
799: * IF THIS IS NOT A MULTIPLE OF PAGE_SIZE, ALL WILL GO TO HELL.
800: */
801: lev2mapsize = roundup(howmany(lev3mapsize, NPTEPG), NPTEPG);
802: lev2map = (pt_entry_t *)
803: pmap_steal_memory(sizeof(pt_entry_t) * lev2mapsize, NULL, NULL);
804:
805: /*
806: * Allocate a level 3 PTE table for the kernel.
807: * Contains lev3mapsize PTEs.
808: */
809: lev3map = (pt_entry_t *)
810: pmap_steal_memory(sizeof(pt_entry_t) * lev3mapsize, NULL, NULL);
811:
812: /*
813: * Allocate memory for the pv_heads. (A few more of the latter
814: * are allocated than are needed.)
815: *
816: * We could do this in pmap_init when we know the actual
817: * managed page pool size, but its better to use kseg0
818: * addresses rather than kernel virtual addresses mapped
819: * through the TLB.
820: */
821: pv_table_npages = physmem;
822: pv_table = (struct pv_head *)
823: pmap_steal_memory(sizeof(struct pv_head) * pv_table_npages,
824: NULL, NULL);
825:
826: /*
827: * ...and initialize the pv_entry list headers.
828: */
829: for (i = 0; i < pv_table_npages; i++) {
830: LIST_INIT(&pv_table[i].pvh_list);
831: simple_lock_init(&pv_table[i].pvh_slock);
832: }
833:
834: /*
835: * Set up level 1 page table
836: */
837:
838: /* Map all of the level 2 pte pages */
839: for (i = 0; i < howmany(lev2mapsize, NPTEPG); i++) {
840: pte = (ALPHA_K0SEG_TO_PHYS(((vaddr_t)lev2map) +
841: (i*PAGE_SIZE)) >> PGSHIFT) << PG_SHIFT;
842: pte |= PG_V | PG_ASM | PG_KRE | PG_KWE | PG_WIRED;
843: kernel_lev1map[l1pte_index(VM_MIN_KERNEL_ADDRESS +
844: (i*PAGE_SIZE*NPTEPG*NPTEPG))] = pte;
845: }
846:
847: /* Map the virtual page table */
848: pte = (ALPHA_K0SEG_TO_PHYS((vaddr_t)kernel_lev1map) >> PGSHIFT)
849: << PG_SHIFT;
850: pte |= PG_V | PG_KRE | PG_KWE; /* NOTE NO ASM */
851: kernel_lev1map[l1pte_index(VPTBASE)] = pte;
852: VPT = (pt_entry_t *)VPTBASE;
853:
854: #ifdef _PMAP_MAY_USE_PROM_CONSOLE
855: {
856: extern pt_entry_t prom_pte; /* XXX */
857: extern int prom_mapped; /* XXX */
858:
859: if (pmap_uses_prom_console()) {
860: /*
861: * XXX Save old PTE so we can remap the PROM, if
862: * XXX necessary.
863: */
864: prom_pte = *(pt_entry_t *)ptaddr & ~PG_ASM;
865: }
866: prom_mapped = 0;
867:
868: /*
869: * Actually, this code lies. The prom is still mapped, and will
870: * remain so until the context switch after alpha_init() returns.
871: */
872: }
873: #endif
874:
875: /*
876: * Set up level 2 page table.
877: */
878: /* Map all of the level 3 pte pages */
879: for (i = 0; i < howmany(lev3mapsize, NPTEPG); i++) {
880: pte = (ALPHA_K0SEG_TO_PHYS(((vaddr_t)lev3map) +
881: (i*PAGE_SIZE)) >> PGSHIFT) << PG_SHIFT;
882: pte |= PG_V | PG_ASM | PG_KRE | PG_KWE | PG_WIRED;
883: lev2map[l2pte_index(VM_MIN_KERNEL_ADDRESS+
884: (i*PAGE_SIZE*NPTEPG))] = pte;
885: }
886:
887: /* Initialize the pmap_growkernel_slock. */
888: simple_lock_init(&pmap_growkernel_slock);
889:
890: /*
891: * Set up level three page table (lev3map)
892: */
893: /* Nothing to do; it's already zeroed */
894:
895: /*
896: * Initialize `FYI' variables. Note we're relying on
897: * the fact that BSEARCH sorts the vm_physmem[] array
898: * for us.
899: */
900: avail_start = ptoa(vm_physmem[0].start);
901: avail_end = ptoa(vm_physmem[vm_nphysseg - 1].end);
902: virtual_avail = VM_MIN_KERNEL_ADDRESS;
903: virtual_end = VM_MIN_KERNEL_ADDRESS + lev3mapsize * PAGE_SIZE;
904:
905: #if 0
906: printf("avail_start = 0x%lx\n", avail_start);
907: printf("avail_end = 0x%lx\n", avail_end);
908: printf("virtual_avail = 0x%lx\n", virtual_avail);
909: printf("virtual_end = 0x%lx\n", virtual_end);
910: #endif
911:
912: /*
913: * Initialize the pmap pools and list.
914: */
915: pmap_ncpuids = ncpuids;
916: pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl",
917: &pool_allocator_nointr);
918: pool_init(&pmap_l1pt_pool, PAGE_SIZE, 0, 0, 0, "l1ptpl",
919: &pmap_l1pt_allocator);
920: pool_cache_init(&pmap_l1pt_cache, &pmap_l1pt_pool, pmap_l1pt_ctor,
921: NULL, NULL);
922: pool_init(&pmap_asn_pool, pmap_ncpuids * sizeof(u_int), 0, 0, 0,
923: "pmasnpl", &pool_allocator_nointr);
924: pool_init(&pmap_asngen_pool, pmap_ncpuids * sizeof(u_long), 0, 0, 0,
925: "pmasngenpl", &pool_allocator_nointr);
926: pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
927: &pmap_pv_allocator);
928:
929: TAILQ_INIT(&pmap_all_pmaps);
930:
931: /*
932: * Initialize the ASN logic.
933: */
934: pmap_max_asn = maxasn;
935: for (i = 0; i < ALPHA_MAXPROCS; i++) {
936: pmap_next_asn[i] = 1;
937: pmap_asn_generation[i] = 0;
938: }
939:
940: /*
941: * Initialize the locks.
942: */
943: simple_lock_init(&pmap_all_pmaps_slock);
944:
945: /*
946: * Initialize kernel pmap. Note that all kernel mappings
947: * have PG_ASM set, so the ASN doesn't really matter for
948: * the kernel pmap. Also, since the kernel pmap always
949: * references kernel_lev1map, it always has an invalid ASN
950: * generation.
951: */
952: memset(pmap_kernel(), 0, sizeof(struct pmap));
953: pmap_kernel()->pm_lev1map = kernel_lev1map;
954: pmap_kernel()->pm_count = 1;
955: pmap_kernel()->pm_asn = kernel_pmap_asn_store;
956: pmap_kernel()->pm_asngen = kernel_pmap_asngen_store;
957: for (i = 0; i < ALPHA_MAXPROCS; i++) {
958: pmap_kernel()->pm_asn[i] = PMAP_ASN_RESERVED;
959: pmap_kernel()->pm_asngen[i] = pmap_asn_generation[i];
960: }
961: simple_lock_init(&pmap_kernel()->pm_slock);
962: TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list);
963:
964: #if defined(MULTIPROCESSOR)
965: /*
966: * Initialize the TLB shootdown queues.
967: */
968: pool_init(&pmap_tlb_shootdown_job_pool,
969: sizeof(struct pmap_tlb_shootdown_job), 0, 0, 0, "pmaptlbpl",
970: NULL);
971: for (i = 0; i < ALPHA_MAXPROCS; i++) {
972: TAILQ_INIT(&pmap_tlb_shootdown_q[i].pq_head);
973: simple_lock_init(&pmap_tlb_shootdown_q[i].pq_slock);
974: }
975: #endif
976:
977: /*
978: * Set up proc0's PCB such that the ptbr points to the right place
979: * and has the kernel pmap's (really unused) ASN.
980: */
981: proc0.p_addr->u_pcb.pcb_hw.apcb_ptbr =
982: ALPHA_K0SEG_TO_PHYS((vaddr_t)kernel_lev1map) >> PGSHIFT;
983: proc0.p_addr->u_pcb.pcb_hw.apcb_asn =
984: pmap_kernel()->pm_asn[cpu_number()];
985:
986: /*
987: * Mark the kernel pmap `active' on this processor.
988: */
989: atomic_setbits_ulong(&pmap_kernel()->pm_cpus,
990: (1UL << cpu_number()));
991: }
992:
993: #ifdef _PMAP_MAY_USE_PROM_CONSOLE
994: int
995: pmap_uses_prom_console(void)
996: {
997:
998: #if defined(NEW_SCC_DRIVER)
999: return (cputype == ST_DEC_21000);
1000: #else
1001: return (cputype == ST_DEC_21000
1002: || cputype == ST_DEC_3000_300
1003: || cputype == ST_DEC_3000_500);
1004: #endif /* NEW_SCC_DRIVER */
1005: }
1006: #endif /* _PMAP_MAY_USE_PROM_CONSOLE */
1007:
1008: void
1009: pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
1010: {
1011: *vstartp = VM_MIN_KERNEL_ADDRESS;
1012: *vendp = VM_MAX_KERNEL_ADDRESS;
1013: }
1014:
1015: /*
1016: * pmap_steal_memory: [ INTERFACE ]
1017: *
1018: * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()).
1019: * This function allows for early dynamic memory allocation until the
1020: * virtual memory system has been bootstrapped. After that point, either
1021: * kmem_alloc or malloc should be used. This function works by stealing
1022: * pages from the (to be) managed page pool, then implicitly mapping the
1023: * pages (by using their k0seg addresses) and zeroing them.
1024: *
1025: * It may be used once the physical memory segments have been pre-loaded
1026: * into the vm_physmem[] array. Early memory allocation MUST use this
1027: * interface! This cannot be used after vm_page_startup(), and will
1028: * generate a panic if tried.
1029: *
1030: * Note that this memory will never be freed, and in essence it is wired
1031: * down.
1032: *
1033: * Note: no locking is necessary in this function.
1034: */
1035: vaddr_t
1036: pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp)
1037: {
1038: int bank, npgs, x;
1039: vaddr_t va;
1040: paddr_t pa;
1041:
1042: size = round_page(size);
1043: npgs = atop(size);
1044:
1045: #if 0
1046: printf("PSM: size 0x%lx (npgs 0x%x)\n", size, npgs);
1047: #endif
1048:
1049: for (bank = 0; bank < vm_nphysseg; bank++) {
1050: if (uvm.page_init_done == TRUE)
1051: panic("pmap_steal_memory: called _after_ bootstrap");
1052:
1053:
1054: #if 0
1055: printf(" bank %d: avail_start 0x%lx, start 0x%lx, "
1056: "avail_end 0x%lx\n", bank, vm_physmem[bank].avail_start,
1057: vm_physmem[bank].start, vm_physmem[bank].avail_end);
1058: #endif
1059:
1060: if (vm_physmem[bank].avail_start != vm_physmem[bank].start ||
1061: vm_physmem[bank].avail_start >= vm_physmem[bank].avail_end)
1062: continue;
1063:
1064: #if 0
1065: printf(" avail_end - avail_start = 0x%lx\n",
1066: vm_physmem[bank].avail_end - vm_physmem[bank].avail_start);
1067: #endif
1068:
1069: if ((vm_physmem[bank].avail_end - vm_physmem[bank].avail_start)
1070: < npgs)
1071: continue;
1072:
1073: /*
1074: * There are enough pages here; steal them!
1075: */
1076: pa = ptoa(vm_physmem[bank].avail_start);
1077: vm_physmem[bank].avail_start += npgs;
1078: vm_physmem[bank].start += npgs;
1079:
1080: /*
1081: * Have we used up this segment?
1082: */
1083: if (vm_physmem[bank].avail_start == vm_physmem[bank].end) {
1084: if (vm_nphysseg == 1)
1085: panic("pmap_steal_memory: out of memory!");
1086:
1087: /* Remove this segment from the list. */
1088: vm_nphysseg--;
1089: for (x = bank; x < vm_nphysseg; x++) {
1090: /* structure copy */
1091: vm_physmem[x] = vm_physmem[x + 1];
1092: }
1093: }
1094:
1095: /*
1096: * Fill these in for the caller; we don't modify them,
1097: * but the upper layers still want to know.
1098: */
1099: if (vstartp)
1100: *vstartp = round_page(virtual_avail);
1101: if (vendp)
1102: *vendp = VM_MAX_KERNEL_ADDRESS;
1103:
1104: va = ALPHA_PHYS_TO_K0SEG(pa);
1105: memset((caddr_t)va, 0, size);
1106: pmap_pages_stolen += npgs;
1107: return (va);
1108: }
1109:
1110: /*
1111: * If we got here, this was no memory left.
1112: */
1113: panic("pmap_steal_memory: no memory to steal");
1114: }
1115:
1116: /*
1117: * pmap_init: [ INTERFACE ]
1118: *
1119: * Initialize the pmap module. Called by vm_init(), to initialize any
1120: * structures that the pmap system needs to map virtual memory.
1121: *
1122: * Note: no locking is necessary in this function.
1123: */
1124: void
1125: pmap_init(void)
1126: {
1127: vsize_t s;
1128: int bank;
1129: struct pv_head *pvh;
1130:
1131: #ifdef DEBUG
1132: if (pmapdebug & PDB_FOLLOW)
1133: printf("pmap_init()\n");
1134: #endif
1135:
1136: /* initialize protection array */
1137: alpha_protection_init();
1138:
1139: /*
1140: * Memory for the pv heads has already been allocated.
1141: * Initialize the physical memory segments.
1142: */
1143: pvh = pv_table;
1144: for (bank = 0; bank < vm_nphysseg; bank++) {
1145: s = vm_physmem[bank].end - vm_physmem[bank].start;
1146: vm_physmem[bank].pmseg.pvhead = pvh;
1147: pvh += s;
1148: }
1149:
1150: /*
1151: * Set a low water mark on the pv_entry pool, so that we are
1152: * more likely to have these around even in extreme memory
1153: * starvation.
1154: */
1155: pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
1156:
1157: /*
1158: * Now it is safe to enable pv entry recording.
1159: */
1160: pmap_initialized = TRUE;
1161:
1162: #if 0
1163: for (bank = 0; bank < vm_nphysseg; bank++) {
1164: printf("bank %d\n", bank);
1165: printf("\tstart = 0x%x\n", ptoa(vm_physmem[bank].start));
1166: printf("\tend = 0x%x\n", ptoa(vm_physmem[bank].end));
1167: printf("\tavail_start = 0x%x\n",
1168: ptoa(vm_physmem[bank].avail_start));
1169: printf("\tavail_end = 0x%x\n",
1170: ptoa(vm_physmem[bank].avail_end));
1171: }
1172: #endif
1173: }
1174:
1175: /*
1176: * pmap_create: [ INTERFACE ]
1177: *
1178: * Create and return a physical map.
1179: *
1180: * Note: no locking is necessary in this function.
1181: */
1182: pmap_t
1183: pmap_create(void)
1184: {
1185: pmap_t pmap;
1186: int i;
1187:
1188: #ifdef DEBUG
1189: if (pmapdebug & (PDB_FOLLOW|PDB_CREATE))
1190: printf("pmap_create()\n");
1191: #endif
1192:
1193: pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
1194: memset(pmap, 0, sizeof(*pmap));
1195:
1196: pmap->pm_asn = pool_get(&pmap_asn_pool, PR_WAITOK);
1197: pmap->pm_asngen = pool_get(&pmap_asngen_pool, PR_WAITOK);
1198:
1199: /*
1200: * Defer allocation of a new level 1 page table until
1201: * the first new mapping is entered; just take a reference
1202: * to the kernel kernel_lev1map.
1203: */
1204: pmap->pm_lev1map = kernel_lev1map;
1205:
1206: pmap->pm_count = 1;
1207: for (i = 0; i < pmap_ncpuids; i++) {
1208: pmap->pm_asn[i] = PMAP_ASN_RESERVED;
1209: /* XXX Locking? */
1210: pmap->pm_asngen[i] = pmap_asn_generation[i];
1211: }
1212: simple_lock_init(&pmap->pm_slock);
1213:
1214: simple_lock(&pmap_all_pmaps_slock);
1215: TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap, pm_list);
1216: simple_unlock(&pmap_all_pmaps_slock);
1217:
1218: return (pmap);
1219: }
1220:
1221: /*
1222: * pmap_destroy: [ INTERFACE ]
1223: *
1224: * Drop the reference count on the specified pmap, releasing
1225: * all resources if the reference count drops to zero.
1226: */
1227: void
1228: pmap_destroy(pmap_t pmap)
1229: {
1230: int refs;
1231:
1232: #ifdef DEBUG
1233: if (pmapdebug & PDB_FOLLOW)
1234: printf("pmap_destroy(%p)\n", pmap);
1235: #endif
1236: if (pmap == NULL)
1237: return;
1238:
1239: PMAP_LOCK(pmap);
1240: refs = --pmap->pm_count;
1241: PMAP_UNLOCK(pmap);
1242:
1243: if (refs > 0)
1244: return;
1245:
1246: /*
1247: * Remove it from the global list of all pmaps.
1248: */
1249: simple_lock(&pmap_all_pmaps_slock);
1250: TAILQ_REMOVE(&pmap_all_pmaps, pmap, pm_list);
1251: simple_unlock(&pmap_all_pmaps_slock);
1252:
1253: #ifdef DIAGNOSTIC
1254: /*
1255: * Since the pmap is supposed to contain no valid
1256: * mappings at this point, this should never happen.
1257: */
1258: if (pmap->pm_lev1map != kernel_lev1map) {
1259: printf("pmap_release: pmap still contains valid mappings!\n");
1260: if (pmap->pm_nlev2)
1261: printf("pmap_release: %ld level 2 tables left\n",
1262: pmap->pm_nlev2);
1263: if (pmap->pm_nlev3)
1264: printf("pmap_release: %ld level 3 tables left\n",
1265: pmap->pm_nlev3);
1266: pmap_remove(pmap, VM_MIN_ADDRESS, VM_MAX_ADDRESS);
1267: pmap_update(pmap);
1268: if (pmap->pm_lev1map != kernel_lev1map)
1269: panic("pmap_release: pmap_remove() didn't");
1270: }
1271: #endif
1272:
1273: pool_put(&pmap_asn_pool, pmap->pm_asn);
1274: pool_put(&pmap_asngen_pool, pmap->pm_asngen);
1275: pool_put(&pmap_pmap_pool, pmap);
1276: }
1277:
1278: /*
1279: * pmap_reference: [ INTERFACE ]
1280: *
1281: * Add a reference to the specified pmap.
1282: */
1283: void
1284: pmap_reference(pmap_t pmap)
1285: {
1286:
1287: #ifdef DEBUG
1288: if (pmapdebug & PDB_FOLLOW)
1289: printf("pmap_reference(%p)\n", pmap);
1290: #endif
1291: if (pmap != NULL) {
1292: PMAP_LOCK(pmap);
1293: pmap->pm_count++;
1294: PMAP_UNLOCK(pmap);
1295: }
1296: }
1297:
1298: /*
1299: * pmap_remove: [ INTERFACE ]
1300: *
1301: * Remove the given range of addresses from the specified map.
1302: *
1303: * It is assumed that the start and end are properly
1304: * rounded to the page size.
1305: */
1306: void
1307: pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
1308: {
1309:
1310: #ifdef DEBUG
1311: if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
1312: printf("pmap_remove(%p, %lx, %lx)\n", pmap, sva, eva);
1313: #endif
1314:
1315: pmap_do_remove(pmap, sva, eva, TRUE);
1316: }
1317:
1318: /*
1319: * pmap_do_remove:
1320: *
1321: * This actually removes the range of addresses from the
1322: * specified map. It is used by pmap_collect() (does not
1323: * want to remove wired mappings) and pmap_remove() (does
1324: * want to remove wired mappings).
1325: */
1326: void
1327: pmap_do_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, boolean_t dowired)
1328: {
1329: pt_entry_t *l1pte, *l2pte, *l3pte;
1330: pt_entry_t *saved_l1pte, *saved_l2pte, *saved_l3pte;
1331: vaddr_t l1eva, l2eva, vptva;
1332: boolean_t needisync = FALSE;
1333: cpuid_t cpu_id = cpu_number();
1334:
1335: #ifdef DEBUG
1336: if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
1337: printf("pmap_remove(%p, %lx, %lx)\n", pmap, sva, eva);
1338: #endif
1339:
1340: if (pmap == NULL)
1341: return;
1342:
1343: /*
1344: * If this is the kernel pmap, we can use a faster method
1345: * for accessing the PTEs (since the PT pages are always
1346: * resident).
1347: *
1348: * Note that this routine should NEVER be called from an
1349: * interrupt context; pmap_kremove() is used for that.
1350: */
1351: if (pmap == pmap_kernel()) {
1352: PMAP_MAP_TO_HEAD_LOCK();
1353: PMAP_LOCK(pmap);
1354:
1355: KASSERT(dowired == TRUE);
1356:
1357: while (sva < eva) {
1358: l3pte = PMAP_KERNEL_PTE(sva);
1359: if (pmap_pte_v(l3pte)) {
1360: #ifdef DIAGNOSTIC
1361: if (PAGE_IS_MANAGED(pmap_pte_pa(l3pte)) &&
1362: pmap_pte_pv(l3pte) == 0)
1363: panic("pmap_remove: managed page "
1364: "without PG_PVLIST for 0x%lx",
1365: sva);
1366: #endif
1367: needisync |= pmap_remove_mapping(pmap, sva,
1368: l3pte, TRUE, cpu_id, NULL);
1369: }
1370: sva += PAGE_SIZE;
1371: }
1372:
1373: PMAP_UNLOCK(pmap);
1374: PMAP_MAP_TO_HEAD_UNLOCK();
1375:
1376: if (needisync)
1377: PMAP_SYNC_ISTREAM_KERNEL();
1378: return;
1379: }
1380:
1381: #ifdef DIAGNOSTIC
1382: if (sva > VM_MAXUSER_ADDRESS || eva > VM_MAXUSER_ADDRESS)
1383: panic("pmap_remove: (0x%lx - 0x%lx) user pmap, kernel "
1384: "address range", sva, eva);
1385: #endif
1386:
1387: PMAP_MAP_TO_HEAD_LOCK();
1388: PMAP_LOCK(pmap);
1389:
1390: /*
1391: * If we're already referencing the kernel_lev1map, there
1392: * is no work for us to do.
1393: */
1394: if (pmap->pm_lev1map == kernel_lev1map)
1395: goto out;
1396:
1397: saved_l1pte = l1pte = pmap_l1pte(pmap, sva);
1398:
1399: /*
1400: * Add a reference to the L1 table to it won't get
1401: * removed from under us.
1402: */
1403: pmap_physpage_addref(saved_l1pte);
1404:
1405: for (; sva < eva; sva = l1eva, l1pte++) {
1406: l1eva = alpha_trunc_l1seg(sva) + ALPHA_L1SEG_SIZE;
1407: if (pmap_pte_v(l1pte)) {
1408: saved_l2pte = l2pte = pmap_l2pte(pmap, sva, l1pte);
1409:
1410: /*
1411: * Add a reference to the L2 table so it won't
1412: * get removed from under us.
1413: */
1414: pmap_physpage_addref(saved_l2pte);
1415:
1416: for (; sva < l1eva && sva < eva; sva = l2eva, l2pte++) {
1417: l2eva =
1418: alpha_trunc_l2seg(sva) + ALPHA_L2SEG_SIZE;
1419: if (pmap_pte_v(l2pte)) {
1420: saved_l3pte = l3pte =
1421: pmap_l3pte(pmap, sva, l2pte);
1422:
1423: /*
1424: * Add a reference to the L3 table so
1425: * it won't get removed from under us.
1426: */
1427: pmap_physpage_addref(saved_l3pte);
1428:
1429: /*
1430: * Remember this sva; if the L3 table
1431: * gets removed, we need to invalidate
1432: * the VPT TLB entry for it.
1433: */
1434: vptva = sva;
1435:
1436: for (; sva < l2eva && sva < eva;
1437: sva += PAGE_SIZE, l3pte++) {
1438: if (pmap_pte_v(l3pte) &&
1439: (dowired == TRUE ||
1440: pmap_pte_w(l3pte) == 0)) {
1441: needisync |=
1442: pmap_remove_mapping(
1443: pmap, sva,
1444: l3pte, TRUE,
1445: cpu_id, NULL);
1446: }
1447: }
1448:
1449: /*
1450: * Remove the reference to the L3
1451: * table that we added above. This
1452: * may free the L3 table.
1453: */
1454: pmap_l3pt_delref(pmap, vptva,
1455: saved_l3pte, cpu_id, NULL);
1456: }
1457: }
1458:
1459: /*
1460: * Remove the reference to the L2 table that we
1461: * added above. This may free the L2 table.
1462: */
1463: pmap_l2pt_delref(pmap, l1pte, saved_l2pte, cpu_id);
1464: }
1465: }
1466:
1467: /*
1468: * Remove the reference to the L1 table that we added above.
1469: * This may free the L1 table.
1470: */
1471: pmap_l1pt_delref(pmap, saved_l1pte, cpu_id);
1472:
1473: if (needisync)
1474: PMAP_SYNC_ISTREAM_USER(pmap);
1475:
1476: out:
1477: PMAP_UNLOCK(pmap);
1478: PMAP_MAP_TO_HEAD_UNLOCK();
1479: }
1480:
1481: /*
1482: * pmap_page_protect: [ INTERFACE ]
1483: *
1484: * Lower the permission for all mappings to a given page to
1485: * the permissions specified.
1486: */
1487: void
1488: pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
1489: {
1490: pmap_t pmap;
1491: struct pv_head *pvh;
1492: pv_entry_t pv, nextpv;
1493: boolean_t needkisync = FALSE;
1494: cpuid_t cpu_id = cpu_number();
1495: paddr_t pa = VM_PAGE_TO_PHYS(pg);
1496:
1497: #ifdef DEBUG
1498: if ((pmapdebug & (PDB_FOLLOW|PDB_PROTECT)) ||
1499: (prot == VM_PROT_NONE && (pmapdebug & PDB_REMOVE)))
1500: printf("pmap_page_protect(%p, %x)\n", pg, prot);
1501: #endif
1502:
1503: switch (prot) {
1504: case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE:
1505: case VM_PROT_READ|VM_PROT_WRITE:
1506: return;
1507: /* copy_on_write */
1508: case VM_PROT_READ|VM_PROT_EXECUTE:
1509: case VM_PROT_READ:
1510: pvh = pa_to_pvh(pa);
1511: PMAP_HEAD_TO_MAP_LOCK();
1512: simple_lock(&pvh->pvh_slock);
1513: /* XXX */ pmap_changebit(pa, 0, ~(PG_KWE | PG_UWE), cpu_id);
1514: simple_unlock(&pvh->pvh_slock);
1515: PMAP_HEAD_TO_MAP_UNLOCK();
1516: return;
1517: /* remove_all */
1518: default:
1519: break;
1520: }
1521:
1522: pvh = pa_to_pvh(pa);
1523: PMAP_HEAD_TO_MAP_LOCK();
1524: simple_lock(&pvh->pvh_slock);
1525: for (pv = LIST_FIRST(&pvh->pvh_list); pv != NULL; pv = nextpv) {
1526: nextpv = LIST_NEXT(pv, pv_list);
1527: pmap = pv->pv_pmap;
1528:
1529: PMAP_LOCK(pmap);
1530: #ifdef DEBUG
1531: if (pmap_pte_v(pmap_l2pte(pv->pv_pmap, pv->pv_va, NULL)) == 0 ||
1532: pmap_pte_pa(pv->pv_pte) != pa)
1533: panic("pmap_page_protect: bad mapping");
1534: #endif
1535: if (pmap_pte_w(pv->pv_pte) == 0) {
1536: if (pmap_remove_mapping(pmap, pv->pv_va, pv->pv_pte,
1537: FALSE, cpu_id, NULL) == TRUE) {
1538: if (pmap == pmap_kernel())
1539: needkisync |= TRUE;
1540: else
1541: PMAP_SYNC_ISTREAM_USER(pmap);
1542: }
1543: }
1544: #ifdef DEBUG
1545: else {
1546: if (pmapdebug & PDB_PARANOIA) {
1547: printf("%s wired mapping for %lx not removed\n",
1548: "pmap_page_protect:", pa);
1549: printf("vm wire count %d\n",
1550: PHYS_TO_VM_PAGE(pa)->wire_count);
1551: }
1552: }
1553: #endif
1554: PMAP_UNLOCK(pmap);
1555: }
1556:
1557: if (needkisync)
1558: PMAP_SYNC_ISTREAM_KERNEL();
1559:
1560: simple_unlock(&pvh->pvh_slock);
1561: PMAP_HEAD_TO_MAP_UNLOCK();
1562: }
1563:
1564: /*
1565: * pmap_protect: [ INTERFACE ]
1566: *
1567: * Set the physical protection on the specified range of this map
1568: * as requested.
1569: */
1570: void
1571: pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1572: {
1573: pt_entry_t *l1pte, *l2pte, *l3pte, bits;
1574: boolean_t isactive;
1575: boolean_t hadasm;
1576: vaddr_t l1eva, l2eva;
1577: cpuid_t cpu_id = cpu_number();
1578:
1579: #ifdef DEBUG
1580: if (pmapdebug & (PDB_FOLLOW|PDB_PROTECT))
1581: printf("pmap_protect(%p, %lx, %lx, %x)\n",
1582: pmap, sva, eva, prot);
1583: #endif
1584:
1585: if (pmap == NULL)
1586: return;
1587:
1588: if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
1589: pmap_remove(pmap, sva, eva);
1590: return;
1591: }
1592:
1593: if (prot & VM_PROT_WRITE)
1594: return;
1595:
1596: PMAP_LOCK(pmap);
1597:
1598: bits = pte_prot(pmap, prot);
1599: if (!pmap_pte_exec(&bits))
1600: bits |= PG_FOE;
1601: isactive = PMAP_ISACTIVE(pmap, cpu_id);
1602:
1603: l1pte = pmap_l1pte(pmap, sva);
1604: for (; sva < eva; sva = l1eva, l1pte++) {
1605: l1eva = alpha_trunc_l1seg(sva) + ALPHA_L1SEG_SIZE;
1606: if (pmap_pte_v(l1pte)) {
1607: l2pte = pmap_l2pte(pmap, sva, l1pte);
1608: for (; sva < l1eva && sva < eva; sva = l2eva, l2pte++) {
1609: l2eva =
1610: alpha_trunc_l2seg(sva) + ALPHA_L2SEG_SIZE;
1611: if (pmap_pte_v(l2pte)) {
1612: l3pte = pmap_l3pte(pmap, sva, l2pte);
1613: for (; sva < l2eva && sva < eva;
1614: sva += PAGE_SIZE, l3pte++) {
1615: if (pmap_pte_v(l3pte) &&
1616: pmap_pte_prot_chg(l3pte,
1617: bits)) {
1618: hadasm =
1619: (pmap_pte_asm(l3pte)
1620: != 0);
1621: pmap_pte_set_prot(l3pte,
1622: bits);
1623: PMAP_INVALIDATE_TLB(
1624: pmap, sva, hadasm,
1625: isactive, cpu_id);
1626: PMAP_TLB_SHOOTDOWN(
1627: pmap, sva,
1628: hadasm ? PG_ASM : 0);
1629: }
1630: }
1631: }
1632: }
1633: }
1634: }
1635:
1636: if (prot & VM_PROT_EXECUTE)
1637: PMAP_SYNC_ISTREAM(pmap);
1638:
1639: PMAP_UNLOCK(pmap);
1640: }
1641:
1642: /*
1643: * pmap_enter: [ INTERFACE ]
1644: *
1645: * Insert the given physical page (p) at
1646: * the specified virtual address (v) in the
1647: * target physical map with the protection requested.
1648: *
1649: * If specified, the page will be wired down, meaning
1650: * that the related pte can not be reclaimed.
1651: *
1652: * Note: This is the only routine which MAY NOT lazy-evaluate
1653: * or lose information. That is, this routine must actually
1654: * insert this page into the given map NOW.
1655: */
1656: int
1657: pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
1658: {
1659: boolean_t managed;
1660: pt_entry_t *pte, npte, opte;
1661: paddr_t opa;
1662: boolean_t tflush = TRUE;
1663: boolean_t hadasm = FALSE; /* XXX gcc -Wuninitialized */
1664: boolean_t needisync = FALSE;
1665: boolean_t setisync = FALSE;
1666: boolean_t isactive;
1667: boolean_t wired;
1668: cpuid_t cpu_id = cpu_number();
1669: int error = 0;
1670:
1671: #ifdef DEBUG
1672: if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
1673: printf("pmap_enter(%p, %lx, %lx, %x, %x)\n",
1674: pmap, va, pa, prot, flags);
1675: #endif
1676: managed = PAGE_IS_MANAGED(pa);
1677: isactive = PMAP_ISACTIVE(pmap, cpu_id);
1678: wired = (flags & PMAP_WIRED) != 0;
1679:
1680: /*
1681: * Determine what we need to do about the I-stream. If
1682: * VM_PROT_EXECUTE is set, we mark a user pmap as needing
1683: * an I-sync on the way back out to userspace. We always
1684: * need an immediate I-sync for the kernel pmap.
1685: */
1686: if (prot & VM_PROT_EXECUTE) {
1687: if (pmap == pmap_kernel())
1688: needisync = TRUE;
1689: else {
1690: setisync = TRUE;
1691: needisync = (pmap->pm_cpus != 0);
1692: }
1693: }
1694:
1695: PMAP_MAP_TO_HEAD_LOCK();
1696: PMAP_LOCK(pmap);
1697:
1698: if (pmap == pmap_kernel()) {
1699: #ifdef DIAGNOSTIC
1700: /*
1701: * Sanity check the virtual address.
1702: */
1703: if (va < VM_MIN_KERNEL_ADDRESS)
1704: panic("pmap_enter: kernel pmap, invalid va 0x%lx", va);
1705: #endif
1706: pte = PMAP_KERNEL_PTE(va);
1707: } else {
1708: pt_entry_t *l1pte, *l2pte;
1709:
1710: #ifdef DIAGNOSTIC
1711: /*
1712: * Sanity check the virtual address.
1713: */
1714: if (va >= VM_MAXUSER_ADDRESS)
1715: panic("pmap_enter: user pmap, invalid va 0x%lx", va);
1716: #endif
1717:
1718: /*
1719: * If we're still referencing the kernel kernel_lev1map,
1720: * create a new level 1 page table. A reference will be
1721: * added to the level 1 table when the level 2 table is
1722: * created.
1723: */
1724: if (pmap->pm_lev1map == kernel_lev1map) {
1725: error = pmap_lev1map_create(pmap, cpu_id);
1726: if (error) {
1727: if (flags & PMAP_CANFAIL)
1728: goto out;
1729: panic("pmap_enter: unable to create lev1map");
1730: }
1731: }
1732:
1733: /*
1734: * Check to see if the level 1 PTE is valid, and
1735: * allocate a new level 2 page table page if it's not.
1736: * A reference will be added to the level 2 table when
1737: * the level 3 table is created.
1738: */
1739: l1pte = pmap_l1pte(pmap, va);
1740: if (pmap_pte_v(l1pte) == 0) {
1741: pmap_physpage_addref(l1pte);
1742: error = pmap_ptpage_alloc(pmap, l1pte, PGU_L2PT);
1743: if (error) {
1744: pmap_l1pt_delref(pmap, l1pte, cpu_id);
1745: if (flags & PMAP_CANFAIL)
1746: goto out;
1747: panic("pmap_enter: unable to create L2 PT "
1748: "page");
1749: }
1750: pmap->pm_nlev2++;
1751: #ifdef DEBUG
1752: if (pmapdebug & PDB_PTPAGE)
1753: printf("pmap_enter: new level 2 table at "
1754: "0x%lx\n", pmap_pte_pa(l1pte));
1755: #endif
1756: }
1757:
1758: /*
1759: * Check to see if the level 2 PTE is valid, and
1760: * allocate a new level 3 page table page if it's not.
1761: * A reference will be added to the level 3 table when
1762: * the mapping is validated.
1763: */
1764: l2pte = pmap_l2pte(pmap, va, l1pte);
1765: if (pmap_pte_v(l2pte) == 0) {
1766: pmap_physpage_addref(l2pte);
1767: error = pmap_ptpage_alloc(pmap, l2pte, PGU_L3PT);
1768: if (error) {
1769: pmap_l2pt_delref(pmap, l1pte, l2pte, cpu_id);
1770: if (flags & PMAP_CANFAIL)
1771: goto out;
1772: panic("pmap_enter: unable to create L3 PT "
1773: "page");
1774: }
1775: pmap->pm_nlev3++;
1776: #ifdef DEBUG
1777: if (pmapdebug & PDB_PTPAGE)
1778: printf("pmap_enter: new level 3 table at "
1779: "0x%lx\n", pmap_pte_pa(l2pte));
1780: #endif
1781: }
1782:
1783: /*
1784: * Get the PTE that will map the page.
1785: */
1786: pte = pmap_l3pte(pmap, va, l2pte);
1787: }
1788:
1789: /* Remember all of the old PTE; used for TBI check later. */
1790: opte = *pte;
1791:
1792: /*
1793: * Check to see if the old mapping is valid. If not, validate the
1794: * new one immediately.
1795: */
1796: if (pmap_pte_v(pte) == 0) {
1797: /*
1798: * No need to invalidate the TLB in this case; an invalid
1799: * mapping won't be in the TLB, and a previously valid
1800: * mapping would have been flushed when it was invalidated.
1801: */
1802: tflush = FALSE;
1803:
1804: /*
1805: * No need to synchronize the I-stream, either, for basically
1806: * the same reason.
1807: */
1808: setisync = needisync = FALSE;
1809:
1810: if (pmap != pmap_kernel()) {
1811: /*
1812: * New mappings gain a reference on the level 3
1813: * table.
1814: */
1815: pmap_physpage_addref(pte);
1816: }
1817: goto validate_enterpv;
1818: }
1819:
1820: opa = pmap_pte_pa(pte);
1821: hadasm = (pmap_pte_asm(pte) != 0);
1822:
1823: if (opa == pa) {
1824: /*
1825: * Mapping has not changed; must be a protection or
1826: * wiring change.
1827: */
1828: if (pmap_pte_w_chg(pte, wired ? PG_WIRED : 0)) {
1829: #ifdef DEBUG
1830: if (pmapdebug & PDB_ENTER)
1831: printf("pmap_enter: wiring change -> %d\n",
1832: wired);
1833: #endif
1834: /*
1835: * Adjust the wiring count.
1836: */
1837: if (wired)
1838: PMAP_STAT_INCR(pmap->pm_stats.wired_count, 1);
1839: else
1840: PMAP_STAT_DECR(pmap->pm_stats.wired_count, 1);
1841: }
1842:
1843: /*
1844: * Set the PTE.
1845: */
1846: goto validate;
1847: }
1848:
1849: /*
1850: * The mapping has changed. We need to invalidate the
1851: * old mapping before creating the new one.
1852: */
1853: #ifdef DEBUG
1854: if (pmapdebug & PDB_ENTER)
1855: printf("pmap_enter: removing old mapping 0x%lx\n", va);
1856: #endif
1857: if (pmap != pmap_kernel()) {
1858: /*
1859: * Gain an extra reference on the level 3 table.
1860: * pmap_remove_mapping() will delete a reference,
1861: * and we don't want the table to be erroneously
1862: * freed.
1863: */
1864: pmap_physpage_addref(pte);
1865: }
1866: needisync |= pmap_remove_mapping(pmap, va, pte, TRUE, cpu_id, NULL);
1867:
1868: validate_enterpv:
1869: /*
1870: * Enter the mapping into the pv_table if appropriate.
1871: */
1872: if (managed) {
1873: error = pmap_pv_enter(pmap, pa, va, pte, TRUE);
1874: if (error) {
1875: pmap_l3pt_delref(pmap, va, pte, cpu_id, NULL);
1876: if (flags & PMAP_CANFAIL)
1877: goto out;
1878: panic("pmap_enter: unable to enter mapping in PV "
1879: "table");
1880: }
1881: }
1882:
1883: /*
1884: * Increment counters.
1885: */
1886: PMAP_STAT_INCR(pmap->pm_stats.resident_count, 1);
1887: if (wired)
1888: PMAP_STAT_INCR(pmap->pm_stats.wired_count, 1);
1889:
1890: validate:
1891: /*
1892: * Build the new PTE.
1893: */
1894: npte = ((pa >> PGSHIFT) << PG_SHIFT) | pte_prot(pmap, prot) | PG_V;
1895: if (managed) {
1896: struct pv_head *pvh = pa_to_pvh(pa);
1897: int attrs;
1898:
1899: #ifdef DIAGNOSTIC
1900: if ((flags & VM_PROT_ALL) & ~prot)
1901: panic("pmap_enter: access type exceeds prot");
1902: #endif
1903: simple_lock(&pvh->pvh_slock);
1904: if (flags & VM_PROT_WRITE)
1905: pvh->pvh_attrs |= (PGA_REFERENCED|PGA_MODIFIED);
1906: else if (flags & VM_PROT_ALL)
1907: pvh->pvh_attrs |= PGA_REFERENCED;
1908: attrs = pvh->pvh_attrs;
1909: simple_unlock(&pvh->pvh_slock);
1910:
1911: /*
1912: * Set up referenced/modified emulation for new mapping.
1913: */
1914: if ((attrs & PGA_REFERENCED) == 0)
1915: npte |= PG_FOR | PG_FOW | PG_FOE;
1916: else if ((attrs & PGA_MODIFIED) == 0)
1917: npte |= PG_FOW;
1918:
1919: /* Always force FOE on non-exec mappings. */
1920: if (!pmap_pte_exec(pte))
1921: npte |= PG_FOE;
1922:
1923: /*
1924: * Mapping was entered on PV list.
1925: */
1926: npte |= PG_PVLIST;
1927: }
1928: if (wired)
1929: npte |= PG_WIRED;
1930: #ifdef DEBUG
1931: if (pmapdebug & PDB_ENTER)
1932: printf("pmap_enter: new pte = 0x%lx\n", npte);
1933: #endif
1934:
1935: /*
1936: * If the PALcode portion of the new PTE is the same as the
1937: * old PTE, no TBI is necessary.
1938: */
1939: if (PG_PALCODE(opte) == PG_PALCODE(npte))
1940: tflush = FALSE;
1941:
1942: /*
1943: * Set the new PTE.
1944: */
1945: PMAP_SET_PTE(pte, npte);
1946:
1947: /*
1948: * Invalidate the TLB entry for this VA and any appropriate
1949: * caches.
1950: */
1951: if (tflush) {
1952: PMAP_INVALIDATE_TLB(pmap, va, hadasm, isactive, cpu_id);
1953: PMAP_TLB_SHOOTDOWN(pmap, va, hadasm ? PG_ASM : 0);
1954: }
1955: if (setisync)
1956: PMAP_SET_NEEDISYNC(pmap);
1957: if (needisync)
1958: PMAP_SYNC_ISTREAM(pmap);
1959:
1960: out:
1961: PMAP_UNLOCK(pmap);
1962: PMAP_MAP_TO_HEAD_UNLOCK();
1963:
1964: return error;
1965: }
1966:
1967: /*
1968: * pmap_kenter_pa: [ INTERFACE ]
1969: *
1970: * Enter a va -> pa mapping into the kernel pmap without any
1971: * physical->virtual tracking.
1972: *
1973: * Note: no locking is necessary in this function.
1974: */
1975: void
1976: pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
1977: {
1978: pt_entry_t *pte, npte;
1979: cpuid_t cpu_id = cpu_number();
1980: boolean_t needisync = FALSE;
1981: pmap_t pmap = pmap_kernel();
1982:
1983: #ifdef DEBUG
1984: if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
1985: printf("pmap_kenter_pa(%lx, %lx, %x)\n",
1986: va, pa, prot);
1987: #endif
1988:
1989: #ifdef DIAGNOSTIC
1990: /*
1991: * Sanity check the virtual address.
1992: */
1993: if (va < VM_MIN_KERNEL_ADDRESS)
1994: panic("pmap_kenter_pa: kernel pmap, invalid va 0x%lx", va);
1995: #endif
1996:
1997: pte = PMAP_KERNEL_PTE(va);
1998:
1999: if (pmap_pte_v(pte) == 0)
2000: PMAP_STAT_INCR(pmap->pm_stats.resident_count, 1);
2001: if (pmap_pte_w(pte) == 0)
2002: PMAP_STAT_DECR(pmap->pm_stats.wired_count, 1);
2003:
2004: if ((prot & VM_PROT_EXECUTE) != 0 || pmap_pte_exec(pte))
2005: needisync = TRUE;
2006:
2007: /*
2008: * Build the new PTE.
2009: */
2010: npte = ((pa >> PGSHIFT) << PG_SHIFT) | pte_prot(pmap_kernel(), prot) |
2011: PG_V | PG_WIRED;
2012:
2013: /*
2014: * Set the new PTE.
2015: */
2016: PMAP_SET_PTE(pte, npte);
2017: #if defined(MULTIPROCESSOR)
2018: alpha_mb(); /* XXX alpha_wmb()? */
2019: #endif
2020:
2021: /*
2022: * Invalidate the TLB entry for this VA and any appropriate
2023: * caches.
2024: */
2025: PMAP_INVALIDATE_TLB(pmap, va, TRUE, TRUE, cpu_id);
2026: PMAP_TLB_SHOOTDOWN(pmap, va, PG_ASM);
2027:
2028: if (needisync)
2029: PMAP_SYNC_ISTREAM_KERNEL();
2030: }
2031:
2032: /*
2033: * pmap_kremove: [ INTERFACE ]
2034: *
2035: * Remove a mapping entered with pmap_kenter_pa()
2036: * starting at va, for size bytes (assumed to be page rounded).
2037: */
2038: void
2039: pmap_kremove(vaddr_t va, vsize_t size)
2040: {
2041: pt_entry_t *pte;
2042: boolean_t needisync = FALSE;
2043: cpuid_t cpu_id = cpu_number();
2044: pmap_t pmap = pmap_kernel();
2045:
2046: #ifdef DEBUG
2047: if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
2048: printf("pmap_kremove(%lx, %lx)\n",
2049: va, size);
2050: #endif
2051:
2052: #ifdef DIAGNOSTIC
2053: if (va < VM_MIN_KERNEL_ADDRESS)
2054: panic("pmap_kremove: user address");
2055: #endif
2056:
2057: for (; size != 0; size -= PAGE_SIZE, va += PAGE_SIZE) {
2058: pte = PMAP_KERNEL_PTE(va);
2059: if (pmap_pte_v(pte)) {
2060: #ifdef DIAGNOSTIC
2061: if (pmap_pte_pv(pte))
2062: panic("pmap_kremove: PG_PVLIST mapping for "
2063: "0x%lx", va);
2064: #endif
2065: if (pmap_pte_exec(pte))
2066: needisync = TRUE;
2067:
2068: /* Zap the mapping. */
2069: PMAP_SET_PTE(pte, PG_NV);
2070: #if defined(MULTIPROCESSOR)
2071: alpha_mb(); /* XXX alpha_wmb()? */
2072: #endif
2073: PMAP_INVALIDATE_TLB(pmap, va, TRUE, TRUE, cpu_id);
2074: PMAP_TLB_SHOOTDOWN(pmap, va, PG_ASM);
2075:
2076: /* Update stats. */
2077: PMAP_STAT_DECR(pmap->pm_stats.resident_count, 1);
2078: PMAP_STAT_DECR(pmap->pm_stats.wired_count, 1);
2079: }
2080: }
2081:
2082: if (needisync)
2083: PMAP_SYNC_ISTREAM_KERNEL();
2084: }
2085:
2086: /*
2087: * pmap_unwire: [ INTERFACE ]
2088: *
2089: * Clear the wired attribute for a map/virtual-address pair.
2090: *
2091: * The mapping must already exist in the pmap.
2092: */
2093: void
2094: pmap_unwire(pmap_t pmap, vaddr_t va)
2095: {
2096: pt_entry_t *pte;
2097:
2098: #ifdef DEBUG
2099: if (pmapdebug & PDB_FOLLOW)
2100: printf("pmap_unwire(%p, %lx)\n", pmap, va);
2101: #endif
2102: if (pmap == NULL)
2103: return;
2104:
2105: PMAP_LOCK(pmap);
2106:
2107: pte = pmap_l3pte(pmap, va, NULL);
2108: #ifdef DIAGNOSTIC
2109: if (pte == NULL || pmap_pte_v(pte) == 0)
2110: panic("pmap_unwire");
2111: #endif
2112:
2113: /*
2114: * If wiring actually changed (always?) clear the wire bit and
2115: * update the wire count. Note that wiring is not a hardware
2116: * characteristic so there is no need to invalidate the TLB.
2117: */
2118: if (pmap_pte_w_chg(pte, 0)) {
2119: pmap_pte_set_w(pte, FALSE);
2120: PMAP_STAT_DECR(pmap->pm_stats.wired_count, 1);
2121: }
2122: #ifdef DIAGNOSTIC
2123: else {
2124: printf("pmap_unwire: wiring for pmap %p va 0x%lx "
2125: "didn't change!\n", pmap, va);
2126: }
2127: #endif
2128:
2129: PMAP_UNLOCK(pmap);
2130: }
2131:
2132: /*
2133: * pmap_extract: [ INTERFACE ]
2134: *
2135: * Extract the physical address associated with the given
2136: * pmap/virtual address pair.
2137: */
2138: boolean_t
2139: pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap)
2140: {
2141: pt_entry_t *l1pte, *l2pte, *l3pte;
2142: boolean_t rv = FALSE;
2143: paddr_t pa;
2144:
2145: #ifdef DEBUG
2146: if (pmapdebug & PDB_FOLLOW)
2147: printf("pmap_extract(%p, %lx) -> ", pmap, va);
2148: #endif
2149:
2150: if (pmap == pmap_kernel()) {
2151: if (va < ALPHA_K0SEG_BASE) {
2152: /* nothing */
2153: } else if (va <= ALPHA_K0SEG_END) {
2154: pa = ALPHA_K0SEG_TO_PHYS(va);
2155: rv = TRUE;
2156: } else {
2157: l3pte = PMAP_KERNEL_PTE(va);
2158: if (pmap_pte_v(l3pte)) {
2159: pa = pmap_pte_pa(l3pte) | (va & PGOFSET);
2160: *pap = pa;
2161: rv = TRUE;
2162: }
2163: }
2164: goto out_nolock;
2165: }
2166:
2167: PMAP_LOCK(pmap);
2168:
2169: l1pte = pmap_l1pte(pmap, va);
2170: if (pmap_pte_v(l1pte) == 0)
2171: goto out;
2172:
2173: l2pte = pmap_l2pte(pmap, va, l1pte);
2174: if (pmap_pte_v(l2pte) == 0)
2175: goto out;
2176:
2177: l3pte = pmap_l3pte(pmap, va, l2pte);
2178: if (pmap_pte_v(l3pte) == 0)
2179: goto out;
2180:
2181: pa = pmap_pte_pa(l3pte) | (va & PGOFSET);
2182: *pap = pa;
2183: rv = TRUE;
2184: out:
2185: PMAP_UNLOCK(pmap);
2186: out_nolock:
2187: #ifdef DEBUG
2188: if (pmapdebug & PDB_FOLLOW) {
2189: if (rv)
2190: printf("0x%lx\n", pa);
2191: else
2192: printf("failed\n");
2193: }
2194: #endif
2195: return (rv);
2196: }
2197:
2198: /*
2199: * pmap_copy: [ INTERFACE ]
2200: *
2201: * Copy the mapping range specified by src_addr/len
2202: * from the source map to the range dst_addr/len
2203: * in the destination map.
2204: *
2205: * This routine is only advisory and need not do anything.
2206: */
2207: void
2208: pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len,
2209: vaddr_t src_addr)
2210: {
2211: #ifdef DEBUG
2212: if (pmapdebug & PDB_FOLLOW)
2213: printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n",
2214: dst_pmap, src_pmap, dst_addr, len, src_addr);
2215: #endif
2216: }
2217:
2218: /*
2219: * pmap_collect: [ INTERFACE ]
2220: *
2221: * Garbage collects the physical map system for pages which are no
2222: * longer used. Success need not be guaranteed -- that is, there
2223: * may well be pages which are not referenced, but others may be
2224: * collected.
2225: *
2226: * Called by the pageout daemon when pages are scarce.
2227: */
2228: void
2229: pmap_collect(pmap_t pmap)
2230: {
2231:
2232: #ifdef DEBUG
2233: if (pmapdebug & PDB_FOLLOW)
2234: printf("pmap_collect(%p)\n", pmap);
2235: #endif
2236:
2237: /*
2238: * If called for the kernel pmap, just return. We
2239: * handle this case in the event that we ever want
2240: * to have swappable kernel threads.
2241: */
2242: if (pmap == pmap_kernel())
2243: return;
2244:
2245: /*
2246: * This process is about to be swapped out; free all of
2247: * the PT pages by removing the physical mappings for its
2248: * entire address space. Note: pmap_remove() performs
2249: * all necessary locking.
2250: */
2251: pmap_do_remove(pmap, VM_MIN_ADDRESS, VM_MAX_ADDRESS, FALSE);
2252: }
2253:
2254: /*
2255: * pmap_activate: [ INTERFACE ]
2256: *
2257: * Activate the pmap used by the specified process. This includes
2258: * reloading the MMU context if the current process, and marking
2259: * the pmap in use by the processor.
2260: *
2261: * Note: We may use only spin locks here, since we are called
2262: * by a critical section in cpu_switch()!
2263: */
2264: void
2265: pmap_activate(struct proc *p)
2266: {
2267: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
2268: cpuid_t cpu_id = cpu_number();
2269:
2270: #ifdef DEBUG
2271: if (pmapdebug & PDB_FOLLOW)
2272: printf("pmap_activate(%p)\n", p);
2273: #endif
2274:
2275: /*
2276: * Mark the pmap in use by this processor.
2277: */
2278: atomic_setbits_ulong(&pmap->pm_cpus, (1UL << cpu_id));
2279:
2280: /*
2281: * Move the pmap to the end of the LRU list.
2282: */
2283: simple_lock(&pmap_all_pmaps_slock);
2284: TAILQ_REMOVE(&pmap_all_pmaps, pmap, pm_list);
2285: TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap, pm_list);
2286: simple_unlock(&pmap_all_pmaps_slock);
2287:
2288: PMAP_LOCK(pmap);
2289:
2290: /*
2291: * Allocate an ASN.
2292: */
2293: pmap_asn_alloc(pmap, cpu_id);
2294:
2295: PMAP_ACTIVATE(pmap, p, cpu_id);
2296:
2297: PMAP_UNLOCK(pmap);
2298: }
2299:
2300: /*
2301: * pmap_deactivate: [ INTERFACE ]
2302: *
2303: * Mark that the pmap used by the specified process is no longer
2304: * in use by the processor.
2305: *
2306: * The comment above pmap_activate() wrt. locking applies here,
2307: * as well. Note that we use only a single `atomic' operation,
2308: * so no locking is necessary.
2309: */
2310: void
2311: pmap_deactivate(struct proc *p)
2312: {
2313: struct pmap *pmap = p->p_vmspace->vm_map.pmap;
2314:
2315: #ifdef DEBUG
2316: if (pmapdebug & PDB_FOLLOW)
2317: printf("pmap_deactivate(%p)\n", p);
2318: #endif
2319:
2320: /*
2321: * Mark the pmap no longer in use by this processor.
2322: */
2323: atomic_clearbits_ulong(&pmap->pm_cpus, (1UL << cpu_number()));
2324: }
2325:
2326: /*
2327: * pmap_zero_page: [ INTERFACE ]
2328: *
2329: * Zero the specified (machine independent) page by mapping the page
2330: * into virtual memory and clear its contents, one machine dependent
2331: * page at a time.
2332: *
2333: * Note: no locking is necessary in this function.
2334: */
2335: void
2336: pmap_zero_page(struct vm_page *pg)
2337: {
2338: paddr_t phys = VM_PAGE_TO_PHYS(pg);
2339: u_long *p0, *p1, *pend;
2340:
2341: #ifdef DEBUG
2342: if (pmapdebug & PDB_FOLLOW)
2343: printf("pmap_zero_page(%lx)\n", phys);
2344: #endif
2345:
2346: p0 = (u_long *)ALPHA_PHYS_TO_K0SEG(phys);
2347: pend = (u_long *)((u_long)p0 + PAGE_SIZE);
2348:
2349: /*
2350: * Unroll the loop a bit, doing 16 quadwords per iteration.
2351: * Do only 8 back-to-back stores, and alternate registers.
2352: */
2353: do {
2354: __asm __volatile(
2355: "# BEGIN loop body\n"
2356: " addq %2, (8 * 8), %1 \n"
2357: " stq $31, (0 * 8)(%0) \n"
2358: " stq $31, (1 * 8)(%0) \n"
2359: " stq $31, (2 * 8)(%0) \n"
2360: " stq $31, (3 * 8)(%0) \n"
2361: " stq $31, (4 * 8)(%0) \n"
2362: " stq $31, (5 * 8)(%0) \n"
2363: " stq $31, (6 * 8)(%0) \n"
2364: " stq $31, (7 * 8)(%0) \n"
2365: " \n"
2366: " addq %3, (8 * 8), %0 \n"
2367: " stq $31, (0 * 8)(%1) \n"
2368: " stq $31, (1 * 8)(%1) \n"
2369: " stq $31, (2 * 8)(%1) \n"
2370: " stq $31, (3 * 8)(%1) \n"
2371: " stq $31, (4 * 8)(%1) \n"
2372: " stq $31, (5 * 8)(%1) \n"
2373: " stq $31, (6 * 8)(%1) \n"
2374: " stq $31, (7 * 8)(%1) \n"
2375: " # END loop body"
2376: : "=r" (p0), "=r" (p1)
2377: : "0" (p0), "1" (p1)
2378: : "memory");
2379: } while (p0 < pend);
2380: }
2381:
2382: /*
2383: * pmap_copy_page: [ INTERFACE ]
2384: *
2385: * Copy the specified (machine independent) page by mapping the page
2386: * into virtual memory and using memcpy to copy the page, one machine
2387: * dependent page at a time.
2388: *
2389: * Note: no locking is necessary in this function.
2390: */
2391: void
2392: pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
2393: {
2394: paddr_t src = VM_PAGE_TO_PHYS(srcpg);
2395: paddr_t dst = VM_PAGE_TO_PHYS(dstpg);
2396: caddr_t s, d;
2397:
2398: #ifdef DEBUG
2399: if (pmapdebug & PDB_FOLLOW)
2400: printf("pmap_copy_page(%lx, %lx)\n", src, dst);
2401: #endif
2402: s = (caddr_t)ALPHA_PHYS_TO_K0SEG(src);
2403: d = (caddr_t)ALPHA_PHYS_TO_K0SEG(dst);
2404: memcpy(d, s, PAGE_SIZE);
2405: }
2406:
2407: /*
2408: * pmap_clear_modify: [ INTERFACE ]
2409: *
2410: * Clear the modify bits on the specified physical page.
2411: */
2412: boolean_t
2413: pmap_clear_modify(struct vm_page *pg)
2414: {
2415: struct pv_head *pvh;
2416: paddr_t pa = VM_PAGE_TO_PHYS(pg);
2417: boolean_t rv = FALSE;
2418: cpuid_t cpu_id = cpu_number();
2419:
2420: #ifdef DEBUG
2421: if (pmapdebug & PDB_FOLLOW)
2422: printf("pmap_clear_modify(%p)\n", pg);
2423: #endif
2424:
2425: pvh = pa_to_pvh(pa);
2426:
2427: PMAP_HEAD_TO_MAP_LOCK();
2428: simple_lock(&pvh->pvh_slock);
2429:
2430: if (pvh->pvh_attrs & PGA_MODIFIED) {
2431: rv = TRUE;
2432: pmap_changebit(pa, PG_FOW, ~0, cpu_id);
2433: pvh->pvh_attrs &= ~PGA_MODIFIED;
2434: }
2435:
2436: simple_unlock(&pvh->pvh_slock);
2437: PMAP_HEAD_TO_MAP_UNLOCK();
2438:
2439: return (rv);
2440: }
2441:
2442: /*
2443: * pmap_clear_reference: [ INTERFACE ]
2444: *
2445: * Clear the reference bit on the specified physical page.
2446: */
2447: boolean_t
2448: pmap_clear_reference(struct vm_page *pg)
2449: {
2450: struct pv_head *pvh;
2451: paddr_t pa = VM_PAGE_TO_PHYS(pg);
2452: boolean_t rv = FALSE;
2453: cpuid_t cpu_id = cpu_number();
2454:
2455: #ifdef DEBUG
2456: if (pmapdebug & PDB_FOLLOW)
2457: printf("pmap_clear_reference(%p)\n", pg);
2458: #endif
2459:
2460: pvh = pa_to_pvh(pa);
2461:
2462: PMAP_HEAD_TO_MAP_LOCK();
2463: simple_lock(&pvh->pvh_slock);
2464:
2465: if (pvh->pvh_attrs & PGA_REFERENCED) {
2466: rv = TRUE;
2467: pmap_changebit(pa, PG_FOR | PG_FOW | PG_FOE, ~0, cpu_id);
2468: pvh->pvh_attrs &= ~PGA_REFERENCED;
2469: }
2470:
2471: simple_unlock(&pvh->pvh_slock);
2472: PMAP_HEAD_TO_MAP_UNLOCK();
2473:
2474: return (rv);
2475: }
2476:
2477: /*
2478: * pmap_is_referenced: [ INTERFACE ]
2479: *
2480: * Return whether or not the specified physical page is referenced
2481: * by any physical maps.
2482: */
2483: boolean_t
2484: pmap_is_referenced(struct vm_page *pg)
2485: {
2486: struct pv_head *pvh;
2487: paddr_t pa = VM_PAGE_TO_PHYS(pg);
2488: boolean_t rv;
2489:
2490: pvh = pa_to_pvh(pa);
2491: rv = ((pvh->pvh_attrs & PGA_REFERENCED) != 0);
2492: #ifdef DEBUG
2493: if (pmapdebug & PDB_FOLLOW) {
2494: printf("pmap_is_referenced(%p) -> %c\n", pg, "FT"[rv]);
2495: }
2496: #endif
2497: return (rv);
2498: }
2499:
2500: /*
2501: * pmap_is_modified: [ INTERFACE ]
2502: *
2503: * Return whether or not the specified physical page is modified
2504: * by any physical maps.
2505: */
2506: boolean_t
2507: pmap_is_modified(struct vm_page *pg)
2508: {
2509: struct pv_head *pvh;
2510: paddr_t pa = VM_PAGE_TO_PHYS(pg);
2511: boolean_t rv;
2512:
2513: pvh = pa_to_pvh(pa);
2514: rv = ((pvh->pvh_attrs & PGA_MODIFIED) != 0);
2515: #ifdef DEBUG
2516: if (pmapdebug & PDB_FOLLOW) {
2517: printf("pmap_is_modified(%p) -> %c\n", pg, "FT"[rv]);
2518: }
2519: #endif
2520: return (rv);
2521: }
2522:
2523: /*
2524: * Miscellaneous support routines follow
2525: */
2526:
2527: /*
2528: * alpha_protection_init:
2529: *
2530: * Initialize Alpha protection code array.
2531: *
2532: * Note: no locking is necessary in this function.
2533: */
2534: void
2535: alpha_protection_init(void)
2536: {
2537: int prot, *kp, *up;
2538:
2539: kp = protection_codes[0];
2540: up = protection_codes[1];
2541:
2542: for (prot = 0; prot < 8; prot++) {
2543: kp[prot] = 0; up[prot] = 0;
2544: switch (prot) {
2545: case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE:
2546: kp[prot] |= PG_ASM;
2547: up[prot] |= 0;
2548: break;
2549:
2550: case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE:
2551: case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE:
2552: kp[prot] |= PG_EXEC; /* software */
2553: up[prot] |= PG_EXEC; /* software */
2554: /* FALLTHROUGH */
2555:
2556: case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE:
2557: kp[prot] |= PG_ASM | PG_KRE;
2558: up[prot] |= PG_URE | PG_KRE;
2559: break;
2560:
2561: case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE:
2562: kp[prot] |= PG_ASM | PG_KWE;
2563: up[prot] |= PG_UWE | PG_KWE;
2564: break;
2565:
2566: case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE:
2567: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE:
2568: kp[prot] |= PG_EXEC; /* software */
2569: up[prot] |= PG_EXEC; /* software */
2570: /* FALLTHROUGH */
2571:
2572: case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE:
2573: kp[prot] |= PG_ASM | PG_KWE | PG_KRE;
2574: up[prot] |= PG_UWE | PG_URE | PG_KWE | PG_KRE;
2575: break;
2576: }
2577: }
2578: }
2579:
2580: /*
2581: * pmap_remove_mapping:
2582: *
2583: * Invalidate a single page denoted by pmap/va.
2584: *
2585: * If (pte != NULL), it is the already computed PTE for the page.
2586: *
2587: * Note: locking in this function is complicated by the fact
2588: * that we can be called when the PV list is already locked.
2589: * (pmap_page_protect()). In this case, the caller must be
2590: * careful to get the next PV entry while we remove this entry
2591: * from beneath it. We assume that the pmap itself is already
2592: * locked; dolock applies only to the PV list.
2593: *
2594: * Returns TRUE or FALSE, indicating if an I-stream sync needs
2595: * to be initiated (for this CPU or for other CPUs).
2596: */
2597: boolean_t
2598: pmap_remove_mapping(pmap_t pmap, vaddr_t va, pt_entry_t *pte,
2599: boolean_t dolock, cpuid_t cpu_id, struct prm_thief *prmt)
2600: {
2601: paddr_t pa;
2602: boolean_t onpv;
2603: boolean_t hadasm;
2604: boolean_t isactive;
2605: boolean_t needisync = FALSE;
2606: struct pv_entry **pvp;
2607: pt_entry_t **ptp;
2608:
2609: #ifdef DEBUG
2610: if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
2611: printf("pmap_remove_mapping(%p, %lx, %p, %d, %ld, %p)\n",
2612: pmap, va, pte, dolock, cpu_id, pvp);
2613: #endif
2614:
2615: if (prmt != NULL) {
2616: if (prmt->prmt_flags & PRMT_PV)
2617: pvp = &prmt->prmt_pv;
2618: else
2619: pvp = NULL;
2620: if (prmt->prmt_flags & PRMT_PTP)
2621: ptp = &prmt->prmt_ptp;
2622: else
2623: ptp = NULL;
2624: } else {
2625: pvp = NULL;
2626: ptp = NULL;
2627: }
2628:
2629: /*
2630: * PTE not provided, compute it from pmap and va.
2631: */
2632: if (pte == PT_ENTRY_NULL) {
2633: pte = pmap_l3pte(pmap, va, NULL);
2634: if (pmap_pte_v(pte) == 0)
2635: return (FALSE);
2636: }
2637:
2638: pa = pmap_pte_pa(pte);
2639: onpv = (pmap_pte_pv(pte) != 0);
2640: hadasm = (pmap_pte_asm(pte) != 0);
2641: isactive = PMAP_ISACTIVE(pmap, cpu_id);
2642:
2643: /*
2644: * Determine what we need to do about the I-stream. If
2645: * PG_EXEC was set, we mark a user pmap as needing an
2646: * I-sync on the way out to userspace. We always need
2647: * an immediate I-sync for the kernel pmap.
2648: */
2649: if (pmap_pte_exec(pte)) {
2650: if (pmap == pmap_kernel())
2651: needisync = TRUE;
2652: else {
2653: PMAP_SET_NEEDISYNC(pmap);
2654: needisync = (pmap->pm_cpus != 0);
2655: }
2656: }
2657:
2658: /*
2659: * Update statistics
2660: */
2661: if (pmap_pte_w(pte))
2662: PMAP_STAT_DECR(pmap->pm_stats.wired_count, 1);
2663: PMAP_STAT_DECR(pmap->pm_stats.resident_count, 1);
2664:
2665: /*
2666: * Invalidate the PTE after saving the reference modify info.
2667: */
2668: #ifdef DEBUG
2669: if (pmapdebug & PDB_REMOVE)
2670: printf("remove: invalidating pte at %p\n", pte);
2671: #endif
2672: PMAP_SET_PTE(pte, PG_NV);
2673:
2674: PMAP_INVALIDATE_TLB(pmap, va, hadasm, isactive, cpu_id);
2675: PMAP_TLB_SHOOTDOWN(pmap, va, hadasm ? PG_ASM : 0);
2676:
2677: /*
2678: * If we're removing a user mapping, check to see if we
2679: * can free page table pages.
2680: */
2681: if (pmap != pmap_kernel()) {
2682: /*
2683: * Delete the reference on the level 3 table. It will
2684: * delete references on the level 2 and 1 tables as
2685: * appropriate.
2686: */
2687: pmap_l3pt_delref(pmap, va, pte, cpu_id, ptp);
2688: }
2689:
2690: /*
2691: * If the mapping wasn't entered on the PV list, we're all done.
2692: */
2693: if (onpv == FALSE) {
2694: #ifdef DIAGNOSTIC
2695: if (pvp != NULL)
2696: panic("pmap_removing_mapping: onpv / pvp inconsistent");
2697: #endif
2698: return (needisync);
2699: }
2700:
2701: /*
2702: * Remove it from the PV table.
2703: */
2704: pmap_pv_remove(pmap, pa, va, dolock, pvp);
2705:
2706: return (needisync);
2707: }
2708:
2709: /*
2710: * pmap_changebit:
2711: *
2712: * Set or clear the specified PTE bits for all mappings on the
2713: * specified page.
2714: *
2715: * Note: we assume that the pv_head is already locked, and that
2716: * the caller has acquired a PV->pmap mutex so that we can lock
2717: * the pmaps as we encounter them.
2718: *
2719: * XXX This routine could stand to have some I-stream
2720: * XXX optimization done.
2721: */
2722: void
2723: pmap_changebit(paddr_t pa, u_long set, u_long mask, cpuid_t cpu_id)
2724: {
2725: struct pv_head *pvh;
2726: pv_entry_t pv;
2727: pt_entry_t *pte, npte;
2728: vaddr_t va;
2729: boolean_t hadasm, isactive;
2730: boolean_t needisync, needkisync = FALSE;
2731:
2732: #ifdef DEBUG
2733: if (pmapdebug & PDB_BITS)
2734: printf("pmap_changebit(0x%lx, 0x%lx, 0x%lx)\n",
2735: pa, set, mask);
2736: #endif
2737: if (!PAGE_IS_MANAGED(pa))
2738: return;
2739:
2740: pvh = pa_to_pvh(pa);
2741: /*
2742: * Loop over all current mappings setting/clearing as appropriate.
2743: */
2744: for (pv = LIST_FIRST(&pvh->pvh_list); pv != NULL;
2745: pv = LIST_NEXT(pv, pv_list)) {
2746: va = pv->pv_va;
2747:
2748: /*
2749: * XXX don't write protect pager mappings
2750: */
2751: if (pv->pv_pmap == pmap_kernel() &&
2752: /* XXX */ mask == ~(PG_KWE | PG_UWE)) {
2753: if (va >= uvm.pager_sva && va < uvm.pager_eva)
2754: continue;
2755: }
2756:
2757: PMAP_LOCK(pv->pv_pmap);
2758:
2759: pte = pv->pv_pte;
2760: npte = (*pte | set) & mask;
2761: if (*pte != npte) {
2762: hadasm = (pmap_pte_asm(pte) != 0);
2763: isactive = PMAP_ISACTIVE(pv->pv_pmap, cpu_id);
2764: /*
2765: * Determine what we need to do about the I-stream.
2766: * If PG_EXEC was set, we mark a user pmap as needing
2767: * an I-sync on the way out to userspace. We always
2768: * need an immediate I-sync for the kernel pmap.
2769: */
2770: needisync = FALSE;
2771: if (pmap_pte_exec(pte)) {
2772: if (pv->pv_pmap == pmap_kernel())
2773: needkisync = TRUE;
2774: else {
2775: PMAP_SET_NEEDISYNC(pv->pv_pmap);
2776: if (pv->pv_pmap->pm_cpus != 0)
2777: needisync = TRUE;
2778: }
2779: } else {
2780: /* Never clear FOE on non-exec mappings. */
2781: npte |= PG_FOE;
2782: }
2783: PMAP_SET_PTE(pte, npte);
2784: if (needisync)
2785: PMAP_SYNC_ISTREAM_USER(pv->pv_pmap);
2786: PMAP_INVALIDATE_TLB(pv->pv_pmap, va, hadasm, isactive,
2787: cpu_id);
2788: PMAP_TLB_SHOOTDOWN(pv->pv_pmap, va,
2789: hadasm ? PG_ASM : 0);
2790: }
2791: PMAP_UNLOCK(pv->pv_pmap);
2792: }
2793:
2794: if (needkisync)
2795: PMAP_SYNC_ISTREAM_KERNEL();
2796: }
2797:
2798: /*
2799: * pmap_emulate_reference:
2800: *
2801: * Emulate reference and/or modified bit hits.
2802: *
2803: * return non-zero if this was a FOE fault and the pte is not
2804: * executable.
2805: */
2806: int
2807: pmap_emulate_reference(struct proc *p, vaddr_t v, int user, int type)
2808: {
2809: pt_entry_t faultoff, *pte;
2810: paddr_t pa;
2811: struct pv_head *pvh;
2812: boolean_t didlock = FALSE;
2813: cpuid_t cpu_id = cpu_number();
2814:
2815: #ifdef DEBUG
2816: if (pmapdebug & PDB_FOLLOW)
2817: printf("pmap_emulate_reference: %p, 0x%lx, %d, %d\n",
2818: p, v, user, type);
2819: #endif
2820:
2821: /*
2822: * Convert process and virtual address to physical address.
2823: */
2824: if (v >= VM_MIN_KERNEL_ADDRESS) {
2825: if (user)
2826: panic("pmap_emulate_reference: user ref to kernel");
2827: /*
2828: * No need to lock here; kernel PT pages never go away.
2829: */
2830: pte = PMAP_KERNEL_PTE(v);
2831: } else {
2832: #ifdef DIAGNOSTIC
2833: if (p == NULL)
2834: panic("pmap_emulate_reference: bad proc");
2835: if (p->p_vmspace == NULL)
2836: panic("pmap_emulate_reference: bad p_vmspace");
2837: #endif
2838: PMAP_LOCK(p->p_vmspace->vm_map.pmap);
2839: didlock = TRUE;
2840: pte = pmap_l3pte(p->p_vmspace->vm_map.pmap, v, NULL);
2841: /*
2842: * We'll unlock below where we're done with the PTE.
2843: */
2844: }
2845: if (!pmap_pte_exec(pte) && type == ALPHA_MMCSR_FOE) {
2846: if (didlock)
2847: PMAP_UNLOCK(p->p_vmspace->vm_map.pmap);
2848: return (1);
2849: }
2850: #ifdef DEBUG
2851: if (pmapdebug & PDB_FOLLOW) {
2852: printf("\tpte = %p, ", pte);
2853: printf("*pte = 0x%lx\n", *pte);
2854: }
2855: #endif
2856: #ifdef DEBUG /* These checks are more expensive */
2857: if (!pmap_pte_v(pte))
2858: panic("pmap_emulate_reference: invalid pte");
2859: #if 0
2860: /*
2861: * Can't do these, because cpu_fork call pmap_emulate_reference(),
2862: * and the bits aren't guaranteed, for them...
2863: */
2864: if (type == ALPHA_MMCSR_FOW) {
2865: if (!(*pte & (user ? PG_UWE : PG_UWE | PG_KWE)))
2866: panic("pmap_emulate_reference: write but unwritable");
2867: if (!(*pte & PG_FOW))
2868: panic("pmap_emulate_reference: write but not FOW");
2869: } else {
2870: if (!(*pte & (user ? PG_URE : PG_URE | PG_KRE)))
2871: panic("pmap_emulate_reference: !write but unreadable");
2872: if (!(*pte & (PG_FOR | PG_FOE)))
2873: panic("pmap_emulate_reference: !write but not FOR|FOE");
2874: }
2875: #endif
2876: /* Other diagnostics? */
2877: #endif
2878: pa = pmap_pte_pa(pte);
2879:
2880: /*
2881: * We're now done with the PTE. If it was a user pmap, unlock
2882: * it now.
2883: */
2884: if (didlock)
2885: PMAP_UNLOCK(p->p_vmspace->vm_map.pmap);
2886:
2887: #ifdef DEBUG
2888: if (pmapdebug & PDB_FOLLOW)
2889: printf("\tpa = 0x%lx\n", pa);
2890: #endif
2891: #ifdef DIAGNOSTIC
2892: if (!PAGE_IS_MANAGED(pa))
2893: panic("pmap_emulate_reference(%p, 0x%lx, %d, %d): pa 0x%lx not managed", p, v, user, type, pa);
2894: #endif
2895:
2896: /*
2897: * Twiddle the appropriate bits to reflect the reference
2898: * and/or modification..
2899: *
2900: * The rules:
2901: * (1) always mark page as used, and
2902: * (2) if it was a write fault, mark page as modified.
2903: */
2904: pvh = pa_to_pvh(pa);
2905:
2906: PMAP_HEAD_TO_MAP_LOCK();
2907: simple_lock(&pvh->pvh_slock);
2908:
2909: if (type == ALPHA_MMCSR_FOW) {
2910: pvh->pvh_attrs |= (PGA_REFERENCED|PGA_MODIFIED);
2911: faultoff = PG_FOR | PG_FOW | PG_FOE;
2912: } else {
2913: pvh->pvh_attrs |= PGA_REFERENCED;
2914: faultoff = PG_FOR | PG_FOE;
2915: }
2916: /*
2917: * If the page is not PG_EXEC, pmap_changebit will automagically
2918: * set PG_FOE (gross, but necessary if I don't want to change the
2919: * whole API).
2920: */
2921: pmap_changebit(pa, 0, ~faultoff, cpu_id);
2922:
2923: simple_unlock(&pvh->pvh_slock);
2924: PMAP_HEAD_TO_MAP_UNLOCK();
2925:
2926: return (0);
2927: }
2928:
2929: #ifdef DEBUG
2930: /*
2931: * pmap_pv_dump:
2932: *
2933: * Dump the physical->virtual data for the specified page.
2934: */
2935: void
2936: pmap_pv_dump(paddr_t pa)
2937: {
2938: struct pv_head *pvh;
2939: pv_entry_t pv;
2940: static const char *usage[] = {
2941: "normal", "pvent", "l1pt", "l2pt", "l3pt",
2942: };
2943:
2944: pvh = pa_to_pvh(pa);
2945:
2946: simple_lock(&pvh->pvh_slock);
2947:
2948: printf("pa 0x%lx (attrs = 0x%x, usage = " /* ) */, pa, pvh->pvh_attrs);
2949: if (pvh->pvh_usage < PGU_NORMAL || pvh->pvh_usage > PGU_L3PT)
2950: /* ( */ printf("??? %d):\n", pvh->pvh_usage);
2951: else
2952: /* ( */ printf("%s):\n", usage[pvh->pvh_usage]);
2953:
2954: for (pv = LIST_FIRST(&pvh->pvh_list); pv != NULL;
2955: pv = LIST_NEXT(pv, pv_list))
2956: printf(" pmap %p, va 0x%lx\n",
2957: pv->pv_pmap, pv->pv_va);
2958: printf("\n");
2959:
2960: simple_unlock(&pvh->pvh_slock);
2961: }
2962: #endif
2963:
2964: /*
2965: * vtophys:
2966: *
2967: * Return the physical address corresponding to the K0SEG or
2968: * K1SEG address provided.
2969: *
2970: * Note: no locking is necessary in this function.
2971: */
2972: paddr_t
2973: vtophys(vaddr_t vaddr)
2974: {
2975: pt_entry_t *pte;
2976: paddr_t paddr = 0;
2977:
2978: if (vaddr < ALPHA_K0SEG_BASE)
2979: printf("vtophys: invalid vaddr 0x%lx", vaddr);
2980: else if (vaddr <= ALPHA_K0SEG_END)
2981: paddr = ALPHA_K0SEG_TO_PHYS(vaddr);
2982: else {
2983: pte = PMAP_KERNEL_PTE(vaddr);
2984: if (pmap_pte_v(pte))
2985: paddr = pmap_pte_pa(pte) | (vaddr & PGOFSET);
2986: }
2987:
2988: #if 0
2989: printf("vtophys(0x%lx) -> 0x%lx\n", vaddr, paddr);
2990: #endif
2991:
2992: return (paddr);
2993: }
2994:
2995: /******************** pv_entry management ********************/
2996:
2997: /*
2998: * pmap_pv_enter:
2999: *
3000: * Add a physical->virtual entry to the pv_table.
3001: */
3002: int
3003: pmap_pv_enter(pmap_t pmap, paddr_t pa, vaddr_t va, pt_entry_t *pte,
3004: boolean_t dolock)
3005: {
3006: struct pv_head *pvh;
3007: pv_entry_t newpv;
3008:
3009: /*
3010: * Allocate and fill in the new pv_entry.
3011: */
3012: newpv = pmap_pv_alloc();
3013: if (newpv == NULL)
3014: return (ENOMEM);
3015: newpv->pv_va = va;
3016: newpv->pv_pmap = pmap;
3017: newpv->pv_pte = pte;
3018:
3019: pvh = pa_to_pvh(pa);
3020:
3021: if (dolock)
3022: simple_lock(&pvh->pvh_slock);
3023:
3024: #ifdef DEBUG
3025: {
3026: pv_entry_t pv;
3027: /*
3028: * Make sure the entry doesn't already exist.
3029: */
3030: for (pv = LIST_FIRST(&pvh->pvh_list); pv != NULL;
3031: pv = LIST_NEXT(pv, pv_list))
3032: if (pmap == pv->pv_pmap && va == pv->pv_va) {
3033: printf("pmap = %p, va = 0x%lx\n", pmap, va);
3034: panic("pmap_pv_enter: already in pv table");
3035: }
3036: }
3037: #endif
3038:
3039: /*
3040: * ...and put it in the list.
3041: */
3042: LIST_INSERT_HEAD(&pvh->pvh_list, newpv, pv_list);
3043:
3044: if (dolock)
3045: simple_unlock(&pvh->pvh_slock);
3046:
3047: return (0);
3048: }
3049:
3050: /*
3051: * pmap_pv_remove:
3052: *
3053: * Remove a physical->virtual entry from the pv_table.
3054: */
3055: void
3056: pmap_pv_remove(pmap_t pmap, paddr_t pa, vaddr_t va, boolean_t dolock,
3057: struct pv_entry **pvp)
3058: {
3059: struct pv_head *pvh;
3060: pv_entry_t pv;
3061:
3062: pvh = pa_to_pvh(pa);
3063:
3064: if (dolock)
3065: simple_lock(&pvh->pvh_slock);
3066:
3067: /*
3068: * Find the entry to remove.
3069: */
3070: for (pv = LIST_FIRST(&pvh->pvh_list); pv != NULL;
3071: pv = LIST_NEXT(pv, pv_list))
3072: if (pmap == pv->pv_pmap && va == pv->pv_va)
3073: break;
3074:
3075: #ifdef DEBUG
3076: if (pv == NULL)
3077: panic("pmap_pv_remove: not in pv table");
3078: #endif
3079:
3080: LIST_REMOVE(pv, pv_list);
3081:
3082: if (dolock)
3083: simple_unlock(&pvh->pvh_slock);
3084:
3085: /*
3086: * If pvp is not NULL, this is pmap_pv_alloc() stealing an
3087: * entry from another mapping, and we return the now unused
3088: * entry in it. Otherwise, free the pv_entry.
3089: */
3090: if (pvp != NULL)
3091: *pvp = pv;
3092: else
3093: pmap_pv_free(pv);
3094: }
3095:
3096: /*
3097: * pmap_pv_alloc:
3098: *
3099: * Allocate a pv_entry.
3100: */
3101: struct pv_entry *
3102: pmap_pv_alloc(void)
3103: {
3104: struct pv_head *pvh;
3105: struct pv_entry *pv;
3106: int bank, npg, pg;
3107: pt_entry_t *pte;
3108: pmap_t pvpmap;
3109: cpuid_t cpu_id;
3110: struct prm_thief prmt;
3111:
3112: pv = pool_get(&pmap_pv_pool, PR_NOWAIT);
3113: if (pv != NULL)
3114: return (pv);
3115:
3116: prmt.prmt_flags = PRMT_PV;
3117:
3118: /*
3119: * We were unable to allocate one from the pool. Try to
3120: * steal one from another mapping. At this point we know that:
3121: *
3122: * (1) We have not locked the pv table, and we already have
3123: * the map-to-head lock, so it is safe for us to do so here.
3124: *
3125: * (2) The pmap that wants this entry *is* locked. We must
3126: * use simple_lock_try() to prevent deadlock from occurring.
3127: *
3128: * XXX Note that in case #2, there is an exception; it *is* safe to
3129: * steal a mapping from the pmap that wants this entry! We may want
3130: * to consider passing the pmap to this function so that we can take
3131: * advantage of this.
3132: */
3133:
3134: /* XXX This search could probably be improved. */
3135: for (bank = 0; bank < vm_nphysseg; bank++) {
3136: npg = vm_physmem[bank].end - vm_physmem[bank].start;
3137: for (pg = 0; pg < npg; pg++) {
3138: pvh = &vm_physmem[bank].pmseg.pvhead[pg];
3139: simple_lock(&pvh->pvh_slock);
3140: for (pv = LIST_FIRST(&pvh->pvh_list);
3141: pv != NULL; pv = LIST_NEXT(pv, pv_list)) {
3142: pvpmap = pv->pv_pmap;
3143:
3144: /* Don't steal from kernel pmap. */
3145: if (pvpmap == pmap_kernel())
3146: continue;
3147:
3148: if (simple_lock_try(&pvpmap->pm_slock) == 0)
3149: continue;
3150:
3151: pte = pv->pv_pte;
3152:
3153: /* Don't steal wired mappings. */
3154: if (pmap_pte_w(pte)) {
3155: simple_unlock(&pvpmap->pm_slock);
3156: continue;
3157: }
3158:
3159: cpu_id = cpu_number();
3160:
3161: /*
3162: * Okay! We have a mapping we can steal;
3163: * remove it and grab the pv_entry.
3164: */
3165: if (pmap_remove_mapping(pvpmap, pv->pv_va,
3166: pte, FALSE, cpu_id, &prmt))
3167: PMAP_SYNC_ISTREAM(pvpmap);
3168:
3169: /* Unlock everything and return. */
3170: simple_unlock(&pvpmap->pm_slock);
3171: simple_unlock(&pvh->pvh_slock);
3172: return (prmt.prmt_pv);
3173: }
3174: simple_unlock(&pvh->pvh_slock);
3175: }
3176: }
3177:
3178: return (NULL);
3179: }
3180:
3181: /*
3182: * pmap_pv_free:
3183: *
3184: * Free a pv_entry.
3185: */
3186: void
3187: pmap_pv_free(struct pv_entry *pv)
3188: {
3189:
3190: pool_put(&pmap_pv_pool, pv);
3191: }
3192:
3193: /*
3194: * pmap_pv_page_alloc:
3195: *
3196: * Allocate a page for the pv_entry pool.
3197: */
3198: void *
3199: pmap_pv_page_alloc(struct pool *pp, int flags)
3200: {
3201: paddr_t pg;
3202:
3203: if (pmap_physpage_alloc(PGU_PVENT, &pg))
3204: return ((void *)ALPHA_PHYS_TO_K0SEG(pg));
3205: return (NULL);
3206: }
3207:
3208: /*
3209: * pmap_pv_page_free:
3210: *
3211: * Free a pv_entry pool page.
3212: */
3213: void
3214: pmap_pv_page_free(struct pool *pp, void *v)
3215: {
3216:
3217: pmap_physpage_free(ALPHA_K0SEG_TO_PHYS((vaddr_t)v));
3218: }
3219:
3220: /******************** misc. functions ********************/
3221:
3222: /*
3223: * pmap_physpage_alloc:
3224: *
3225: * Allocate a single page from the VM system and return the
3226: * physical address for that page.
3227: */
3228: boolean_t
3229: pmap_physpage_alloc(int usage, paddr_t *pap)
3230: {
3231: struct vm_page *pg;
3232: struct pv_head *pvh;
3233: paddr_t pa;
3234:
3235: /*
3236: * Don't ask for a zeroed page in the L1PT case -- we will
3237: * properly initialize it in the constructor.
3238: */
3239:
3240: pg = uvm_pagealloc(NULL, 0, NULL, usage == PGU_L1PT ?
3241: UVM_PGA_USERESERVE : UVM_PGA_USERESERVE|UVM_PGA_ZERO);
3242: if (pg != NULL) {
3243: pa = VM_PAGE_TO_PHYS(pg);
3244:
3245: pvh = pa_to_pvh(pa);
3246: simple_lock(&pvh->pvh_slock);
3247: #ifdef DIAGNOSTIC
3248: if (pvh->pvh_usage != PGU_NORMAL) {
3249: printf("pmap_physpage_alloc: page 0x%lx is "
3250: "in use (%s)\n", pa,
3251: pmap_pgu_strings[pvh->pvh_usage]);
3252: panic("pmap_physpage_alloc");
3253: }
3254: if (pvh->pvh_refcnt != 0) {
3255: printf("pmap_physpage_alloc: page 0x%lx has "
3256: "%d references\n", pa, pvh->pvh_refcnt);
3257: panic("pmap_physpage_alloc");
3258: }
3259: #endif
3260: pvh->pvh_usage = usage;
3261: simple_unlock(&pvh->pvh_slock);
3262: *pap = pa;
3263: return (TRUE);
3264: }
3265: return (FALSE);
3266: }
3267:
3268: /*
3269: * pmap_physpage_free:
3270: *
3271: * Free the single page table page at the specified physical address.
3272: */
3273: void
3274: pmap_physpage_free(paddr_t pa)
3275: {
3276: struct pv_head *pvh;
3277: struct vm_page *pg;
3278:
3279: if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
3280: panic("pmap_physpage_free: bogus physical page address");
3281:
3282: pvh = pa_to_pvh(pa);
3283:
3284: simple_lock(&pvh->pvh_slock);
3285: #ifdef DIAGNOSTIC
3286: if (pvh->pvh_usage == PGU_NORMAL)
3287: panic("pmap_physpage_free: not in use?!");
3288: if (pvh->pvh_refcnt != 0)
3289: panic("pmap_physpage_free: page still has references");
3290: #endif
3291: pvh->pvh_usage = PGU_NORMAL;
3292: simple_unlock(&pvh->pvh_slock);
3293:
3294: uvm_pagefree(pg);
3295: }
3296:
3297: /*
3298: * pmap_physpage_addref:
3299: *
3300: * Add a reference to the specified special use page.
3301: */
3302: int
3303: pmap_physpage_addref(void *kva)
3304: {
3305: struct pv_head *pvh;
3306: paddr_t pa;
3307: int rval;
3308:
3309: pa = ALPHA_K0SEG_TO_PHYS(trunc_page((vaddr_t)kva));
3310: pvh = pa_to_pvh(pa);
3311:
3312: simple_lock(&pvh->pvh_slock);
3313: #ifdef DIAGNOSTIC
3314: if (pvh->pvh_usage == PGU_NORMAL)
3315: panic("pmap_physpage_addref: not a special use page");
3316: #endif
3317:
3318: rval = ++pvh->pvh_refcnt;
3319: simple_unlock(&pvh->pvh_slock);
3320:
3321: return (rval);
3322: }
3323:
3324: /*
3325: * pmap_physpage_delref:
3326: *
3327: * Delete a reference to the specified special use page.
3328: */
3329: int
3330: pmap_physpage_delref(void *kva)
3331: {
3332: struct pv_head *pvh;
3333: paddr_t pa;
3334: int rval;
3335:
3336: pa = ALPHA_K0SEG_TO_PHYS(trunc_page((vaddr_t)kva));
3337: pvh = pa_to_pvh(pa);
3338:
3339: simple_lock(&pvh->pvh_slock);
3340: #ifdef DIAGNOSTIC
3341: if (pvh->pvh_usage == PGU_NORMAL)
3342: panic("pmap_physpage_delref: not a special use page");
3343: #endif
3344:
3345: rval = --pvh->pvh_refcnt;
3346:
3347: #ifdef DIAGNOSTIC
3348: /*
3349: * Make sure we never have a negative reference count.
3350: */
3351: if (pvh->pvh_refcnt < 0)
3352: panic("pmap_physpage_delref: negative reference count");
3353: #endif
3354: simple_unlock(&pvh->pvh_slock);
3355:
3356: return (rval);
3357: }
3358:
3359: /******************** page table page management ********************/
3360:
3361: /*
3362: * pmap_growkernel: [ INTERFACE ]
3363: *
3364: * Grow the kernel address space. This is a hint from the
3365: * upper layer to pre-allocate more kernel PT pages.
3366: */
3367: vaddr_t
3368: pmap_growkernel(vaddr_t maxkvaddr)
3369: {
3370: struct pmap *kpm = pmap_kernel(), *pm;
3371: paddr_t ptaddr;
3372: pt_entry_t *l1pte, *l2pte, pte;
3373: vaddr_t va;
3374: int s, l1idx;
3375:
3376: if (maxkvaddr <= virtual_end)
3377: goto out; /* we are OK */
3378:
3379: s = splhigh(); /* to be safe */
3380: simple_lock(&pmap_growkernel_slock);
3381:
3382: va = virtual_end;
3383:
3384: while (va < maxkvaddr) {
3385: /*
3386: * If there is no valid L1 PTE (i.e. no L2 PT page),
3387: * allocate a new L2 PT page and insert it into the
3388: * L1 map.
3389: */
3390: l1pte = pmap_l1pte(kpm, va);
3391: if (pmap_pte_v(l1pte) == 0) {
3392: /*
3393: * XXX PGU_NORMAL? It's not a "traditional" PT page.
3394: */
3395: if (uvm.page_init_done == FALSE) {
3396: /*
3397: * We're growing the kernel pmap early (from
3398: * uvm_pageboot_alloc()). This case must
3399: * be handled a little differently.
3400: */
3401: ptaddr = ALPHA_K0SEG_TO_PHYS(
3402: pmap_steal_memory(PAGE_SIZE, NULL, NULL));
3403: } else if (pmap_physpage_alloc(PGU_NORMAL,
3404: &ptaddr) == FALSE)
3405: goto die;
3406: pte = (atop(ptaddr) << PG_SHIFT) |
3407: PG_V | PG_ASM | PG_KRE | PG_KWE | PG_WIRED;
3408: *l1pte = pte;
3409:
3410: l1idx = l1pte_index(va);
3411:
3412: /* Update all the user pmaps. */
3413: simple_lock(&pmap_all_pmaps_slock);
3414: for (pm = TAILQ_FIRST(&pmap_all_pmaps);
3415: pm != NULL; pm = TAILQ_NEXT(pm, pm_list)) {
3416: /* Skip the kernel pmap. */
3417: if (pm == pmap_kernel())
3418: continue;
3419:
3420: PMAP_LOCK(pm);
3421: if (pm->pm_lev1map == kernel_lev1map) {
3422: PMAP_UNLOCK(pm);
3423: continue;
3424: }
3425: pm->pm_lev1map[l1idx] = pte;
3426: PMAP_UNLOCK(pm);
3427: }
3428: simple_unlock(&pmap_all_pmaps_slock);
3429: }
3430:
3431: /*
3432: * Have an L2 PT page now, add the L3 PT page.
3433: */
3434: l2pte = pmap_l2pte(kpm, va, l1pte);
3435: KASSERT(pmap_pte_v(l2pte) == 0);
3436: if (uvm.page_init_done == FALSE) {
3437: /*
3438: * See above.
3439: */
3440: ptaddr = ALPHA_K0SEG_TO_PHYS(
3441: pmap_steal_memory(PAGE_SIZE, NULL, NULL));
3442: } else if (pmap_physpage_alloc(PGU_NORMAL, &ptaddr) == FALSE)
3443: goto die;
3444: *l2pte = (atop(ptaddr) << PG_SHIFT) |
3445: PG_V | PG_ASM | PG_KRE | PG_KWE | PG_WIRED;
3446: va += ALPHA_L2SEG_SIZE;
3447: }
3448:
3449: /* Invalidate the L1 PT cache. */
3450: pool_cache_invalidate(&pmap_l1pt_cache);
3451:
3452: virtual_end = va;
3453:
3454: simple_unlock(&pmap_growkernel_slock);
3455: splx(s);
3456:
3457: out:
3458: return (virtual_end);
3459:
3460: die:
3461: panic("pmap_growkernel: out of memory");
3462: }
3463:
3464: /*
3465: * pmap_lev1map_create:
3466: *
3467: * Create a new level 1 page table for the specified pmap.
3468: *
3469: * Note: the pmap must already be locked.
3470: */
3471: int
3472: pmap_lev1map_create(pmap_t pmap, cpuid_t cpu_id)
3473: {
3474: pt_entry_t *l1pt;
3475:
3476: #ifdef DIAGNOSTIC
3477: if (pmap == pmap_kernel())
3478: panic("pmap_lev1map_create: got kernel pmap");
3479:
3480: if (pmap->pm_asn[cpu_id] != PMAP_ASN_RESERVED)
3481: panic("pmap_lev1map_create: pmap uses non-reserved ASN");
3482: #endif
3483:
3484: simple_lock(&pmap_growkernel_slock);
3485:
3486: l1pt = pool_cache_get(&pmap_l1pt_cache, PR_NOWAIT);
3487: if (l1pt == NULL) {
3488: simple_unlock(&pmap_growkernel_slock);
3489: return (ENOMEM);
3490: }
3491:
3492: pmap->pm_lev1map = l1pt;
3493:
3494: simple_unlock(&pmap_growkernel_slock);
3495:
3496: /*
3497: * The page table base has changed; if the pmap was active,
3498: * reactivate it.
3499: */
3500: if (PMAP_ISACTIVE(pmap, cpu_id)) {
3501: pmap_asn_alloc(pmap, cpu_id);
3502: PMAP_ACTIVATE(pmap, curproc, cpu_id);
3503: }
3504: return (0);
3505: }
3506:
3507: /*
3508: * pmap_lev1map_destroy:
3509: *
3510: * Destroy the level 1 page table for the specified pmap.
3511: *
3512: * Note: the pmap must already be locked.
3513: */
3514: void
3515: pmap_lev1map_destroy(pmap_t pmap, cpuid_t cpu_id)
3516: {
3517: pt_entry_t *l1pt = pmap->pm_lev1map;
3518:
3519: #ifdef DIAGNOSTIC
3520: if (pmap == pmap_kernel())
3521: panic("pmap_lev1map_destroy: got kernel pmap");
3522: #endif
3523:
3524: /*
3525: * Go back to referencing the global kernel_lev1map.
3526: */
3527: pmap->pm_lev1map = kernel_lev1map;
3528:
3529: /*
3530: * The page table base has changed; if the pmap was active,
3531: * reactivate it. Note that allocation of a new ASN is
3532: * not necessary here:
3533: *
3534: * (1) We've gotten here because we've deleted all
3535: * user mappings in the pmap, invalidating the
3536: * TLB entries for them as we go.
3537: *
3538: * (2) kernel_lev1map contains only kernel mappings, which
3539: * were identical in the user pmap, and all of
3540: * those mappings have PG_ASM, so the ASN doesn't
3541: * matter.
3542: *
3543: * We do, however, ensure that the pmap is using the
3544: * reserved ASN, to ensure that no two pmaps never have
3545: * clashing TLB entries.
3546: */
3547: PMAP_INVALIDATE_ASN(pmap, cpu_id);
3548: if (PMAP_ISACTIVE(pmap, cpu_id))
3549: PMAP_ACTIVATE(pmap, curproc, cpu_id);
3550:
3551: /*
3552: * Free the old level 1 page table page.
3553: */
3554: pool_cache_put(&pmap_l1pt_cache, l1pt);
3555: }
3556:
3557: /*
3558: * pmap_l1pt_ctor:
3559: *
3560: * Pool cache constructor for L1 PT pages.
3561: */
3562: int
3563: pmap_l1pt_ctor(void *arg, void *object, int flags)
3564: {
3565: pt_entry_t *l1pt = object, pte;
3566: int i;
3567:
3568: /*
3569: * Initialize the new level 1 table by zeroing the
3570: * user portion and copying the kernel mappings into
3571: * the kernel portion.
3572: */
3573: for (i = 0; i < l1pte_index(VM_MIN_KERNEL_ADDRESS); i++)
3574: l1pt[i] = 0;
3575:
3576: for (i = l1pte_index(VM_MIN_KERNEL_ADDRESS);
3577: i <= l1pte_index(VM_MAX_KERNEL_ADDRESS); i++)
3578: l1pt[i] = kernel_lev1map[i];
3579:
3580: /*
3581: * Now, map the new virtual page table. NOTE: NO ASM!
3582: */
3583: pte = ((ALPHA_K0SEG_TO_PHYS((vaddr_t) l1pt) >> PGSHIFT) << PG_SHIFT) |
3584: PG_V | PG_KRE | PG_KWE;
3585: l1pt[l1pte_index(VPTBASE)] = pte;
3586:
3587: return (0);
3588: }
3589:
3590: /*
3591: * pmap_l1pt_alloc:
3592: *
3593: * Page allocator for L1 PT pages.
3594: */
3595: void *
3596: pmap_l1pt_alloc(struct pool *pp, int flags)
3597: {
3598: paddr_t ptpa;
3599:
3600: /*
3601: * Attempt to allocate a free page.
3602: */
3603: if (pmap_physpage_alloc(PGU_L1PT, &ptpa) == FALSE) {
3604: #if 0
3605: /*
3606: * Yow! No free pages! Try to steal a PT page from
3607: * another pmap!
3608: */
3609: if (pmap_ptpage_steal(pmap, PGU_L1PT, &ptpa) == FALSE)
3610: #endif
3611: return (NULL);
3612: }
3613:
3614: return ((void *) ALPHA_PHYS_TO_K0SEG(ptpa));
3615: }
3616:
3617: /*
3618: * pmap_l1pt_free:
3619: *
3620: * Page freer for L1 PT pages.
3621: */
3622: void
3623: pmap_l1pt_free(struct pool *pp, void *v)
3624: {
3625:
3626: pmap_physpage_free(ALPHA_K0SEG_TO_PHYS((vaddr_t) v));
3627: }
3628:
3629: /*
3630: * pmap_ptpage_alloc:
3631: *
3632: * Allocate a level 2 or level 3 page table page, and
3633: * initialize the PTE that references it.
3634: *
3635: * Note: the pmap must already be locked.
3636: */
3637: int
3638: pmap_ptpage_alloc(pmap_t pmap, pt_entry_t *pte, int usage)
3639: {
3640: paddr_t ptpa;
3641:
3642: /*
3643: * Allocate the page table page.
3644: */
3645: if (pmap_physpage_alloc(usage, &ptpa) == FALSE) {
3646: /*
3647: * Yow! No free pages! Try to steal a PT page from
3648: * another pmap!
3649: */
3650: if (pmap_ptpage_steal(pmap, usage, &ptpa) == FALSE)
3651: return (ENOMEM);
3652: }
3653:
3654: /*
3655: * Initialize the referencing PTE.
3656: */
3657: PMAP_SET_PTE(pte, ((ptpa >> PGSHIFT) << PG_SHIFT) |
3658: PG_V | PG_KRE | PG_KWE | PG_WIRED |
3659: (pmap == pmap_kernel() ? PG_ASM : 0));
3660:
3661: return (0);
3662: }
3663:
3664: /*
3665: * pmap_ptpage_free:
3666: *
3667: * Free the level 2 or level 3 page table page referenced
3668: * be the provided PTE.
3669: *
3670: * Note: the pmap must already be locked.
3671: */
3672: void
3673: pmap_ptpage_free(pmap_t pmap, pt_entry_t *pte, pt_entry_t **ptp)
3674: {
3675: paddr_t ptpa;
3676:
3677: /*
3678: * Extract the physical address of the page from the PTE
3679: * and clear the entry.
3680: */
3681: ptpa = pmap_pte_pa(pte);
3682: PMAP_SET_PTE(pte, PG_NV);
3683:
3684: /*
3685: * Check to see if we're stealing the PT page. If we are,
3686: * zero it, and return the KSEG address of the page.
3687: */
3688: if (ptp != NULL) {
3689: pmap_zero_page(PHYS_TO_VM_PAGE(ptpa));
3690: *ptp = (pt_entry_t *)ALPHA_PHYS_TO_K0SEG(ptpa);
3691: } else {
3692: #ifdef DEBUG
3693: pmap_zero_page(PHYS_TO_VM_PAGE(ptpa));
3694: #endif
3695: pmap_physpage_free(ptpa);
3696: }
3697: }
3698:
3699: /*
3700: * pmap_ptpage_steal:
3701: *
3702: * Steal a PT page from a pmap.
3703: */
3704: boolean_t
3705: pmap_ptpage_steal(pmap_t pmap, int usage, paddr_t *pap)
3706: {
3707: struct pv_head *pvh;
3708: pmap_t spmap;
3709: int l1idx, l2idx, l3idx;
3710: pt_entry_t *lev2map, *lev3map;
3711: vaddr_t va;
3712: paddr_t pa;
3713: struct prm_thief prmt;
3714: cpuid_t cpu_id = cpu_number();
3715: boolean_t needisync = FALSE;
3716:
3717: prmt.prmt_flags = PRMT_PTP;
3718: prmt.prmt_ptp = NULL;
3719:
3720: /*
3721: * We look for pmaps which do not reference kernel_lev1map (which
3722: * would indicate that they are either the kernel pmap, or a user
3723: * pmap with no valid mappings). Since the list of all pmaps is
3724: * maintained in an LRU fashion, we should get a pmap that is
3725: * `more inactive' than our current pmap (although this may not
3726: * always be the case).
3727: *
3728: * We start looking for valid L1 PTEs at the lowest address,
3729: * go to that L2, look for the first valid L2 PTE, and steal
3730: * that L3 PT page.
3731: */
3732: simple_lock(&pmap_all_pmaps_slock);
3733: for (spmap = TAILQ_FIRST(&pmap_all_pmaps);
3734: spmap != NULL; spmap = TAILQ_NEXT(spmap, pm_list)) {
3735: /*
3736: * Skip the kernel pmap and ourselves.
3737: */
3738: if (spmap == pmap_kernel() || spmap == pmap)
3739: continue;
3740:
3741: PMAP_LOCK(spmap);
3742: if (spmap->pm_lev1map == kernel_lev1map) {
3743: PMAP_UNLOCK(spmap);
3744: continue;
3745: }
3746:
3747: /*
3748: * Have a candidate pmap. Loop through the PT pages looking
3749: * for one we can steal.
3750: */
3751: for (l1idx = 0;
3752: l1idx < l1pte_index(VM_MAXUSER_ADDRESS); l1idx++) {
3753: if (pmap_pte_v(&spmap->pm_lev1map[l1idx]) == 0)
3754: continue;
3755:
3756: lev2map = (pt_entry_t *)ALPHA_PHYS_TO_K0SEG(
3757: pmap_pte_pa(&spmap->pm_lev1map[l1idx]));
3758: for (l2idx = 0; l2idx < NPTEPG; l2idx++) {
3759: if (pmap_pte_v(&lev2map[l2idx]) == 0)
3760: continue;
3761: lev3map = (pt_entry_t *)ALPHA_PHYS_TO_K0SEG(
3762: pmap_pte_pa(&lev2map[l2idx]));
3763: for (l3idx = 0; l3idx < NPTEPG; l3idx++) {
3764: /*
3765: * If the entry is valid and wired,
3766: * we cannot steal this page.
3767: */
3768: if (pmap_pte_v(&lev3map[l3idx]) &&
3769: pmap_pte_w(&lev3map[l3idx]))
3770: break;
3771: }
3772:
3773: /*
3774: * If we scanned all of the current L3 table
3775: * without finding a wired entry, we can
3776: * steal this page!
3777: */
3778: if (l3idx == NPTEPG)
3779: goto found_one;
3780: }
3781: }
3782:
3783: /*
3784: * Didn't find something we could steal in this
3785: * pmap, try the next one.
3786: */
3787: PMAP_UNLOCK(spmap);
3788: continue;
3789:
3790: found_one:
3791: /* ...don't need this anymore. */
3792: simple_unlock(&pmap_all_pmaps_slock);
3793:
3794: /*
3795: * Okay! We have a PT page we can steal. l1idx and
3796: * l2idx indicate which L1 PTP and L2 PTP we should
3797: * use to compute the virtual addresses the L3 PTP
3798: * maps. Loop through all the L3 PTEs in this range
3799: * and nuke the mappings for them. When we're through,
3800: * we'll have a PT page pointed to by prmt.prmt_ptp!
3801: */
3802: for (l3idx = 0,
3803: va = (l1idx * ALPHA_L1SEG_SIZE) +
3804: (l2idx * ALPHA_L2SEG_SIZE);
3805: l3idx < NPTEPG && prmt.prmt_ptp == NULL;
3806: l3idx++, va += PAGE_SIZE) {
3807: if (pmap_pte_v(&lev3map[l3idx])) {
3808: needisync |= pmap_remove_mapping(spmap, va,
3809: &lev3map[l3idx], TRUE, cpu_id, &prmt);
3810: }
3811: }
3812:
3813: if (needisync)
3814: PMAP_SYNC_ISTREAM(pmap);
3815:
3816: PMAP_UNLOCK(spmap);
3817:
3818: #ifdef DIAGNOSTIC
3819: if (prmt.prmt_ptp == NULL)
3820: panic("pmap_ptptage_steal: failed");
3821: if (prmt.prmt_ptp != lev3map)
3822: panic("pmap_ptpage_steal: inconsistent");
3823: #endif
3824: pa = ALPHA_K0SEG_TO_PHYS((vaddr_t)prmt.prmt_ptp);
3825:
3826: /*
3827: * Don't bother locking here; the assignment is atomic.
3828: */
3829: pvh = pa_to_pvh(pa);
3830: pvh->pvh_usage = usage;
3831:
3832: *pap = pa;
3833: return (TRUE);
3834: }
3835: simple_unlock(&pmap_all_pmaps_slock);
3836: return (FALSE);
3837: }
3838:
3839: /*
3840: * pmap_l3pt_delref:
3841: *
3842: * Delete a reference on a level 3 PT page. If the reference drops
3843: * to zero, free it.
3844: *
3845: * Note: the pmap must already be locked.
3846: */
3847: void
3848: pmap_l3pt_delref(pmap_t pmap, vaddr_t va, pt_entry_t *l3pte, cpuid_t cpu_id,
3849: pt_entry_t **ptp)
3850: {
3851: pt_entry_t *l1pte, *l2pte;
3852:
3853: l1pte = pmap_l1pte(pmap, va);
3854: l2pte = pmap_l2pte(pmap, va, l1pte);
3855:
3856: #ifdef DIAGNOSTIC
3857: if (pmap == pmap_kernel())
3858: panic("pmap_l3pt_delref: kernel pmap");
3859: #endif
3860:
3861: if (pmap_physpage_delref(l3pte) == 0) {
3862: /*
3863: * No more mappings; we can free the level 3 table.
3864: */
3865: #ifdef DEBUG
3866: if (pmapdebug & PDB_PTPAGE)
3867: printf("pmap_l3pt_delref: freeing level 3 table at "
3868: "0x%lx\n", pmap_pte_pa(l2pte));
3869: #endif
3870: pmap_ptpage_free(pmap, l2pte, ptp);
3871: pmap->pm_nlev3--;
3872:
3873: /*
3874: * We've freed a level 3 table, so we must
3875: * invalidate the TLB entry for that PT page
3876: * in the Virtual Page Table VA range, because
3877: * otherwise the PALcode will service a TLB
3878: * miss using the stale VPT TLB entry it entered
3879: * behind our back to shortcut to the VA's PTE.
3880: */
3881: PMAP_INVALIDATE_TLB(pmap,
3882: (vaddr_t)(&VPT[VPT_INDEX(va)]), FALSE,
3883: PMAP_ISACTIVE(pmap, cpu_id), cpu_id);
3884: PMAP_TLB_SHOOTDOWN(pmap,
3885: (vaddr_t)(&VPT[VPT_INDEX(va)]), 0);
3886:
3887: /*
3888: * We've freed a level 3 table, so delete the reference
3889: * on the level 2 table.
3890: */
3891: pmap_l2pt_delref(pmap, l1pte, l2pte, cpu_id);
3892: }
3893: }
3894:
3895: /*
3896: * pmap_l2pt_delref:
3897: *
3898: * Delete a reference on a level 2 PT page. If the reference drops
3899: * to zero, free it.
3900: *
3901: * Note: the pmap must already be locked.
3902: */
3903: void
3904: pmap_l2pt_delref(pmap_t pmap, pt_entry_t *l1pte, pt_entry_t *l2pte,
3905: cpuid_t cpu_id)
3906: {
3907:
3908: #ifdef DIAGNOSTIC
3909: if (pmap == pmap_kernel())
3910: panic("pmap_l2pt_delref: kernel pmap");
3911: #endif
3912:
3913: if (pmap_physpage_delref(l2pte) == 0) {
3914: /*
3915: * No more mappings in this segment; we can free the
3916: * level 2 table.
3917: */
3918: #ifdef DEBUG
3919: if (pmapdebug & PDB_PTPAGE)
3920: printf("pmap_l2pt_delref: freeing level 2 table at "
3921: "0x%lx\n", pmap_pte_pa(l1pte));
3922: #endif
3923: pmap_ptpage_free(pmap, l1pte, NULL);
3924: pmap->pm_nlev2--;
3925:
3926: /*
3927: * We've freed a level 2 table, so delete the reference
3928: * on the level 1 table.
3929: */
3930: pmap_l1pt_delref(pmap, l1pte, cpu_id);
3931: }
3932: }
3933:
3934: /*
3935: * pmap_l1pt_delref:
3936: *
3937: * Delete a reference on a level 1 PT page. If the reference drops
3938: * to zero, free it.
3939: *
3940: * Note: the pmap must already be locked.
3941: */
3942: void
3943: pmap_l1pt_delref(pmap_t pmap, pt_entry_t *l1pte, cpuid_t cpu_id)
3944: {
3945:
3946: #ifdef DIAGNOSTIC
3947: if (pmap == pmap_kernel())
3948: panic("pmap_l1pt_delref: kernel pmap");
3949: #endif
3950:
3951: if (pmap_physpage_delref(l1pte) == 0) {
3952: /*
3953: * No more level 2 tables left, go back to the global
3954: * kernel_lev1map.
3955: */
3956: pmap_lev1map_destroy(pmap, cpu_id);
3957: }
3958: }
3959:
3960: /******************** Address Space Number management ********************/
3961:
3962: /*
3963: * pmap_asn_alloc:
3964: *
3965: * Allocate and assign an ASN to the specified pmap.
3966: *
3967: * Note: the pmap must already be locked.
3968: */
3969: void
3970: pmap_asn_alloc(pmap_t pmap, cpuid_t cpu_id)
3971: {
3972:
3973: #ifdef DEBUG
3974: if (pmapdebug & (PDB_FOLLOW|PDB_ASN))
3975: printf("pmap_asn_alloc(%p)\n", pmap);
3976: #endif
3977:
3978: /*
3979: * If the pmap is still using the global kernel_lev1map, there
3980: * is no need to assign an ASN at this time, because only
3981: * kernel mappings exist in that map, and all kernel mappings
3982: * have PG_ASM set. If the pmap eventually gets its own
3983: * lev1map, an ASN will be allocated at that time.
3984: */
3985: if (pmap->pm_lev1map == kernel_lev1map) {
3986: #ifdef DEBUG
3987: if (pmapdebug & PDB_ASN)
3988: printf("pmap_asn_alloc: still references "
3989: "kernel_lev1map\n");
3990: #endif
3991: #ifdef DIAGNOSTIC
3992: if (pmap->pm_asn[cpu_id] != PMAP_ASN_RESERVED)
3993: panic("pmap_asn_alloc: kernel_lev1map without "
3994: "PMAP_ASN_RESERVED");
3995: #endif
3996: return;
3997: }
3998:
3999: /*
4000: * On processors which do not implement ASNs, the swpctx PALcode
4001: * operation will automatically invalidate the TLB and I-cache,
4002: * so we don't need to do that here.
4003: */
4004: if (pmap_max_asn == 0) {
4005: /*
4006: * Refresh the pmap's generation number, to
4007: * simplify logic elsewhere.
4008: */
4009: pmap->pm_asngen[cpu_id] = pmap_asn_generation[cpu_id];
4010: #ifdef DEBUG
4011: if (pmapdebug & PDB_ASN)
4012: printf("pmap_asn_alloc: no ASNs, using asngen %lu\n",
4013: pmap->pm_asngen[cpu_id]);
4014: #endif
4015: return;
4016: }
4017:
4018: /*
4019: * Hopefully, we can continue using the one we have...
4020: */
4021: if (pmap->pm_asn[cpu_id] != PMAP_ASN_RESERVED &&
4022: pmap->pm_asngen[cpu_id] == pmap_asn_generation[cpu_id]) {
4023: /*
4024: * ASN is still in the current generation; keep on using it.
4025: */
4026: #ifdef DEBUG
4027: if (pmapdebug & PDB_ASN)
4028: printf("pmap_asn_alloc: same generation, keeping %u\n",
4029: pmap->pm_asn[cpu_id]);
4030: #endif
4031: return;
4032: }
4033:
4034: /*
4035: * Need to assign a new ASN. Grab the next one, incrementing
4036: * the generation number if we have to.
4037: */
4038: if (pmap_next_asn[cpu_id] > pmap_max_asn) {
4039: /*
4040: * Invalidate all non-PG_ASM TLB entries and the
4041: * I-cache, and bump the generation number.
4042: */
4043: ALPHA_TBIAP();
4044: alpha_pal_imb();
4045:
4046: pmap_next_asn[cpu_id] = 1;
4047:
4048: pmap_asn_generation[cpu_id]++;
4049: #ifdef DIAGNOSTIC
4050: if (pmap_asn_generation[cpu_id] == 0) {
4051: /*
4052: * The generation number has wrapped. We could
4053: * handle this scenario by traversing all of
4054: * the pmaps, and invalidating the generation
4055: * number on those which are not currently
4056: * in use by this processor.
4057: *
4058: * However... considering that we're using
4059: * an unsigned 64-bit integer for generation
4060: * numbers, on non-ASN CPUs, we won't wrap
4061: * for approx. 585 million years, or 75 billion
4062: * years on a 128-ASN CPU (assuming 1000 switch
4063: * operations per second).
4064: *
4065: * So, we don't bother.
4066: */
4067: panic("pmap_asn_alloc: too much uptime");
4068: }
4069: #endif
4070: #ifdef DEBUG
4071: if (pmapdebug & PDB_ASN)
4072: printf("pmap_asn_alloc: generation bumped to %lu\n",
4073: pmap_asn_generation[cpu_id]);
4074: #endif
4075: }
4076:
4077: /*
4078: * Assign the new ASN and validate the generation number.
4079: */
4080: pmap->pm_asn[cpu_id] = pmap_next_asn[cpu_id]++;
4081: pmap->pm_asngen[cpu_id] = pmap_asn_generation[cpu_id];
4082:
4083: #ifdef DEBUG
4084: if (pmapdebug & PDB_ASN)
4085: printf("pmap_asn_alloc: assigning %u to pmap %p\n",
4086: pmap->pm_asn[cpu_id], pmap);
4087: #endif
4088:
4089: /*
4090: * Have a new ASN, so there's no need to sync the I-stream
4091: * on the way back out to userspace.
4092: */
4093: atomic_clearbits_ulong(&pmap->pm_needisync, (1UL << cpu_id));
4094: }
4095:
4096: #if defined(MULTIPROCESSOR)
4097: /******************** TLB shootdown code ********************/
4098:
4099: /*
4100: * pmap_tlb_shootdown:
4101: *
4102: * Cause the TLB entry for pmap/va to be shot down.
4103: */
4104: void
4105: pmap_tlb_shootdown(pmap_t pmap, vaddr_t va, pt_entry_t pte)
4106: {
4107: u_long ipinum;
4108: cpuid_t i, cpu_id = cpu_number();
4109: struct pmap_tlb_shootdown_q *pq;
4110: struct pmap_tlb_shootdown_job *pj;
4111: int s;
4112:
4113: for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) {
4114: if (i == cpu_id || (cpus_running & (1UL << i)) == 0)
4115: continue;
4116:
4117: pq = &pmap_tlb_shootdown_q[i];
4118:
4119: PSJQ_LOCK(pq, s);
4120:
4121: pj = pmap_tlb_shootdown_job_get(pq);
4122: pq->pq_pte |= pte;
4123: if (pj == NULL) {
4124: /*
4125: * Couldn't allocate a job entry. Just do a
4126: * TBIA[P].
4127: */
4128: if (pq->pq_pte & PG_ASM)
4129: ipinum = ALPHA_IPI_SHOOTDOWN;
4130: else
4131: ipinum = ALPHA_IPI_IMB;
4132: alpha_send_ipi(i, ipinum);
4133: } else {
4134: pj->pj_pmap = pmap;
4135: pj->pj_va = va;
4136: pj->pj_pte = pte;
4137: TAILQ_INSERT_TAIL(&pq->pq_head, pj, pj_list);
4138: ipinum = ALPHA_IPI_SHOOTDOWN;
4139: }
4140:
4141: alpha_send_ipi(i, ipinum);
4142:
4143: PSJQ_UNLOCK(pq, s);
4144: }
4145: }
4146:
4147: /*
4148: * pmap_do_tlb_shootdown:
4149: *
4150: * Process pending TLB shootdown operations for this processor.
4151: */
4152: void
4153: pmap_do_tlb_shootdown(struct cpu_info *ci, struct trapframe *framep)
4154: {
4155: cpuid_t cpu_id = ci->ci_cpuid;
4156: u_long cpu_mask = (1UL << cpu_id);
4157: struct pmap_tlb_shootdown_q *pq = &pmap_tlb_shootdown_q[cpu_id];
4158: struct pmap_tlb_shootdown_job *pj;
4159: int s;
4160:
4161: PSJQ_LOCK(pq, s);
4162:
4163: while ((pj = TAILQ_FIRST(&pq->pq_head)) != NULL) {
4164: TAILQ_REMOVE(&pq->pq_head, pj, pj_list);
4165: PMAP_INVALIDATE_TLB(pj->pj_pmap, pj->pj_va,
4166: pj->pj_pte & PG_ASM, pj->pj_pmap->pm_cpus & cpu_mask,
4167: cpu_id);
4168: pmap_tlb_shootdown_job_put(pq, pj);
4169: }
4170: pq->pq_pte = 0;
4171:
4172: PSJQ_UNLOCK(pq, s);
4173: }
4174:
4175: /*
4176: * pmap_tlb_shootdown_q_drain:
4177: *
4178: * Drain a processor's TLB shootdown queue. We do not perform
4179: * the shootdown operations. This is merely a convenience
4180: * function.
4181: */
4182: void
4183: pmap_tlb_shootdown_q_drain(cpuid_t cpu_id, boolean_t all)
4184: {
4185: struct pmap_tlb_shootdown_q *pq = &pmap_tlb_shootdown_q[cpu_id];
4186: struct pmap_tlb_shootdown_job *pj, *npj;
4187: pt_entry_t npte = 0;
4188: int s;
4189:
4190: PSJQ_LOCK(pq, s);
4191:
4192: for (pj = TAILQ_FIRST(&pq->pq_head); pj != NULL; pj = npj) {
4193: npj = TAILQ_NEXT(pj, pj_list);
4194: if (all || (pj->pj_pte & PG_ASM) == 0) {
4195: TAILQ_REMOVE(&pq->pq_head, pj, pj_list);
4196: pmap_tlb_shootdown_job_put(pq, pj);
4197: } else
4198: npte |= pj->pj_pte;
4199: }
4200: pq->pq_pte = npte;
4201:
4202: PSJQ_UNLOCK(pq, s);
4203: }
4204:
4205: /*
4206: * pmap_tlb_shootdown_job_get:
4207: *
4208: * Get a TLB shootdown job queue entry. This places a limit on
4209: * the number of outstanding jobs a processor may have.
4210: *
4211: * Note: We expect the queue to be locked.
4212: */
4213: struct pmap_tlb_shootdown_job *
4214: pmap_tlb_shootdown_job_get(struct pmap_tlb_shootdown_q *pq)
4215: {
4216: struct pmap_tlb_shootdown_job *pj;
4217:
4218: if (pq->pq_count >= PMAP_TLB_SHOOTDOWN_MAXJOBS)
4219: return (NULL);
4220: pj = pool_get(&pmap_tlb_shootdown_job_pool, PR_NOWAIT);
4221: if (pj != NULL)
4222: pq->pq_count++;
4223: return (pj);
4224: }
4225:
4226: /*
4227: * pmap_tlb_shootdown_job_put:
4228: *
4229: * Put a TLB shootdown job queue entry onto the free list.
4230: *
4231: * Note: We expect the queue to be locked.
4232: */
4233: void
4234: pmap_tlb_shootdown_job_put(struct pmap_tlb_shootdown_q *pq,
4235: struct pmap_tlb_shootdown_job *pj)
4236: {
4237:
4238: #ifdef DIAGNOSTIC
4239: if (pq->pq_count == 0)
4240: panic("pmap_tlb_shootdown_job_put: queue length inconsistency");
4241: #endif
4242: pool_put(&pmap_tlb_shootdown_job_pool, pj);
4243: pq->pq_count--;
4244: }
4245: #endif /* MULTIPROCESSOR */
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to pmap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / alpha / alpha