![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_map.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm|
Return to uvm_map.c CVS log | Up to [local] / sys / uvm |
1.1 nbrk 1: /* $OpenBSD: uvm_map.c,v 1.97 2007/07/18 17:00:20 art Exp $ */
2: /* $NetBSD: uvm_map.c,v 1.86 2000/11/27 08:40:03 chs Exp $ */
3:
4: /*
5: * Copyright (c) 1997 Charles D. Cranor and Washington University.
6: * Copyright (c) 1991, 1993, The Regents of the University of California.
7: *
8: * All rights reserved.
9: *
10: * This code is derived from software contributed to Berkeley by
11: * The Mach Operating System project at Carnegie-Mellon University.
12: *
13: * Redistribution and use in source and binary forms, with or without
14: * modification, are permitted provided that the following conditions
15: * are met:
16: * 1. Redistributions of source code must retain the above copyright
17: * notice, this list of conditions and the following disclaimer.
18: * 2. Redistributions in binary form must reproduce the above copyright
19: * notice, this list of conditions and the following disclaimer in the
20: * documentation and/or other materials provided with the distribution.
21: * 3. All advertising materials mentioning features or use of this software
22: * must display the following acknowledgement:
23: * This product includes software developed by Charles D. Cranor,
24: * Washington University, the University of California, Berkeley and
25: * its contributors.
26: * 4. Neither the name of the University nor the names of its contributors
27: * may be used to endorse or promote products derived from this software
28: * without specific prior written permission.
29: *
30: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
31: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
32: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
34: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
35: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
36: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
37: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
38: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
39: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40: * SUCH DAMAGE.
41: *
42: * @(#)vm_map.c 8.3 (Berkeley) 1/12/94
43: * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
44: *
45: *
46: * Copyright (c) 1987, 1990 Carnegie-Mellon University.
47: * All rights reserved.
48: *
49: * Permission to use, copy, modify and distribute this software and
50: * its documentation is hereby granted, provided that both the copyright
51: * notice and this permission notice appear in all copies of the
52: * software, derivative works or modified versions, and any portions
53: * thereof, and that both notices appear in supporting documentation.
54: *
55: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57: * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58: *
59: * Carnegie Mellon requests users of this software to return to
60: *
61: * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
62: * School of Computer Science
63: * Carnegie Mellon University
64: * Pittsburgh PA 15213-3890
65: *
66: * any improvements or extensions that they make and grant Carnegie the
67: * rights to redistribute these changes.
68: */
69:
70: /*
71: * uvm_map.c: uvm map operations
72: */
73:
74: #include <sys/param.h>
75: #include <sys/systm.h>
76: #include <sys/mman.h>
77: #include <sys/proc.h>
78: #include <sys/malloc.h>
79: #include <sys/pool.h>
80: #include <sys/kernel.h>
81:
82: #include <dev/rndvar.h>
83:
84: #ifdef SYSVSHM
85: #include <sys/shm.h>
86: #endif
87:
88: #define UVM_MAP
89: #include <uvm/uvm.h>
90: #undef RB_AUGMENT
91: #define RB_AUGMENT(x) uvm_rb_augment(x)
92:
93: #ifdef DDB
94: #include <uvm/uvm_ddb.h>
95: #endif
96:
97: static struct timeval uvm_kmapent_last_warn_time;
98: static struct timeval uvm_kmapent_warn_rate = { 10, 0 };
99:
100: struct uvm_cnt uvm_map_call, map_backmerge, map_forwmerge;
101: struct uvm_cnt uvm_mlk_call, uvm_mlk_hint;
102: const char vmmapbsy[] = "vmmapbsy";
103:
104: /*
105: * Da history books
106: */
107: UVMHIST_DECL(maphist);
108: UVMHIST_DECL(pdhist);
109:
110: /*
111: * pool for vmspace structures.
112: */
113:
114: struct pool uvm_vmspace_pool;
115:
116: /*
117: * pool for dynamically-allocated map entries.
118: */
119:
120: struct pool uvm_map_entry_pool;
121: struct pool uvm_map_entry_kmem_pool;
122:
123: #ifdef PMAP_GROWKERNEL
124: /*
125: * This global represents the end of the kernel virtual address
126: * space. If we want to exceed this, we must grow the kernel
127: * virtual address space dynamically.
128: *
129: * Note, this variable is locked by kernel_map's lock.
130: */
131: vaddr_t uvm_maxkaddr;
132: #endif
133:
134: /*
135: * macros
136: */
137:
138: /*
139: * uvm_map_entry_link: insert entry into a map
140: *
141: * => map must be locked
142: */
143: #define uvm_map_entry_link(map, after_where, entry) do { \
144: (map)->nentries++; \
145: (entry)->prev = (after_where); \
146: (entry)->next = (after_where)->next; \
147: (entry)->prev->next = (entry); \
148: (entry)->next->prev = (entry); \
149: uvm_rb_insert(map, entry); \
150: } while (0)
151:
152: /*
153: * uvm_map_entry_unlink: remove entry from a map
154: *
155: * => map must be locked
156: */
157: #define uvm_map_entry_unlink(map, entry) do { \
158: (map)->nentries--; \
159: (entry)->next->prev = (entry)->prev; \
160: (entry)->prev->next = (entry)->next; \
161: uvm_rb_remove(map, entry); \
162: } while (0)
163:
164: /*
165: * SAVE_HINT: saves the specified entry as the hint for future lookups.
166: *
167: * => map need not be locked (protected by hint_lock).
168: */
169: #define SAVE_HINT(map,check,value) do { \
170: simple_lock(&(map)->hint_lock); \
171: if ((map)->hint == (check)) \
172: (map)->hint = (value); \
173: simple_unlock(&(map)->hint_lock); \
174: } while (0)
175:
176: /*
177: * VM_MAP_RANGE_CHECK: check and correct range
178: *
179: * => map must at least be read locked
180: */
181:
182: #define VM_MAP_RANGE_CHECK(map, start, end) do { \
183: if (start < vm_map_min(map)) \
184: start = vm_map_min(map); \
185: if (end > vm_map_max(map)) \
186: end = vm_map_max(map); \
187: if (start > end) \
188: start = end; \
189: } while (0)
190:
191: /*
192: * local prototypes
193: */
194:
195: void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *);
196: void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
197: void uvm_map_reference_amap(struct vm_map_entry *, int);
198: void uvm_map_unreference_amap(struct vm_map_entry *, int);
199: int uvm_map_spacefits(struct vm_map *, vaddr_t *, vsize_t,
200: struct vm_map_entry *, voff_t, vsize_t);
201:
202: struct vm_map_entry *uvm_mapent_alloc(struct vm_map *);
203: void uvm_mapent_free(struct vm_map_entry *);
204:
205:
206: /*
207: * Tree manipulation.
208: */
209: void uvm_rb_insert(struct vm_map *, struct vm_map_entry *);
210: void uvm_rb_remove(struct vm_map *, struct vm_map_entry *);
211: vsize_t uvm_rb_space(struct vm_map *, struct vm_map_entry *);
212:
213: #ifdef DEBUG
214: int _uvm_tree_sanity(struct vm_map *map, const char *name);
215: #endif
216: vsize_t uvm_rb_subtree_space(struct vm_map_entry *);
217: void uvm_rb_fixup(struct vm_map *, struct vm_map_entry *);
218:
219: static __inline int
220: uvm_compare(struct vm_map_entry *a, struct vm_map_entry *b)
221: {
222: if (a->start < b->start)
223: return (-1);
224: else if (a->start > b->start)
225: return (1);
226:
227: return (0);
228: }
229:
230:
231: static __inline void
232: uvm_rb_augment(struct vm_map_entry *entry)
233: {
234: entry->space = uvm_rb_subtree_space(entry);
235: }
236:
237: RB_PROTOTYPE(uvm_tree, vm_map_entry, rb_entry, uvm_compare);
238:
239: RB_GENERATE(uvm_tree, vm_map_entry, rb_entry, uvm_compare);
240:
241: vsize_t
242: uvm_rb_space(struct vm_map *map, struct vm_map_entry *entry)
243: {
244: struct vm_map_entry *next;
245: vaddr_t space;
246:
247: if ((next = entry->next) == &map->header)
248: space = map->max_offset - entry->end;
249: else {
250: KASSERT(next);
251: space = next->start - entry->end;
252: }
253: return (space);
254: }
255:
256: vsize_t
257: uvm_rb_subtree_space(struct vm_map_entry *entry)
258: {
259: vaddr_t space, tmp;
260:
261: space = entry->ownspace;
262: if (RB_LEFT(entry, rb_entry)) {
263: tmp = RB_LEFT(entry, rb_entry)->space;
264: if (tmp > space)
265: space = tmp;
266: }
267:
268: if (RB_RIGHT(entry, rb_entry)) {
269: tmp = RB_RIGHT(entry, rb_entry)->space;
270: if (tmp > space)
271: space = tmp;
272: }
273:
274: return (space);
275: }
276:
277: void
278: uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry)
279: {
280: /* We need to traverse to the very top */
281: do {
282: entry->ownspace = uvm_rb_space(map, entry);
283: entry->space = uvm_rb_subtree_space(entry);
284: } while ((entry = RB_PARENT(entry, rb_entry)) != NULL);
285: }
286:
287: void
288: uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry)
289: {
290: vaddr_t space = uvm_rb_space(map, entry);
291: struct vm_map_entry *tmp;
292:
293: entry->ownspace = entry->space = space;
294: tmp = RB_INSERT(uvm_tree, &(map)->rbhead, entry);
295: #ifdef DIAGNOSTIC
296: if (tmp != NULL)
297: panic("uvm_rb_insert: duplicate entry?");
298: #endif
299: uvm_rb_fixup(map, entry);
300: if (entry->prev != &map->header)
301: uvm_rb_fixup(map, entry->prev);
302: }
303:
304: void
305: uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry)
306: {
307: struct vm_map_entry *parent;
308:
309: parent = RB_PARENT(entry, rb_entry);
310: RB_REMOVE(uvm_tree, &(map)->rbhead, entry);
311: if (entry->prev != &map->header)
312: uvm_rb_fixup(map, entry->prev);
313: if (parent)
314: uvm_rb_fixup(map, parent);
315: }
316:
317: #ifdef DEBUG
318: #define uvm_tree_sanity(x,y) _uvm_tree_sanity(x,y)
319: #else
320: #define uvm_tree_sanity(x,y)
321: #endif
322:
323: #ifdef DEBUG
324: int
325: _uvm_tree_sanity(struct vm_map *map, const char *name)
326: {
327: struct vm_map_entry *tmp, *trtmp;
328: int n = 0, i = 1;
329:
330: RB_FOREACH(tmp, uvm_tree, &map->rbhead) {
331: if (tmp->ownspace != uvm_rb_space(map, tmp)) {
332: printf("%s: %d/%d ownspace %x != %x %s\n",
333: name, n + 1, map->nentries,
334: tmp->ownspace, uvm_rb_space(map, tmp),
335: tmp->next == &map->header ? "(last)" : "");
336: goto error;
337: }
338: }
339: trtmp = NULL;
340: RB_FOREACH(tmp, uvm_tree, &map->rbhead) {
341: if (tmp->space != uvm_rb_subtree_space(tmp)) {
342: printf("%s: space %d != %d\n",
343: name, tmp->space, uvm_rb_subtree_space(tmp));
344: goto error;
345: }
346: if (trtmp != NULL && trtmp->start >= tmp->start) {
347: printf("%s: corrupt: 0x%lx >= 0x%lx\n",
348: name, trtmp->start, tmp->start);
349: goto error;
350: }
351: n++;
352:
353: trtmp = tmp;
354: }
355:
356: if (n != map->nentries) {
357: printf("%s: nentries: %d vs %d\n",
358: name, n, map->nentries);
359: goto error;
360: }
361:
362: for (tmp = map->header.next; tmp && tmp != &map->header;
363: tmp = tmp->next, i++) {
364: trtmp = RB_FIND(uvm_tree, &map->rbhead, tmp);
365: if (trtmp != tmp) {
366: printf("%s: lookup: %d: %p - %p: %p\n",
367: name, i, tmp, trtmp,
368: RB_PARENT(tmp, rb_entry));
369: goto error;
370: }
371: }
372:
373: return (0);
374: error:
375: #ifdef DDB
376: /* handy breakpoint location for error case */
377: __asm(".globl treesanity_label\ntreesanity_label:");
378: #endif
379: return (-1);
380: }
381: #endif
382:
383: /*
384: * uvm_mapent_alloc: allocate a map entry
385: */
386:
387: struct vm_map_entry *
388: uvm_mapent_alloc(struct vm_map *map)
389: {
390: struct vm_map_entry *me, *ne;
391: int s, i;
392: UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist);
393:
394: if (map->flags & VM_MAP_INTRSAFE || cold) {
395: s = splvm();
396: simple_lock(&uvm.kentry_lock);
397: me = uvm.kentry_free;
398: if (me == NULL) {
399: ne = uvm_km_getpage(0);
400: if (ne == NULL)
401: panic("uvm_mapent_alloc: cannot allocate map "
402: "entry");
403: for (i = 0;
404: i < PAGE_SIZE / sizeof(struct vm_map_entry) - 1;
405: i++)
406: ne[i].next = &ne[i + 1];
407: ne[i].next = NULL;
408: me = ne;
409: if (ratecheck(&uvm_kmapent_last_warn_time,
410: &uvm_kmapent_warn_rate))
411: printf("uvm_mapent_alloc: out of static "
412: "map entries\n");
413: }
414: uvm.kentry_free = me->next;
415: uvmexp.kmapent++;
416: simple_unlock(&uvm.kentry_lock);
417: splx(s);
418: me->flags = UVM_MAP_STATIC;
419: } else if (map == kernel_map) {
420: splassert(IPL_NONE);
421: me = pool_get(&uvm_map_entry_kmem_pool, PR_WAITOK);
422: me->flags = UVM_MAP_KMEM;
423: } else {
424: splassert(IPL_NONE);
425: me = pool_get(&uvm_map_entry_pool, PR_WAITOK);
426: me->flags = 0;
427: }
428:
429: UVMHIST_LOG(maphist, "<- new entry=%p [kentry=%ld]", me,
430: ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map), 0, 0);
431: return(me);
432: }
433:
434: /*
435: * uvm_mapent_free: free map entry
436: *
437: * => XXX: static pool for kernel map?
438: */
439:
440: void
441: uvm_mapent_free(struct vm_map_entry *me)
442: {
443: int s;
444: UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist);
445:
446: UVMHIST_LOG(maphist,"<- freeing map entry=%p [flags=%ld]",
447: me, me->flags, 0, 0);
448: if (me->flags & UVM_MAP_STATIC) {
449: s = splvm();
450: simple_lock(&uvm.kentry_lock);
451: me->next = uvm.kentry_free;
452: uvm.kentry_free = me;
453: uvmexp.kmapent--;
454: simple_unlock(&uvm.kentry_lock);
455: splx(s);
456: } else if (me->flags & UVM_MAP_KMEM) {
457: splassert(IPL_NONE);
458: pool_put(&uvm_map_entry_kmem_pool, me);
459: } else {
460: splassert(IPL_NONE);
461: pool_put(&uvm_map_entry_pool, me);
462: }
463: }
464:
465: /*
466: * uvm_mapent_copy: copy a map entry, preserving flags
467: */
468:
469: void
470: uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
471: {
472: memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) -
473: ((char *)src));
474: }
475:
476: /*
477: * uvm_map_entry_unwire: unwire a map entry
478: *
479: * => map should be locked by caller
480: */
481: void
482: uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry)
483: {
484:
485: entry->wired_count = 0;
486: uvm_fault_unwire_locked(map, entry->start, entry->end);
487: }
488:
489:
490: /*
491: * wrapper for calling amap_ref()
492: */
493: void
494: uvm_map_reference_amap(struct vm_map_entry *entry, int flags)
495: {
496: amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff,
497: (entry->end - entry->start) >> PAGE_SHIFT, flags);
498: }
499:
500:
501: /*
502: * wrapper for calling amap_unref()
503: */
504: void
505: uvm_map_unreference_amap(struct vm_map_entry *entry, int flags)
506: {
507: amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff,
508: (entry->end - entry->start) >> PAGE_SHIFT, flags);
509: }
510:
511:
512: /*
513: * uvm_map_init: init mapping system at boot time. note that we allocate
514: * and init the static pool of structs vm_map_entry for the kernel here.
515: */
516:
517: void
518: uvm_map_init(void)
519: {
520: static struct vm_map_entry kernel_map_entry[MAX_KMAPENT];
521: #if defined(UVMHIST)
522: static struct uvm_history_ent maphistbuf[100];
523: static struct uvm_history_ent pdhistbuf[100];
524: #endif
525: int lcv;
526:
527: /*
528: * first, init logging system.
529: */
530:
531: UVMHIST_FUNC("uvm_map_init");
532: UVMHIST_INIT_STATIC(maphist, maphistbuf);
533: UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
534: UVMHIST_CALLED(maphist);
535: UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
536: UVMCNT_INIT(uvm_map_call, UVMCNT_CNT, 0,
537: "# uvm_map() successful calls", 0);
538: UVMCNT_INIT(map_backmerge, UVMCNT_CNT, 0, "# uvm_map() back merges", 0);
539: UVMCNT_INIT(map_forwmerge, UVMCNT_CNT, 0, "# uvm_map() missed forward",
540: 0);
541: UVMCNT_INIT(uvm_mlk_call, UVMCNT_CNT, 0, "# map lookup calls", 0);
542: UVMCNT_INIT(uvm_mlk_hint, UVMCNT_CNT, 0, "# map lookup hint hits", 0);
543:
544: /*
545: * now set up static pool of kernel map entries ...
546: */
547:
548: simple_lock_init(&uvm.kentry_lock);
549: uvm.kentry_free = NULL;
550: for (lcv = 0 ; lcv < MAX_KMAPENT ; lcv++) {
551: kernel_map_entry[lcv].next = uvm.kentry_free;
552: uvm.kentry_free = &kernel_map_entry[lcv];
553: }
554:
555: /*
556: * initialize the map-related pools.
557: */
558: pool_init(&uvm_vmspace_pool, sizeof(struct vmspace),
559: 0, 0, 0, "vmsppl", &pool_allocator_nointr);
560: pool_init(&uvm_map_entry_pool, sizeof(struct vm_map_entry),
561: 0, 0, 0, "vmmpepl", &pool_allocator_nointr);
562: pool_init(&uvm_map_entry_kmem_pool, sizeof(struct vm_map_entry),
563: 0, 0, 0, "vmmpekpl", NULL);
564: pool_sethiwat(&uvm_map_entry_pool, 8192);
565: }
566:
567: /*
568: * clippers
569: */
570:
571: /*
572: * uvm_map_clip_start: ensure that the entry begins at or after
573: * the starting address, if it doesn't we split the entry.
574: *
575: * => caller should use UVM_MAP_CLIP_START macro rather than calling
576: * this directly
577: * => map must be locked by caller
578: */
579:
580: void
581: uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry,
582: vaddr_t start)
583: {
584: struct vm_map_entry *new_entry;
585: vaddr_t new_adj;
586:
587: /* uvm_map_simplify_entry(map, entry); */ /* XXX */
588:
589: uvm_tree_sanity(map, "clip_start entry");
590:
591: /*
592: * Split off the front portion. note that we must insert the new
593: * entry BEFORE this one, so that this entry has the specified
594: * starting address.
595: */
596:
597: new_entry = uvm_mapent_alloc(map);
598: uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
599:
600: new_entry->end = start;
601: new_adj = start - new_entry->start;
602: if (entry->object.uvm_obj)
603: entry->offset += new_adj; /* shift start over */
604:
605: /* Does not change order for the RB tree */
606: entry->start = start;
607:
608: if (new_entry->aref.ar_amap) {
609: amap_splitref(&new_entry->aref, &entry->aref, new_adj);
610: }
611:
612: uvm_map_entry_link(map, entry->prev, new_entry);
613:
614: if (UVM_ET_ISSUBMAP(entry)) {
615: /* ... unlikely to happen, but play it safe */
616: uvm_map_reference(new_entry->object.sub_map);
617: } else {
618: if (UVM_ET_ISOBJ(entry) &&
619: entry->object.uvm_obj->pgops &&
620: entry->object.uvm_obj->pgops->pgo_reference)
621: entry->object.uvm_obj->pgops->pgo_reference(
622: entry->object.uvm_obj);
623: }
624:
625: uvm_tree_sanity(map, "clip_start leave");
626: }
627:
628: /*
629: * uvm_map_clip_end: ensure that the entry ends at or before
630: * the ending address, if it doesn't we split the reference
631: *
632: * => caller should use UVM_MAP_CLIP_END macro rather than calling
633: * this directly
634: * => map must be locked by caller
635: */
636:
637: void
638: uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end)
639: {
640: struct vm_map_entry *new_entry;
641: vaddr_t new_adj; /* #bytes we move start forward */
642:
643: uvm_tree_sanity(map, "clip_end entry");
644: /*
645: * Create a new entry and insert it
646: * AFTER the specified entry
647: */
648:
649: new_entry = uvm_mapent_alloc(map);
650: uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
651:
652: new_entry->start = entry->end = end;
653: new_adj = end - entry->start;
654: if (new_entry->object.uvm_obj)
655: new_entry->offset += new_adj;
656:
657: if (entry->aref.ar_amap)
658: amap_splitref(&entry->aref, &new_entry->aref, new_adj);
659:
660: uvm_rb_fixup(map, entry);
661:
662: uvm_map_entry_link(map, entry, new_entry);
663:
664: if (UVM_ET_ISSUBMAP(entry)) {
665: /* ... unlikely to happen, but play it safe */
666: uvm_map_reference(new_entry->object.sub_map);
667: } else {
668: if (UVM_ET_ISOBJ(entry) &&
669: entry->object.uvm_obj->pgops &&
670: entry->object.uvm_obj->pgops->pgo_reference)
671: entry->object.uvm_obj->pgops->pgo_reference(
672: entry->object.uvm_obj);
673: }
674: uvm_tree_sanity(map, "clip_end leave");
675: }
676:
677:
678: /*
679: * M A P - m a i n e n t r y p o i n t
680: */
681: /*
682: * uvm_map: establish a valid mapping in a map
683: *
684: * => assume startp is page aligned.
685: * => assume size is a multiple of PAGE_SIZE.
686: * => assume sys_mmap provides enough of a "hint" to have us skip
687: * over text/data/bss area.
688: * => map must be unlocked (we will lock it)
689: * => <uobj,uoffset> value meanings (4 cases):
690: * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER
691: * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER
692: * [3] <uobj,uoffset> == normal mapping
693: * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA
694: *
695: * case [4] is for kernel mappings where we don't know the offset until
696: * we've found a virtual address. note that kernel object offsets are
697: * always relative to vm_map_min(kernel_map).
698: *
699: * => if `align' is non-zero, we try to align the virtual address to
700: * the specified alignment. this is only a hint; if we can't
701: * do it, the address will be unaligned. this is provided as
702: * a mechanism for large pages.
703: *
704: * => XXXCDC: need way to map in external amap?
705: */
706:
707: int
708: uvm_map_p(struct vm_map *map, vaddr_t *startp, vsize_t size,
709: struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags,
710: struct proc *p)
711: {
712: struct vm_map_entry *prev_entry, *new_entry;
713: vm_prot_t prot = UVM_PROTECTION(flags), maxprot =
714: UVM_MAXPROTECTION(flags);
715: vm_inherit_t inherit = UVM_INHERIT(flags);
716: int advice = UVM_ADVICE(flags);
717: int error;
718: UVMHIST_FUNC("uvm_map");
719: UVMHIST_CALLED(maphist);
720:
721: UVMHIST_LOG(maphist, "(map=%p, *startp=0x%lx, size=%ld, flags=0x%lx)",
722: map, *startp, size, flags);
723: UVMHIST_LOG(maphist, " uobj/offset %p/%ld", uobj, (u_long)uoffset,0,0);
724:
725: uvm_tree_sanity(map, "map entry");
726:
727: if ((map->flags & VM_MAP_INTRSAFE) == 0)
728: splassert(IPL_NONE);
729:
730: /*
731: * step 0: sanity check of protection code
732: */
733:
734: if ((prot & maxprot) != prot) {
735: UVMHIST_LOG(maphist, "<- prot. failure: prot=0x%lx, max=0x%lx",
736: prot, maxprot,0,0);
737: return (EACCES);
738: }
739:
740: /*
741: * step 1: figure out where to put new VM range
742: */
743:
744: if (vm_map_lock_try(map) == FALSE) {
745: if (flags & UVM_FLAG_TRYLOCK)
746: return (EFAULT);
747: vm_map_lock(map); /* could sleep here */
748: }
749: if ((prev_entry = uvm_map_findspace(map, *startp, size, startp,
750: uobj, uoffset, align, flags)) == NULL) {
751: UVMHIST_LOG(maphist,"<- uvm_map_findspace failed!",0,0,0,0);
752: vm_map_unlock(map);
753: return (ENOMEM);
754: }
755:
756: #ifdef PMAP_GROWKERNEL
757: {
758: /*
759: * If the kernel pmap can't map the requested space,
760: * then allocate more resources for it.
761: */
762: if (map == kernel_map && uvm_maxkaddr < (*startp + size))
763: uvm_maxkaddr = pmap_growkernel(*startp + size);
764: }
765: #endif
766:
767: UVMCNT_INCR(uvm_map_call);
768:
769: /*
770: * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
771: * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET. in
772: * either case we want to zero it before storing it in the map entry
773: * (because it looks strange and confusing when debugging...)
774: *
775: * if uobj is not null
776: * if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
777: * and we do not need to change uoffset.
778: * if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
779: * now (based on the starting address of the map). this case is
780: * for kernel object mappings where we don't know the offset until
781: * the virtual address is found (with uvm_map_findspace). the
782: * offset is the distance we are from the start of the map.
783: */
784:
785: if (uobj == NULL) {
786: uoffset = 0;
787: } else {
788: if (uoffset == UVM_UNKNOWN_OFFSET) {
789: KASSERT(UVM_OBJ_IS_KERN_OBJECT(uobj));
790: uoffset = *startp - vm_map_min(kernel_map);
791: }
792: }
793:
794: /*
795: * step 2: try and insert in map by extending previous entry, if
796: * possible
797: * XXX: we don't try and pull back the next entry. might be useful
798: * for a stack, but we are currently allocating our stack in advance.
799: */
800:
801: if ((flags & UVM_FLAG_NOMERGE) == 0 &&
802: prev_entry->end == *startp && prev_entry != &map->header &&
803: prev_entry->object.uvm_obj == uobj) {
804:
805: if (uobj && prev_entry->offset +
806: (prev_entry->end - prev_entry->start) != uoffset)
807: goto step3;
808:
809: if (UVM_ET_ISSUBMAP(prev_entry))
810: goto step3;
811:
812: if (prev_entry->protection != prot ||
813: prev_entry->max_protection != maxprot)
814: goto step3;
815:
816: if (prev_entry->inheritance != inherit ||
817: prev_entry->advice != advice)
818: goto step3;
819:
820: /* wiring status must match (new area is unwired) */
821: if (VM_MAPENT_ISWIRED(prev_entry))
822: goto step3;
823:
824: /*
825: * can't extend a shared amap. note: no need to lock amap to
826: * look at refs since we don't care about its exact value.
827: * if it is one (i.e. we have only reference) it will stay there
828: */
829:
830: if (prev_entry->aref.ar_amap &&
831: amap_refs(prev_entry->aref.ar_amap) != 1) {
832: goto step3;
833: }
834:
835: if (prev_entry->aref.ar_amap) {
836: error = amap_extend(prev_entry, size);
837: if (error) {
838: vm_map_unlock(map);
839: return (error);
840: }
841: }
842:
843: UVMCNT_INCR(map_backmerge);
844: UVMHIST_LOG(maphist," starting back merge", 0, 0, 0, 0);
845:
846: /*
847: * drop our reference to uobj since we are extending a reference
848: * that we already have (the ref count can not drop to zero).
849: */
850:
851: if (uobj && uobj->pgops->pgo_detach)
852: uobj->pgops->pgo_detach(uobj);
853:
854: prev_entry->end += size;
855: uvm_rb_fixup(map, prev_entry);
856: map->size += size;
857: if (p && uobj == NULL)
858: p->p_vmspace->vm_dused += btoc(size);
859:
860: uvm_tree_sanity(map, "map leave 2");
861:
862: UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
863: vm_map_unlock(map);
864: return (0);
865:
866: }
867: step3:
868: UVMHIST_LOG(maphist," allocating new map entry", 0, 0, 0, 0);
869:
870: /*
871: * check for possible forward merge (which we don't do) and count
872: * the number of times we missed a *possible* chance to merge more
873: */
874:
875: if ((flags & UVM_FLAG_NOMERGE) == 0 &&
876: prev_entry->next != &map->header &&
877: prev_entry->next->start == (*startp + size))
878: UVMCNT_INCR(map_forwmerge);
879:
880: /*
881: * step 3: allocate new entry and link it in
882: */
883:
884: new_entry = uvm_mapent_alloc(map);
885: new_entry->start = *startp;
886: new_entry->end = new_entry->start + size;
887: new_entry->object.uvm_obj = uobj;
888: new_entry->offset = uoffset;
889:
890: if (uobj)
891: new_entry->etype = UVM_ET_OBJ;
892: else
893: new_entry->etype = 0;
894:
895: if (flags & UVM_FLAG_COPYONW) {
896: new_entry->etype |= UVM_ET_COPYONWRITE;
897: if ((flags & UVM_FLAG_OVERLAY) == 0)
898: new_entry->etype |= UVM_ET_NEEDSCOPY;
899: }
900:
901: new_entry->protection = prot;
902: new_entry->max_protection = maxprot;
903: new_entry->inheritance = inherit;
904: new_entry->wired_count = 0;
905: new_entry->advice = advice;
906: if (flags & UVM_FLAG_OVERLAY) {
907: /*
908: * to_add: for BSS we overallocate a little since we
909: * are likely to extend
910: */
911: vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
912: UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
913: struct vm_amap *amap = amap_alloc(size, to_add, M_WAITOK);
914: new_entry->aref.ar_pageoff = 0;
915: new_entry->aref.ar_amap = amap;
916: } else {
917: new_entry->aref.ar_pageoff = 0;
918: new_entry->aref.ar_amap = NULL;
919: }
920:
921: uvm_map_entry_link(map, prev_entry, new_entry);
922:
923: map->size += size;
924: if (p && uobj == NULL)
925: p->p_vmspace->vm_dused += btoc(size);
926:
927:
928: /*
929: * Update the free space hint
930: */
931:
932: if ((map->first_free == prev_entry) &&
933: (prev_entry->end >= new_entry->start))
934: map->first_free = new_entry;
935:
936: uvm_tree_sanity(map, "map leave");
937:
938: UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
939: vm_map_unlock(map);
940: return (0);
941: }
942:
943: /*
944: * uvm_map_lookup_entry: find map entry at or before an address
945: *
946: * => map must at least be read-locked by caller
947: * => entry is returned in "entry"
948: * => return value is true if address is in the returned entry
949: */
950:
951: boolean_t
952: uvm_map_lookup_entry(struct vm_map *map, vaddr_t address,
953: struct vm_map_entry **entry)
954: {
955: struct vm_map_entry *cur;
956: struct vm_map_entry *last;
957: int use_tree = 0;
958: UVMHIST_FUNC("uvm_map_lookup_entry");
959: UVMHIST_CALLED(maphist);
960:
961: UVMHIST_LOG(maphist,"(map=%p,addr=0x%lx,ent=%p)",
962: map, address, entry, 0);
963:
964: /*
965: * start looking either from the head of the
966: * list, or from the hint.
967: */
968:
969: simple_lock(&map->hint_lock);
970: cur = map->hint;
971: simple_unlock(&map->hint_lock);
972:
973: if (cur == &map->header)
974: cur = cur->next;
975:
976: UVMCNT_INCR(uvm_mlk_call);
977: if (address >= cur->start) {
978: /*
979: * go from hint to end of list.
980: *
981: * but first, make a quick check to see if
982: * we are already looking at the entry we
983: * want (which is usually the case).
984: * note also that we don't need to save the hint
985: * here... it is the same hint (unless we are
986: * at the header, in which case the hint didn't
987: * buy us anything anyway).
988: */
989: last = &map->header;
990: if ((cur != last) && (cur->end > address)) {
991: UVMCNT_INCR(uvm_mlk_hint);
992: *entry = cur;
993: UVMHIST_LOG(maphist,"<- got it via hint (%p)",
994: cur, 0, 0, 0);
995: return (TRUE);
996: }
997:
998: if (map->nentries > 30)
999: use_tree = 1;
1000: } else {
1001: /*
1002: * go from start to hint, *inclusively*
1003: */
1004: last = cur->next;
1005: cur = map->header.next;
1006: use_tree = 1;
1007: }
1008:
1009: uvm_tree_sanity(map, __func__);
1010:
1011: if (use_tree) {
1012: struct vm_map_entry *prev = &map->header;
1013: cur = RB_ROOT(&map->rbhead);
1014:
1015: /*
1016: * Simple lookup in the tree. Happens when the hint is
1017: * invalid, or nentries reach a threshold.
1018: */
1019: while (cur) {
1020: if (address >= cur->start) {
1021: if (address < cur->end) {
1022: *entry = cur;
1023: SAVE_HINT(map, map->hint, cur);
1024: return (TRUE);
1025: }
1026: prev = cur;
1027: cur = RB_RIGHT(cur, rb_entry);
1028: } else
1029: cur = RB_LEFT(cur, rb_entry);
1030: }
1031: *entry = prev;
1032: UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1033: return (FALSE);
1034: }
1035:
1036: /*
1037: * search linearly
1038: */
1039:
1040: while (cur != last) {
1041: if (cur->end > address) {
1042: if (address >= cur->start) {
1043: /*
1044: * save this lookup for future
1045: * hints, and return
1046: */
1047:
1048: *entry = cur;
1049: SAVE_HINT(map, map->hint, cur);
1050: UVMHIST_LOG(maphist,"<- search got it (%p)",
1051: cur, 0, 0, 0);
1052: return (TRUE);
1053: }
1054: break;
1055: }
1056: cur = cur->next;
1057: }
1058:
1059: *entry = cur->prev;
1060: SAVE_HINT(map, map->hint, *entry);
1061: UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1062: return (FALSE);
1063: }
1064:
1065: /*
1066: * Checks if address pointed to be phint fits into the empty
1067: * space before the vm_map_entry after. Takes aligment and
1068: * offset into consideration.
1069: */
1070:
1071: int
1072: uvm_map_spacefits(struct vm_map *map, vaddr_t *phint, vsize_t length,
1073: struct vm_map_entry *after, voff_t uoffset, vsize_t align)
1074: {
1075: vaddr_t hint = *phint;
1076: vaddr_t end;
1077:
1078: #ifdef PMAP_PREFER
1079: /*
1080: * push hint forward as needed to avoid VAC alias problems.
1081: * we only do this if a valid offset is specified.
1082: */
1083: if (uoffset != UVM_UNKNOWN_OFFSET)
1084: PMAP_PREFER(uoffset, &hint);
1085: #endif
1086: if (align != 0)
1087: if ((hint & (align - 1)) != 0)
1088: hint = roundup(hint, align);
1089: *phint = hint;
1090:
1091: end = hint + length;
1092: if (end > map->max_offset || end < hint)
1093: return (FALSE);
1094: if (after != NULL && after != &map->header && after->start < end)
1095: return (FALSE);
1096:
1097: return (TRUE);
1098: }
1099:
1100: /*
1101: * uvm_map_hint: return the beginning of the best area suitable for
1102: * creating a new mapping with "prot" protection.
1103: */
1104: vaddr_t
1105: uvm_map_hint(struct proc *p, vm_prot_t prot)
1106: {
1107: vaddr_t addr;
1108:
1109: #ifdef __i386__
1110: /*
1111: * If executable skip first two pages, otherwise start
1112: * after data + heap region.
1113: */
1114: if ((prot & VM_PROT_EXECUTE) &&
1115: ((vaddr_t)p->p_vmspace->vm_daddr >= I386_MAX_EXE_ADDR)) {
1116: addr = (PAGE_SIZE*2) +
1117: (arc4random() & (I386_MAX_EXE_ADDR / 2 - 1));
1118: return (round_page(addr));
1119: }
1120: #endif
1121: addr = (vaddr_t)p->p_vmspace->vm_daddr + MAXDSIZ;
1122: #if !defined(__vax__)
1123: addr += arc4random() & (MIN((256 * 1024 * 1024), MAXDSIZ) - 1);
1124: #else
1125: /* start malloc/mmap after the brk */
1126: addr = (vaddr_t)p->p_vmspace->vm_daddr + BRKSIZ;
1127: #endif
1128: return (round_page(addr));
1129: }
1130:
1131: /*
1132: * uvm_map_findspace: find "length" sized space in "map".
1133: *
1134: * => "hint" is a hint about where we want it, unless FINDSPACE_FIXED is
1135: * set (in which case we insist on using "hint").
1136: * => "result" is VA returned
1137: * => uobj/uoffset are to be used to handle VAC alignment, if required
1138: * => if `align' is non-zero, we attempt to align to that value.
1139: * => caller must at least have read-locked map
1140: * => returns NULL on failure, or pointer to prev. map entry if success
1141: * => note this is a cross between the old vm_map_findspace and vm_map_find
1142: */
1143:
1144: struct vm_map_entry *
1145: uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length,
1146: vaddr_t *result, struct uvm_object *uobj, voff_t uoffset, vsize_t align,
1147: int flags)
1148: {
1149: struct vm_map_entry *entry, *next, *tmp;
1150: struct vm_map_entry *child, *prev = NULL;
1151:
1152: vaddr_t end, orig_hint;
1153: UVMHIST_FUNC("uvm_map_findspace");
1154: UVMHIST_CALLED(maphist);
1155:
1156: UVMHIST_LOG(maphist, "(map=%p, hint=0x%lx, len=%ld, flags=0x%lx)",
1157: map, hint, length, flags);
1158: KASSERT((align & (align - 1)) == 0);
1159: KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
1160:
1161: uvm_tree_sanity(map, "map_findspace entry");
1162:
1163: /*
1164: * remember the original hint. if we are aligning, then we
1165: * may have to try again with no alignment constraint if
1166: * we fail the first time.
1167: */
1168:
1169: orig_hint = hint;
1170: if (hint < map->min_offset) { /* check ranges ... */
1171: if (flags & UVM_FLAG_FIXED) {
1172: UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
1173: return(NULL);
1174: }
1175: hint = map->min_offset;
1176: }
1177: if (hint > map->max_offset) {
1178: UVMHIST_LOG(maphist,"<- VA 0x%lx > range [0x%lx->0x%lx]",
1179: hint, map->min_offset, map->max_offset, 0);
1180: return(NULL);
1181: }
1182:
1183: /*
1184: * Look for the first possible address; if there's already
1185: * something at this address, we have to start after it.
1186: */
1187:
1188: if ((flags & UVM_FLAG_FIXED) == 0 && hint == map->min_offset) {
1189: if ((entry = map->first_free) != &map->header)
1190: hint = entry->end;
1191: } else {
1192: if (uvm_map_lookup_entry(map, hint, &tmp)) {
1193: /* "hint" address already in use ... */
1194: if (flags & UVM_FLAG_FIXED) {
1195: UVMHIST_LOG(maphist,"<- fixed & VA in use",
1196: 0, 0, 0, 0);
1197: return(NULL);
1198: }
1199: hint = tmp->end;
1200: }
1201: entry = tmp;
1202: }
1203:
1204: if (flags & UVM_FLAG_FIXED) {
1205: end = hint + length;
1206: if (end > map->max_offset || end < hint) {
1207: UVMHIST_LOG(maphist,"<- failed (off end)", 0,0,0,0);
1208: goto error;
1209: }
1210: next = entry->next;
1211: if (next == &map->header || next->start >= end)
1212: goto found;
1213: UVMHIST_LOG(maphist,"<- fixed mapping failed", 0,0,0,0);
1214: return(NULL); /* only one shot at it ... */
1215: }
1216:
1217: /* Try to find the space in the red-black tree */
1218:
1219: /* Check slot before any entry */
1220: if (uvm_map_spacefits(map, &hint, length, entry->next, uoffset, align))
1221: goto found;
1222:
1223: /* If there is not enough space in the whole tree, we fail */
1224: tmp = RB_ROOT(&map->rbhead);
1225: if (tmp == NULL || tmp->space < length)
1226: goto error;
1227:
1228: /* Find an entry close to hint that has enough space */
1229: for (; tmp;) {
1230: if (tmp->end >= hint &&
1231: (prev == NULL || tmp->end < prev->end)) {
1232: if (tmp->ownspace >= length)
1233: prev = tmp;
1234: else if ((child = RB_RIGHT(tmp, rb_entry)) != NULL &&
1235: child->space >= length)
1236: prev = tmp;
1237: }
1238: if (tmp->end < hint)
1239: child = RB_RIGHT(tmp, rb_entry);
1240: else if (tmp->end > hint)
1241: child = RB_LEFT(tmp, rb_entry);
1242: else {
1243: if (tmp->ownspace >= length)
1244: break;
1245: child = RB_RIGHT(tmp, rb_entry);
1246: }
1247: if (child == NULL || child->space < length)
1248: break;
1249: tmp = child;
1250: }
1251:
1252: if (tmp != NULL && hint < tmp->end + tmp->ownspace) {
1253: /*
1254: * Check if the entry that we found satifies the
1255: * space requirement
1256: */
1257: if (hint < tmp->end)
1258: hint = tmp->end;
1259: if (uvm_map_spacefits(map, &hint, length, tmp->next, uoffset,
1260: align)) {
1261: entry = tmp;
1262: goto found;
1263: } else if (tmp->ownspace >= length)
1264: goto listsearch;
1265: }
1266: if (prev == NULL)
1267: goto error;
1268:
1269: hint = prev->end;
1270: if (uvm_map_spacefits(map, &hint, length, prev->next, uoffset,
1271: align)) {
1272: entry = prev;
1273: goto found;
1274: } else if (prev->ownspace >= length)
1275: goto listsearch;
1276:
1277: tmp = RB_RIGHT(prev, rb_entry);
1278: for (;;) {
1279: KASSERT(tmp && tmp->space >= length);
1280: child = RB_LEFT(tmp, rb_entry);
1281: if (child && child->space >= length) {
1282: tmp = child;
1283: continue;
1284: }
1285: if (tmp->ownspace >= length)
1286: break;
1287: tmp = RB_RIGHT(tmp, rb_entry);
1288: }
1289:
1290: hint = tmp->end;
1291: if (uvm_map_spacefits(map, &hint, length, tmp->next, uoffset, align)) {
1292: entry = tmp;
1293: goto found;
1294: }
1295:
1296: /*
1297: * The tree fails to find an entry because of offset or alignment
1298: * restrictions. Search the list instead.
1299: */
1300: listsearch:
1301: /*
1302: * Look through the rest of the map, trying to fit a new region in
1303: * the gap between existing regions, or after the very last region.
1304: * note: entry->end = base VA of current gap,
1305: * next->start = VA of end of current gap
1306: */
1307: for (;; hint = (entry = next)->end) {
1308: /*
1309: * Find the end of the proposed new region. Be sure we didn't
1310: * go beyond the end of the map, or wrap around the address;
1311: * if so, we lose. Otherwise, if this is the last entry, or
1312: * if the proposed new region fits before the next entry, we
1313: * win.
1314: */
1315:
1316: #ifdef PMAP_PREFER
1317: /*
1318: * push hint forward as needed to avoid VAC alias problems.
1319: * we only do this if a valid offset is specified.
1320: */
1321: if (uoffset != UVM_UNKNOWN_OFFSET)
1322: PMAP_PREFER(uoffset, &hint);
1323: #endif
1324: if (align != 0) {
1325: if ((hint & (align - 1)) != 0)
1326: hint = roundup(hint, align);
1327: /*
1328: * XXX Should we PMAP_PREFER() here again?
1329: */
1330: }
1331: end = hint + length;
1332: if (end > map->max_offset || end < hint) {
1333: UVMHIST_LOG(maphist,"<- failed (off end)", 0,0,0,0);
1334: goto error;
1335: }
1336: next = entry->next;
1337: if (next == &map->header || next->start >= end)
1338: break;
1339: }
1340: found:
1341: SAVE_HINT(map, map->hint, entry);
1342: *result = hint;
1343: UVMHIST_LOG(maphist,"<- got it! (result=0x%lx)", hint, 0,0,0);
1344: return (entry);
1345:
1346: error:
1347: if (align != 0) {
1348: UVMHIST_LOG(maphist,
1349: "calling recursively, no align",
1350: 0,0,0,0);
1351: return (uvm_map_findspace(map, orig_hint,
1352: length, result, uobj, uoffset, 0, flags));
1353: }
1354: return (NULL);
1355: }
1356:
1357: /*
1358: * U N M A P - m a i n h e l p e r f u n c t i o n s
1359: */
1360:
1361: /*
1362: * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
1363: *
1364: * => caller must check alignment and size
1365: * => map must be locked by caller
1366: * => we return a list of map entries that we've remove from the map
1367: * in "entry_list"
1368: */
1369:
1370: void
1371: uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
1372: struct vm_map_entry **entry_list, struct proc *p)
1373: {
1374: struct vm_map_entry *entry, *first_entry, *next;
1375: vaddr_t len;
1376: UVMHIST_FUNC("uvm_unmap_remove");
1377: UVMHIST_CALLED(maphist);
1378:
1379: UVMHIST_LOG(maphist,"(map=%p, start=0x%lx, end=0x%lx)",
1380: map, start, end, 0);
1381:
1382: VM_MAP_RANGE_CHECK(map, start, end);
1383:
1384: uvm_tree_sanity(map, "unmap_remove entry");
1385:
1386: if ((map->flags & VM_MAP_INTRSAFE) == 0)
1387: splassert(IPL_NONE);
1388:
1389: /*
1390: * find first entry
1391: */
1392: if (uvm_map_lookup_entry(map, start, &first_entry) == TRUE) {
1393: /* clip and go... */
1394: entry = first_entry;
1395: UVM_MAP_CLIP_START(map, entry, start);
1396: /* critical! prevents stale hint */
1397: SAVE_HINT(map, entry, entry->prev);
1398:
1399: } else {
1400: entry = first_entry->next;
1401: }
1402:
1403: /*
1404: * Save the free space hint
1405: */
1406:
1407: if (map->first_free->start >= start)
1408: map->first_free = entry->prev;
1409:
1410: /*
1411: * note: we now re-use first_entry for a different task. we remove
1412: * a number of map entries from the map and save them in a linked
1413: * list headed by "first_entry". once we remove them from the map
1414: * the caller should unlock the map and drop the references to the
1415: * backing objects [c.f. uvm_unmap_detach]. the object is to
1416: * separate unmapping from reference dropping. why?
1417: * [1] the map has to be locked for unmapping
1418: * [2] the map need not be locked for reference dropping
1419: * [3] dropping references may trigger pager I/O, and if we hit
1420: * a pager that does synchronous I/O we may have to wait for it.
1421: * [4] we would like all waiting for I/O to occur with maps unlocked
1422: * so that we don't block other threads.
1423: */
1424: first_entry = NULL;
1425: *entry_list = NULL; /* to be safe */
1426:
1427: /*
1428: * break up the area into map entry sized regions and unmap. note
1429: * that all mappings have to be removed before we can even consider
1430: * dropping references to amaps or VM objects (otherwise we could end
1431: * up with a mapping to a page on the free list which would be very bad)
1432: */
1433:
1434: while ((entry != &map->header) && (entry->start < end)) {
1435:
1436: UVM_MAP_CLIP_END(map, entry, end);
1437: next = entry->next;
1438: len = entry->end - entry->start;
1439: if (p && entry->object.uvm_obj == NULL)
1440: p->p_vmspace->vm_dused -= btoc(len);
1441:
1442: /*
1443: * unwire before removing addresses from the pmap; otherwise
1444: * unwiring will put the entries back into the pmap (XXX).
1445: */
1446:
1447: if (VM_MAPENT_ISWIRED(entry))
1448: uvm_map_entry_unwire(map, entry);
1449:
1450: /*
1451: * special case: handle mappings to anonymous kernel objects.
1452: * we want to free these pages right away...
1453: */
1454: if (map->flags & VM_MAP_INTRSAFE) {
1455: uvm_km_pgremove_intrsafe(entry->start, entry->end);
1456: pmap_kremove(entry->start, len);
1457: } else if (UVM_ET_ISOBJ(entry) &&
1458: UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
1459: KASSERT(vm_map_pmap(map) == pmap_kernel());
1460:
1461: /*
1462: * note: kernel object mappings are currently used in
1463: * two ways:
1464: * [1] "normal" mappings of pages in the kernel object
1465: * [2] uvm_km_valloc'd allocations in which we
1466: * pmap_enter in some non-kernel-object page
1467: * (e.g. vmapbuf).
1468: *
1469: * for case [1], we need to remove the mapping from
1470: * the pmap and then remove the page from the kernel
1471: * object (because, once pages in a kernel object are
1472: * unmapped they are no longer needed, unlike, say,
1473: * a vnode where you might want the data to persist
1474: * until flushed out of a queue).
1475: *
1476: * for case [2], we need to remove the mapping from
1477: * the pmap. there shouldn't be any pages at the
1478: * specified offset in the kernel object [but it
1479: * doesn't hurt to call uvm_km_pgremove just to be
1480: * safe?]
1481: *
1482: * uvm_km_pgremove currently does the following:
1483: * for pages in the kernel object in range:
1484: * - drops the swap slot
1485: * - uvm_pagefree the page
1486: *
1487: * note there is version of uvm_km_pgremove() that
1488: * is used for "intrsafe" objects.
1489: */
1490:
1491: /*
1492: * remove mappings from pmap and drop the pages
1493: * from the object. offsets are always relative
1494: * to vm_map_min(kernel_map).
1495: */
1496: pmap_remove(pmap_kernel(), entry->start, entry->end);
1497: uvm_km_pgremove(entry->object.uvm_obj,
1498: entry->start - vm_map_min(kernel_map),
1499: entry->end - vm_map_min(kernel_map));
1500:
1501: /*
1502: * null out kernel_object reference, we've just
1503: * dropped it
1504: */
1505: entry->etype &= ~UVM_ET_OBJ;
1506: entry->object.uvm_obj = NULL; /* to be safe */
1507:
1508: } else {
1509: /*
1510: * remove mappings the standard way.
1511: */
1512: pmap_remove(map->pmap, entry->start, entry->end);
1513: }
1514:
1515: /*
1516: * remove entry from map and put it on our list of entries
1517: * that we've nuked. then go do next entry.
1518: */
1519: UVMHIST_LOG(maphist, " removed map entry %p", entry, 0, 0,0);
1520:
1521: /* critical! prevents stale hint */
1522: SAVE_HINT(map, entry, entry->prev);
1523:
1524: uvm_map_entry_unlink(map, entry);
1525: map->size -= len;
1526: entry->next = first_entry;
1527: first_entry = entry;
1528: entry = next; /* next entry, please */
1529: }
1530: /* if ((map->flags & VM_MAP_DYING) == 0) { */
1531: pmap_update(vm_map_pmap(map));
1532: /* } */
1533:
1534:
1535: uvm_tree_sanity(map, "unmap_remove leave");
1536:
1537: /*
1538: * now we've cleaned up the map and are ready for the caller to drop
1539: * references to the mapped objects.
1540: */
1541:
1542: *entry_list = first_entry;
1543: UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1544: }
1545:
1546: /*
1547: * uvm_unmap_detach: drop references in a chain of map entries
1548: *
1549: * => we will free the map entries as we traverse the list.
1550: */
1551:
1552: void
1553: uvm_unmap_detach(struct vm_map_entry *first_entry, int flags)
1554: {
1555: struct vm_map_entry *next_entry;
1556: UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
1557:
1558: while (first_entry) {
1559: KASSERT(!VM_MAPENT_ISWIRED(first_entry));
1560: UVMHIST_LOG(maphist,
1561: " detach 0x%lx: amap=%p, obj=%p, submap?=%ld",
1562: first_entry, first_entry->aref.ar_amap,
1563: first_entry->object.uvm_obj,
1564: UVM_ET_ISSUBMAP(first_entry));
1565:
1566: /*
1567: * drop reference to amap, if we've got one
1568: */
1569:
1570: if (first_entry->aref.ar_amap)
1571: uvm_map_unreference_amap(first_entry, flags);
1572:
1573: /*
1574: * drop reference to our backing object, if we've got one
1575: */
1576:
1577: if (UVM_ET_ISSUBMAP(first_entry)) {
1578: /* ... unlikely to happen, but play it safe */
1579: uvm_map_deallocate(first_entry->object.sub_map);
1580: } else {
1581: if (UVM_ET_ISOBJ(first_entry) &&
1582: first_entry->object.uvm_obj->pgops->pgo_detach)
1583: first_entry->object.uvm_obj->pgops->
1584: pgo_detach(first_entry->object.uvm_obj);
1585: }
1586:
1587: next_entry = first_entry->next;
1588: uvm_mapent_free(first_entry);
1589: first_entry = next_entry;
1590: }
1591: UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
1592: }
1593:
1594: /*
1595: * E X T R A C T I O N F U N C T I O N S
1596: */
1597:
1598: /*
1599: * uvm_map_reserve: reserve space in a vm_map for future use.
1600: *
1601: * => we reserve space in a map by putting a dummy map entry in the
1602: * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
1603: * => map should be unlocked (we will write lock it)
1604: * => we return true if we were able to reserve space
1605: * => XXXCDC: should be inline?
1606: */
1607:
1608: int
1609: uvm_map_reserve(struct vm_map *map, vsize_t size, vaddr_t offset,
1610: vsize_t align, vaddr_t *raddr)
1611: {
1612: UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
1613:
1614: UVMHIST_LOG(maphist, "(map=%p, size=0x%lx, offset=0x%lx,addr=0x%lx)",
1615: map,size,offset,raddr);
1616:
1617: size = round_page(size);
1618: if (*raddr < vm_map_min(map))
1619: *raddr = vm_map_min(map); /* hint */
1620:
1621: /*
1622: * reserve some virtual space.
1623: */
1624:
1625: if (uvm_map(map, raddr, size, NULL, offset, 0,
1626: UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
1627: UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != 0) {
1628: UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
1629: return (FALSE);
1630: }
1631:
1632: UVMHIST_LOG(maphist, "<- done (*raddr=0x%lx)", *raddr,0,0,0);
1633: return (TRUE);
1634: }
1635:
1636: /*
1637: * uvm_map_replace: replace a reserved (blank) area of memory with
1638: * real mappings.
1639: *
1640: * => caller must WRITE-LOCK the map
1641: * => we return TRUE if replacement was a success
1642: * => we expect the newents chain to have nnewents entries on it and
1643: * we expect newents->prev to point to the last entry on the list
1644: * => note newents is allowed to be NULL
1645: */
1646:
1647: int
1648: uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end,
1649: struct vm_map_entry *newents, int nnewents)
1650: {
1651: struct vm_map_entry *oldent, *last;
1652:
1653: uvm_tree_sanity(map, "map_replace entry");
1654:
1655: /*
1656: * first find the blank map entry at the specified address
1657: */
1658:
1659: if (!uvm_map_lookup_entry(map, start, &oldent)) {
1660: return(FALSE);
1661: }
1662:
1663: /*
1664: * check to make sure we have a proper blank entry
1665: */
1666:
1667: if (oldent->start != start || oldent->end != end ||
1668: oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
1669: return (FALSE);
1670: }
1671:
1672: #ifdef DIAGNOSTIC
1673: /*
1674: * sanity check the newents chain
1675: */
1676: {
1677: struct vm_map_entry *tmpent = newents;
1678: int nent = 0;
1679: vaddr_t cur = start;
1680:
1681: while (tmpent) {
1682: nent++;
1683: if (tmpent->start < cur)
1684: panic("uvm_map_replace1");
1685: if (tmpent->start > tmpent->end || tmpent->end > end) {
1686: printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n",
1687: tmpent->start, tmpent->end, end);
1688: panic("uvm_map_replace2");
1689: }
1690: cur = tmpent->end;
1691: if (tmpent->next) {
1692: if (tmpent->next->prev != tmpent)
1693: panic("uvm_map_replace3");
1694: } else {
1695: if (newents->prev != tmpent)
1696: panic("uvm_map_replace4");
1697: }
1698: tmpent = tmpent->next;
1699: }
1700: if (nent != nnewents)
1701: panic("uvm_map_replace5");
1702: }
1703: #endif
1704:
1705: /*
1706: * map entry is a valid blank! replace it. (this does all the
1707: * work of map entry link/unlink...).
1708: */
1709:
1710: if (newents) {
1711: last = newents->prev; /* we expect this */
1712:
1713: /* critical: flush stale hints out of map */
1714: SAVE_HINT(map, map->hint, newents);
1715: if (map->first_free == oldent)
1716: map->first_free = last;
1717:
1718: last->next = oldent->next;
1719: last->next->prev = last;
1720:
1721: /* Fix RB tree */
1722: uvm_rb_remove(map, oldent);
1723:
1724: newents->prev = oldent->prev;
1725: newents->prev->next = newents;
1726: map->nentries = map->nentries + (nnewents - 1);
1727:
1728: /* Fixup the RB tree */
1729: {
1730: int i;
1731: struct vm_map_entry *tmp;
1732:
1733: tmp = newents;
1734: for (i = 0; i < nnewents && tmp; i++) {
1735: uvm_rb_insert(map, tmp);
1736: tmp = tmp->next;
1737: }
1738: }
1739: } else {
1740:
1741: /* critical: flush stale hints out of map */
1742: SAVE_HINT(map, map->hint, oldent->prev);
1743: if (map->first_free == oldent)
1744: map->first_free = oldent->prev;
1745:
1746: /* NULL list of new entries: just remove the old one */
1747: uvm_map_entry_unlink(map, oldent);
1748: }
1749:
1750:
1751: uvm_tree_sanity(map, "map_replace leave");
1752:
1753: /*
1754: * now we can free the old blank entry, unlock the map and return.
1755: */
1756:
1757: uvm_mapent_free(oldent);
1758: return(TRUE);
1759: }
1760:
1761: /*
1762: * uvm_map_extract: extract a mapping from a map and put it somewhere
1763: * (maybe removing the old mapping)
1764: *
1765: * => maps should be unlocked (we will write lock them)
1766: * => returns 0 on success, error code otherwise
1767: * => start must be page aligned
1768: * => len must be page sized
1769: * => flags:
1770: * UVM_EXTRACT_REMOVE: remove mappings from srcmap
1771: * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
1772: * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
1773: * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
1774: * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
1775: * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
1776: * be used from within the kernel in a kernel level map <<<
1777: */
1778:
1779: int
1780: uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len,
1781: struct vm_map *dstmap, vaddr_t *dstaddrp, int flags)
1782: {
1783: vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge,
1784: oldstart;
1785: struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry;
1786: struct vm_map_entry *deadentry, *oldentry;
1787: vsize_t elen;
1788: int nchain, error, copy_ok;
1789: UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
1790:
1791: UVMHIST_LOG(maphist,"(srcmap=%p,start=0x%lx, len=0x%lx", srcmap, start,
1792: len,0);
1793: UVMHIST_LOG(maphist," ...,dstmap=%p, flags=0x%lx)", dstmap,flags,0,0);
1794:
1795: uvm_tree_sanity(srcmap, "map_extract src enter");
1796: uvm_tree_sanity(dstmap, "map_extract dst enter");
1797:
1798: /*
1799: * step 0: sanity check: start must be on a page boundary, length
1800: * must be page sized. can't ask for CONTIG/QREF if you asked for
1801: * REMOVE.
1802: */
1803:
1804: KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0);
1805: KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 ||
1806: (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0);
1807:
1808: /*
1809: * step 1: reserve space in the target map for the extracted area
1810: */
1811:
1812: dstaddr = vm_map_min(dstmap);
1813: if (uvm_map_reserve(dstmap, len, start, 0, &dstaddr) == FALSE)
1814: return(ENOMEM);
1815: *dstaddrp = dstaddr; /* pass address back to caller */
1816: UVMHIST_LOG(maphist, " dstaddr=0x%lx", dstaddr,0,0,0);
1817:
1818: /*
1819: * step 2: setup for the extraction process loop by init'ing the
1820: * map entry chain, locking src map, and looking up the first useful
1821: * entry in the map.
1822: */
1823:
1824: end = start + len;
1825: newend = dstaddr + len;
1826: chain = endchain = NULL;
1827: nchain = 0;
1828: vm_map_lock(srcmap);
1829:
1830: if (uvm_map_lookup_entry(srcmap, start, &entry)) {
1831:
1832: /* "start" is within an entry */
1833: if (flags & UVM_EXTRACT_QREF) {
1834:
1835: /*
1836: * for quick references we don't clip the entry, so
1837: * the entry may map space "before" the starting
1838: * virtual address... this is the "fudge" factor
1839: * (which can be non-zero only the first time
1840: * through the "while" loop in step 3).
1841: */
1842:
1843: fudge = start - entry->start;
1844: } else {
1845:
1846: /*
1847: * normal reference: we clip the map to fit (thus
1848: * fudge is zero)
1849: */
1850:
1851: UVM_MAP_CLIP_START(srcmap, entry, start);
1852: SAVE_HINT(srcmap, srcmap->hint, entry->prev);
1853: fudge = 0;
1854: }
1855: } else {
1856:
1857: /* "start" is not within an entry ... skip to next entry */
1858: if (flags & UVM_EXTRACT_CONTIG) {
1859: error = EINVAL;
1860: goto bad; /* definite hole here ... */
1861: }
1862:
1863: entry = entry->next;
1864: fudge = 0;
1865: }
1866:
1867: /* save values from srcmap for step 6 */
1868: orig_entry = entry;
1869: orig_fudge = fudge;
1870:
1871: /*
1872: * step 3: now start looping through the map entries, extracting
1873: * as we go.
1874: */
1875:
1876: while (entry->start < end && entry != &srcmap->header) {
1877:
1878: /* if we are not doing a quick reference, clip it */
1879: if ((flags & UVM_EXTRACT_QREF) == 0)
1880: UVM_MAP_CLIP_END(srcmap, entry, end);
1881:
1882: /* clear needs_copy (allow chunking) */
1883: if (UVM_ET_ISNEEDSCOPY(entry)) {
1884: if (fudge)
1885: oldstart = entry->start;
1886: else
1887: oldstart = 0; /* XXX: gcc */
1888: amap_copy(srcmap, entry, M_NOWAIT, TRUE, start, end);
1889: if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */
1890: error = ENOMEM;
1891: goto bad;
1892: }
1893:
1894: /* amap_copy could clip (during chunk)! update fudge */
1895: if (fudge) {
1896: fudge = fudge - (entry->start - oldstart);
1897: orig_fudge = fudge;
1898: }
1899: }
1900:
1901: /* calculate the offset of this from "start" */
1902: oldoffset = (entry->start + fudge) - start;
1903:
1904: /* allocate a new map entry */
1905: newentry = uvm_mapent_alloc(dstmap);
1906: if (newentry == NULL) {
1907: error = ENOMEM;
1908: goto bad;
1909: }
1910:
1911: /* set up new map entry */
1912: newentry->next = NULL;
1913: newentry->prev = endchain;
1914: newentry->start = dstaddr + oldoffset;
1915: newentry->end =
1916: newentry->start + (entry->end - (entry->start + fudge));
1917: if (newentry->end > newend || newentry->end < newentry->start)
1918: newentry->end = newend;
1919: newentry->object.uvm_obj = entry->object.uvm_obj;
1920: if (newentry->object.uvm_obj) {
1921: if (newentry->object.uvm_obj->pgops->pgo_reference)
1922: newentry->object.uvm_obj->pgops->
1923: pgo_reference(newentry->object.uvm_obj);
1924: newentry->offset = entry->offset + fudge;
1925: } else {
1926: newentry->offset = 0;
1927: }
1928: newentry->etype = entry->etype;
1929: newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
1930: entry->max_protection : entry->protection;
1931: newentry->max_protection = entry->max_protection;
1932: newentry->inheritance = entry->inheritance;
1933: newentry->wired_count = 0;
1934: newentry->aref.ar_amap = entry->aref.ar_amap;
1935: if (newentry->aref.ar_amap) {
1936: newentry->aref.ar_pageoff =
1937: entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
1938: uvm_map_reference_amap(newentry, AMAP_SHARED |
1939: ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
1940: } else {
1941: newentry->aref.ar_pageoff = 0;
1942: }
1943: newentry->advice = entry->advice;
1944:
1945: /* now link it on the chain */
1946: nchain++;
1947: if (endchain == NULL) {
1948: chain = endchain = newentry;
1949: } else {
1950: endchain->next = newentry;
1951: endchain = newentry;
1952: }
1953:
1954: /* end of 'while' loop! */
1955: if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
1956: (entry->next == &srcmap->header ||
1957: entry->next->start != entry->end)) {
1958: error = EINVAL;
1959: goto bad;
1960: }
1961: entry = entry->next;
1962: fudge = 0;
1963: }
1964:
1965: /*
1966: * step 4: close off chain (in format expected by uvm_map_replace)
1967: */
1968:
1969: if (chain)
1970: chain->prev = endchain;
1971:
1972: /*
1973: * step 5: attempt to lock the dest map so we can pmap_copy.
1974: * note usage of copy_ok:
1975: * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
1976: * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
1977: */
1978:
1979: if (srcmap == dstmap || vm_map_lock_try(dstmap) == TRUE) {
1980: copy_ok = 1;
1981: if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
1982: nchain)) {
1983: if (srcmap != dstmap)
1984: vm_map_unlock(dstmap);
1985: error = EIO;
1986: goto bad;
1987: }
1988: } else {
1989: copy_ok = 0;
1990: /* replace defered until step 7 */
1991: }
1992:
1993: /*
1994: * step 6: traverse the srcmap a second time to do the following:
1995: * - if we got a lock on the dstmap do pmap_copy
1996: * - if UVM_EXTRACT_REMOVE remove the entries
1997: * we make use of orig_entry and orig_fudge (saved in step 2)
1998: */
1999:
2000: if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
2001:
2002: /* purge possible stale hints from srcmap */
2003: if (flags & UVM_EXTRACT_REMOVE) {
2004: SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev);
2005: if (srcmap->first_free->start >= start)
2006: srcmap->first_free = orig_entry->prev;
2007: }
2008:
2009: entry = orig_entry;
2010: fudge = orig_fudge;
2011: deadentry = NULL; /* for UVM_EXTRACT_REMOVE */
2012:
2013: while (entry->start < end && entry != &srcmap->header) {
2014: if (copy_ok) {
2015: oldoffset = (entry->start + fudge) - start;
2016: elen = MIN(end, entry->end) -
2017: (entry->start + fudge);
2018: pmap_copy(dstmap->pmap, srcmap->pmap,
2019: dstaddr + oldoffset, elen,
2020: entry->start + fudge);
2021: }
2022:
2023: /* we advance "entry" in the following if statement */
2024: if (flags & UVM_EXTRACT_REMOVE) {
2025: pmap_remove(srcmap->pmap, entry->start,
2026: entry->end);
2027: oldentry = entry; /* save entry */
2028: entry = entry->next; /* advance */
2029: uvm_map_entry_unlink(srcmap, oldentry);
2030: /* add to dead list */
2031: oldentry->next = deadentry;
2032: deadentry = oldentry;
2033: } else {
2034: entry = entry->next; /* advance */
2035: }
2036:
2037: /* end of 'while' loop */
2038: fudge = 0;
2039: }
2040: pmap_update(srcmap->pmap);
2041:
2042: /*
2043: * unlock dstmap. we will dispose of deadentry in
2044: * step 7 if needed
2045: */
2046:
2047: if (copy_ok && srcmap != dstmap)
2048: vm_map_unlock(dstmap);
2049:
2050: }
2051: else
2052: deadentry = NULL; /* XXX: gcc */
2053:
2054: /*
2055: * step 7: we are done with the source map, unlock. if copy_ok
2056: * is 0 then we have not replaced the dummy mapping in dstmap yet
2057: * and we need to do so now.
2058: */
2059:
2060: vm_map_unlock(srcmap);
2061: if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
2062: uvm_unmap_detach(deadentry, 0); /* dispose of old entries */
2063:
2064: /* now do the replacement if we didn't do it in step 5 */
2065: if (copy_ok == 0) {
2066: vm_map_lock(dstmap);
2067: error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2068: nchain);
2069: vm_map_unlock(dstmap);
2070:
2071: if (error == FALSE) {
2072: error = EIO;
2073: goto bad2;
2074: }
2075: }
2076:
2077: uvm_tree_sanity(srcmap, "map_extract src leave");
2078: uvm_tree_sanity(dstmap, "map_extract dst leave");
2079:
2080: return(0);
2081:
2082: /*
2083: * bad: failure recovery
2084: */
2085: bad:
2086: vm_map_unlock(srcmap);
2087: bad2: /* src already unlocked */
2088: if (chain)
2089: uvm_unmap_detach(chain,
2090: (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
2091:
2092: uvm_tree_sanity(srcmap, "map_extract src err leave");
2093: uvm_tree_sanity(dstmap, "map_extract dst err leave");
2094:
2095: uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */
2096: return(error);
2097: }
2098:
2099: /* end of extraction functions */
2100:
2101: /*
2102: * uvm_map_submap: punch down part of a map into a submap
2103: *
2104: * => only the kernel_map is allowed to be submapped
2105: * => the purpose of submapping is to break up the locking granularity
2106: * of a larger map
2107: * => the range specified must have been mapped previously with a uvm_map()
2108: * call [with uobj==NULL] to create a blank map entry in the main map.
2109: * [And it had better still be blank!]
2110: * => maps which contain submaps should never be copied or forked.
2111: * => to remove a submap, use uvm_unmap() on the main map
2112: * and then uvm_map_deallocate() the submap.
2113: * => main map must be unlocked.
2114: * => submap must have been init'd and have a zero reference count.
2115: * [need not be locked as we don't actually reference it]
2116: */
2117:
2118: int
2119: uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end,
2120: struct vm_map *submap)
2121: {
2122: struct vm_map_entry *entry;
2123: int result;
2124:
2125: vm_map_lock(map);
2126:
2127: VM_MAP_RANGE_CHECK(map, start, end);
2128:
2129: if (uvm_map_lookup_entry(map, start, &entry)) {
2130: UVM_MAP_CLIP_START(map, entry, start);
2131: UVM_MAP_CLIP_END(map, entry, end); /* to be safe */
2132: } else {
2133: entry = NULL;
2134: }
2135:
2136: if (entry != NULL &&
2137: entry->start == start && entry->end == end &&
2138: entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
2139: !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
2140: entry->etype |= UVM_ET_SUBMAP;
2141: entry->object.sub_map = submap;
2142: entry->offset = 0;
2143: uvm_map_reference(submap);
2144: result = 0;
2145: } else {
2146: result = EINVAL;
2147: }
2148: vm_map_unlock(map);
2149: return(result);
2150: }
2151:
2152:
2153: /*
2154: * uvm_map_protect: change map protection
2155: *
2156: * => set_max means set max_protection.
2157: * => map must be unlocked.
2158: */
2159:
2160: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
2161: ~VM_PROT_WRITE : VM_PROT_ALL)
2162: #define max(a,b) ((a) > (b) ? (a) : (b))
2163:
2164: int
2165: uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
2166: vm_prot_t new_prot, boolean_t set_max)
2167: {
2168: struct vm_map_entry *current, *entry;
2169: int error = 0;
2170: UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
2171: UVMHIST_LOG(maphist,"(map=%p,start=0x%lx,end=0x%lx,new_prot=0x%lx)",
2172: map, start, end, new_prot);
2173:
2174: vm_map_lock(map);
2175:
2176: VM_MAP_RANGE_CHECK(map, start, end);
2177:
2178: if (uvm_map_lookup_entry(map, start, &entry)) {
2179: UVM_MAP_CLIP_START(map, entry, start);
2180: } else {
2181: entry = entry->next;
2182: }
2183:
2184: /*
2185: * make a first pass to check for protection violations.
2186: */
2187:
2188: current = entry;
2189: while ((current != &map->header) && (current->start < end)) {
2190: if (UVM_ET_ISSUBMAP(current)) {
2191: error = EINVAL;
2192: goto out;
2193: }
2194: if ((new_prot & current->max_protection) != new_prot) {
2195: error = EACCES;
2196: goto out;
2197: }
2198: current = current->next;
2199: }
2200:
2201: /* go back and fix up protections (no need to clip this time). */
2202:
2203: current = entry;
2204:
2205: while ((current != &map->header) && (current->start < end)) {
2206: vm_prot_t old_prot;
2207:
2208: UVM_MAP_CLIP_END(map, current, end);
2209:
2210: old_prot = current->protection;
2211: if (set_max)
2212: current->protection =
2213: (current->max_protection = new_prot) & old_prot;
2214: else
2215: current->protection = new_prot;
2216:
2217: /*
2218: * update physical map if necessary. worry about copy-on-write
2219: * here -- CHECK THIS XXX
2220: */
2221:
2222: if (current->protection != old_prot) {
2223: /* update pmap! */
2224: if ((current->protection & MASK(entry)) == PROT_NONE &&
2225: VM_MAPENT_ISWIRED(entry))
2226: current->wired_count--;
2227: pmap_protect(map->pmap, current->start, current->end,
2228: current->protection & MASK(entry));
2229: }
2230:
2231: /*
2232: * If the map is configured to lock any future mappings,
2233: * wire this entry now if the old protection was VM_PROT_NONE
2234: * and the new protection is not VM_PROT_NONE.
2235: */
2236:
2237: if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
2238: VM_MAPENT_ISWIRED(entry) == 0 &&
2239: old_prot == VM_PROT_NONE &&
2240: new_prot != VM_PROT_NONE) {
2241: if (uvm_map_pageable(map, entry->start, entry->end,
2242: FALSE, UVM_LK_ENTER|UVM_LK_EXIT) != 0) {
2243: /*
2244: * If locking the entry fails, remember the
2245: * error if it's the first one. Note we
2246: * still continue setting the protection in
2247: * the map, but will return the resource
2248: * shortage condition regardless.
2249: *
2250: * XXX Ignore what the actual error is,
2251: * XXX just call it a resource shortage
2252: * XXX so that it doesn't get confused
2253: * XXX what uvm_map_protect() itself would
2254: * XXX normally return.
2255: */
2256: error = ENOMEM;
2257: }
2258: }
2259:
2260: current = current->next;
2261: }
2262: pmap_update(map->pmap);
2263:
2264: out:
2265: vm_map_unlock(map);
2266: UVMHIST_LOG(maphist, "<- done, rv=%ld",error,0,0,0);
2267: return (error);
2268: }
2269:
2270: #undef max
2271: #undef MASK
2272:
2273: /*
2274: * uvm_map_inherit: set inheritance code for range of addrs in map.
2275: *
2276: * => map must be unlocked
2277: * => note that the inherit code is used during a "fork". see fork
2278: * code for details.
2279: */
2280:
2281: int
2282: uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end,
2283: vm_inherit_t new_inheritance)
2284: {
2285: struct vm_map_entry *entry, *temp_entry;
2286: UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
2287: UVMHIST_LOG(maphist,"(map=%p,start=0x%lx,end=0x%lx,new_inh=0x%lx)",
2288: map, start, end, new_inheritance);
2289:
2290: switch (new_inheritance) {
2291: case MAP_INHERIT_NONE:
2292: case MAP_INHERIT_COPY:
2293: case MAP_INHERIT_SHARE:
2294: break;
2295: default:
2296: UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
2297: return (EINVAL);
2298: }
2299:
2300: vm_map_lock(map);
2301:
2302: VM_MAP_RANGE_CHECK(map, start, end);
2303:
2304: if (uvm_map_lookup_entry(map, start, &temp_entry)) {
2305: entry = temp_entry;
2306: UVM_MAP_CLIP_START(map, entry, start);
2307: } else {
2308: entry = temp_entry->next;
2309: }
2310:
2311: while ((entry != &map->header) && (entry->start < end)) {
2312: UVM_MAP_CLIP_END(map, entry, end);
2313: entry->inheritance = new_inheritance;
2314: entry = entry->next;
2315: }
2316:
2317: vm_map_unlock(map);
2318: UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
2319: return (0);
2320: }
2321:
2322: /*
2323: * uvm_map_advice: set advice code for range of addrs in map.
2324: *
2325: * => map must be unlocked
2326: */
2327:
2328: int
2329: uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice)
2330: {
2331: struct vm_map_entry *entry, *temp_entry;
2332: UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
2333: UVMHIST_LOG(maphist,"(map=%p,start=0x%lx,end=0x%lx,new_adv=0x%lx)",
2334: map, start, end, new_advice);
2335:
2336: vm_map_lock(map);
2337: VM_MAP_RANGE_CHECK(map, start, end);
2338: if (uvm_map_lookup_entry(map, start, &temp_entry)) {
2339: entry = temp_entry;
2340: UVM_MAP_CLIP_START(map, entry, start);
2341: } else {
2342: entry = temp_entry->next;
2343: }
2344:
2345: /*
2346: * XXXJRT: disallow holes?
2347: */
2348:
2349: while ((entry != &map->header) && (entry->start < end)) {
2350: UVM_MAP_CLIP_END(map, entry, end);
2351:
2352: switch (new_advice) {
2353: case MADV_NORMAL:
2354: case MADV_RANDOM:
2355: case MADV_SEQUENTIAL:
2356: /* nothing special here */
2357: break;
2358:
2359: default:
2360: vm_map_unlock(map);
2361: UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
2362: return (EINVAL);
2363: }
2364: entry->advice = new_advice;
2365: entry = entry->next;
2366: }
2367:
2368: vm_map_unlock(map);
2369: UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
2370: return (0);
2371: }
2372:
2373: /*
2374: * uvm_map_pageable: sets the pageability of a range in a map.
2375: *
2376: * => wires map entries. should not be used for transient page locking.
2377: * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
2378: * => regions sepcified as not pageable require lock-down (wired) memory
2379: * and page tables.
2380: * => map must never be read-locked
2381: * => if islocked is TRUE, map is already write-locked
2382: * => we always unlock the map, since we must downgrade to a read-lock
2383: * to call uvm_fault_wire()
2384: * => XXXCDC: check this and try and clean it up.
2385: */
2386:
2387: int
2388: uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end,
2389: boolean_t new_pageable, int lockflags)
2390: {
2391: struct vm_map_entry *entry, *start_entry, *failed_entry;
2392: int rv;
2393: #ifdef DIAGNOSTIC
2394: u_int timestamp_save;
2395: #endif
2396: UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
2397: UVMHIST_LOG(maphist,"(map=%p,start=0x%lx,end=0x%lx,new_pageable=0x%lx)",
2398: map, start, end, new_pageable);
2399: KASSERT(map->flags & VM_MAP_PAGEABLE);
2400:
2401: if ((lockflags & UVM_LK_ENTER) == 0)
2402: vm_map_lock(map);
2403:
2404: VM_MAP_RANGE_CHECK(map, start, end);
2405:
2406: /*
2407: * only one pageability change may take place at one time, since
2408: * uvm_fault_wire assumes it will be called only once for each
2409: * wiring/unwiring. therefore, we have to make sure we're actually
2410: * changing the pageability for the entire region. we do so before
2411: * making any changes.
2412: */
2413:
2414: if (uvm_map_lookup_entry(map, start, &start_entry) == FALSE) {
2415: if ((lockflags & UVM_LK_EXIT) == 0)
2416: vm_map_unlock(map);
2417:
2418: UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
2419: return (EFAULT);
2420: }
2421: entry = start_entry;
2422:
2423: /*
2424: * handle wiring and unwiring separately.
2425: */
2426:
2427: if (new_pageable) { /* unwire */
2428: UVM_MAP_CLIP_START(map, entry, start);
2429:
2430: /*
2431: * unwiring. first ensure that the range to be unwired is
2432: * really wired down and that there are no holes.
2433: */
2434:
2435: while ((entry != &map->header) && (entry->start < end)) {
2436: if (entry->wired_count == 0 ||
2437: (entry->end < end &&
2438: (entry->next == &map->header ||
2439: entry->next->start > entry->end))) {
2440: if ((lockflags & UVM_LK_EXIT) == 0)
2441: vm_map_unlock(map);
2442: UVMHIST_LOG(maphist,
2443: "<- done (INVALID UNWIRE ARG)",0,0,0,0);
2444: return (EINVAL);
2445: }
2446: entry = entry->next;
2447: }
2448:
2449: /*
2450: * POSIX 1003.1b - a single munlock call unlocks a region,
2451: * regardless of the number of mlock calls made on that
2452: * region.
2453: */
2454:
2455: entry = start_entry;
2456: while ((entry != &map->header) && (entry->start < end)) {
2457: UVM_MAP_CLIP_END(map, entry, end);
2458: if (VM_MAPENT_ISWIRED(entry))
2459: uvm_map_entry_unwire(map, entry);
2460: entry = entry->next;
2461: }
2462: if ((lockflags & UVM_LK_EXIT) == 0)
2463: vm_map_unlock(map);
2464: UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
2465: return (0);
2466: }
2467:
2468: /*
2469: * wire case: in two passes [XXXCDC: ugly block of code here]
2470: *
2471: * 1: holding the write lock, we create any anonymous maps that need
2472: * to be created. then we clip each map entry to the region to
2473: * be wired and increment its wiring count.
2474: *
2475: * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
2476: * in the pages for any newly wired area (wired_count == 1).
2477: *
2478: * downgrading to a read lock for uvm_fault_wire avoids a possible
2479: * deadlock with another thread that may have faulted on one of
2480: * the pages to be wired (it would mark the page busy, blocking
2481: * us, then in turn block on the map lock that we hold). because
2482: * of problems in the recursive lock package, we cannot upgrade
2483: * to a write lock in vm_map_lookup. thus, any actions that
2484: * require the write lock must be done beforehand. because we
2485: * keep the read lock on the map, the copy-on-write status of the
2486: * entries we modify here cannot change.
2487: */
2488:
2489: while ((entry != &map->header) && (entry->start < end)) {
2490: if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
2491:
2492: /*
2493: * perform actions of vm_map_lookup that need the
2494: * write lock on the map: create an anonymous map
2495: * for a copy-on-write region, or an anonymous map
2496: * for a zero-fill region. (XXXCDC: submap case
2497: * ok?)
2498: */
2499:
2500: if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
2501: if (UVM_ET_ISNEEDSCOPY(entry) &&
2502: ((entry->protection & VM_PROT_WRITE) ||
2503: (entry->object.uvm_obj == NULL))) {
2504: amap_copy(map, entry, M_WAITOK, TRUE,
2505: start, end);
2506: /* XXXCDC: wait OK? */
2507: }
2508: }
2509: }
2510: UVM_MAP_CLIP_START(map, entry, start);
2511: UVM_MAP_CLIP_END(map, entry, end);
2512: entry->wired_count++;
2513:
2514: /*
2515: * Check for holes
2516: */
2517:
2518: if (entry->protection == VM_PROT_NONE ||
2519: (entry->end < end &&
2520: (entry->next == &map->header ||
2521: entry->next->start > entry->end))) {
2522:
2523: /*
2524: * found one. amap creation actions do not need to
2525: * be undone, but the wired counts need to be restored.
2526: */
2527:
2528: while (entry != &map->header && entry->end > start) {
2529: entry->wired_count--;
2530: entry = entry->prev;
2531: }
2532: if ((lockflags & UVM_LK_EXIT) == 0)
2533: vm_map_unlock(map);
2534: UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
2535: return (EINVAL);
2536: }
2537: entry = entry->next;
2538: }
2539:
2540: /*
2541: * Pass 2.
2542: */
2543:
2544: #ifdef DIAGNOSTIC
2545: timestamp_save = map->timestamp;
2546: #endif
2547: vm_map_busy(map);
2548: vm_map_downgrade(map);
2549:
2550: rv = 0;
2551: entry = start_entry;
2552: while (entry != &map->header && entry->start < end) {
2553: if (entry->wired_count == 1) {
2554: rv = uvm_fault_wire(map, entry->start, entry->end,
2555: entry->protection);
2556: if (rv) {
2557: /*
2558: * wiring failed. break out of the loop.
2559: * we'll clean up the map below, once we
2560: * have a write lock again.
2561: */
2562: break;
2563: }
2564: }
2565: entry = entry->next;
2566: }
2567:
2568: if (rv) { /* failed? */
2569:
2570: /*
2571: * Get back to an exclusive (write) lock.
2572: */
2573:
2574: vm_map_upgrade(map);
2575: vm_map_unbusy(map);
2576:
2577: #ifdef DIAGNOSTIC
2578: if (timestamp_save != map->timestamp)
2579: panic("uvm_map_pageable: stale map");
2580: #endif
2581:
2582: /*
2583: * first drop the wiring count on all the entries
2584: * which haven't actually been wired yet.
2585: */
2586:
2587: failed_entry = entry;
2588: while (entry != &map->header && entry->start < end) {
2589: entry->wired_count--;
2590: entry = entry->next;
2591: }
2592:
2593: /*
2594: * now, unwire all the entries that were successfully
2595: * wired above.
2596: */
2597:
2598: entry = start_entry;
2599: while (entry != failed_entry) {
2600: entry->wired_count--;
2601: if (VM_MAPENT_ISWIRED(entry) == 0)
2602: uvm_map_entry_unwire(map, entry);
2603: entry = entry->next;
2604: }
2605: if ((lockflags & UVM_LK_EXIT) == 0)
2606: vm_map_unlock(map);
2607: UVMHIST_LOG(maphist, "<- done (RV=%ld)", rv,0,0,0);
2608: return(rv);
2609: }
2610:
2611: /* We are holding a read lock here. */
2612: if ((lockflags & UVM_LK_EXIT) == 0) {
2613: vm_map_unbusy(map);
2614: vm_map_unlock_read(map);
2615: } else {
2616:
2617: /*
2618: * Get back to an exclusive (write) lock.
2619: */
2620:
2621: vm_map_upgrade(map);
2622: vm_map_unbusy(map);
2623: }
2624:
2625: UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
2626: return (0);
2627: }
2628:
2629: /*
2630: * uvm_map_pageable_all: special case of uvm_map_pageable - affects
2631: * all mapped regions.
2632: *
2633: * => map must not be locked.
2634: * => if no flags are specified, all regions are unwired.
2635: * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
2636: */
2637:
2638: int
2639: uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit)
2640: {
2641: struct vm_map_entry *entry, *failed_entry;
2642: vsize_t size;
2643: int error;
2644: #ifdef DIAGNOSTIC
2645: u_int timestamp_save;
2646: #endif
2647: UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
2648: UVMHIST_LOG(maphist,"(map=%p,flags=0x%lx)", map, flags, 0, 0);
2649:
2650: KASSERT(map->flags & VM_MAP_PAGEABLE);
2651:
2652: vm_map_lock(map);
2653:
2654: /*
2655: * handle wiring and unwiring separately.
2656: */
2657:
2658: if (flags == 0) { /* unwire */
2659: /*
2660: * POSIX 1003.1b -- munlockall unlocks all regions,
2661: * regardless of how many times mlockall has been called.
2662: */
2663: for (entry = map->header.next; entry != &map->header;
2664: entry = entry->next) {
2665: if (VM_MAPENT_ISWIRED(entry))
2666: uvm_map_entry_unwire(map, entry);
2667: }
2668: vm_map_modflags(map, 0, VM_MAP_WIREFUTURE);
2669: vm_map_unlock(map);
2670: UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
2671: return (0);
2672:
2673: /*
2674: * end of unwire case!
2675: */
2676: }
2677:
2678: if (flags & MCL_FUTURE) {
2679: /*
2680: * must wire all future mappings; remember this.
2681: */
2682: vm_map_modflags(map, VM_MAP_WIREFUTURE, 0);
2683: }
2684:
2685: if ((flags & MCL_CURRENT) == 0) {
2686: /*
2687: * no more work to do!
2688: */
2689: UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
2690: vm_map_unlock(map);
2691: return (0);
2692: }
2693:
2694: /*
2695: * wire case: in three passes [XXXCDC: ugly block of code here]
2696: *
2697: * 1: holding the write lock, count all pages mapped by non-wired
2698: * entries. if this would cause us to go over our limit, we fail.
2699: *
2700: * 2: still holding the write lock, we create any anonymous maps that
2701: * need to be created. then we increment its wiring count.
2702: *
2703: * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
2704: * in the pages for any newly wired area (wired_count == 1).
2705: *
2706: * downgrading to a read lock for uvm_fault_wire avoids a possible
2707: * deadlock with another thread that may have faulted on one of
2708: * the pages to be wired (it would mark the page busy, blocking
2709: * us, then in turn block on the map lock that we hold). because
2710: * of problems in the recursive lock package, we cannot upgrade
2711: * to a write lock in vm_map_lookup. thus, any actions that
2712: * require the write lock must be done beforehand. because we
2713: * keep the read lock on the map, the copy-on-write status of the
2714: * entries we modify here cannot change.
2715: */
2716:
2717: for (size = 0, entry = map->header.next; entry != &map->header;
2718: entry = entry->next) {
2719: if (entry->protection != VM_PROT_NONE &&
2720: VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
2721: size += entry->end - entry->start;
2722: }
2723: }
2724:
2725: if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
2726: vm_map_unlock(map);
2727: return (ENOMEM); /* XXX overloaded */
2728: }
2729:
2730: /* XXX non-pmap_wired_count case must be handled by caller */
2731: #ifdef pmap_wired_count
2732: if (limit != 0 &&
2733: (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
2734: vm_map_unlock(map);
2735: return (ENOMEM); /* XXX overloaded */
2736: }
2737: #endif
2738:
2739: /*
2740: * Pass 2.
2741: */
2742:
2743: for (entry = map->header.next; entry != &map->header;
2744: entry = entry->next) {
2745: if (entry->protection == VM_PROT_NONE)
2746: continue;
2747: if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
2748: /*
2749: * perform actions of vm_map_lookup that need the
2750: * write lock on the map: create an anonymous map
2751: * for a copy-on-write region, or an anonymous map
2752: * for a zero-fill region. (XXXCDC: submap case
2753: * ok?)
2754: */
2755: if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
2756: if (UVM_ET_ISNEEDSCOPY(entry) &&
2757: ((entry->protection & VM_PROT_WRITE) ||
2758: (entry->object.uvm_obj == NULL))) {
2759: amap_copy(map, entry, M_WAITOK, TRUE,
2760: entry->start, entry->end);
2761: /* XXXCDC: wait OK? */
2762: }
2763: }
2764: }
2765: entry->wired_count++;
2766: }
2767:
2768: /*
2769: * Pass 3.
2770: */
2771:
2772: #ifdef DIAGNOSTIC
2773: timestamp_save = map->timestamp;
2774: #endif
2775: vm_map_busy(map);
2776: vm_map_downgrade(map);
2777:
2778: for (error = 0, entry = map->header.next;
2779: entry != &map->header && error == 0;
2780: entry = entry->next) {
2781: if (entry->wired_count == 1) {
2782: error = uvm_fault_wire(map, entry->start, entry->end,
2783: entry->protection);
2784: }
2785: }
2786:
2787: if (error) { /* failed? */
2788: /*
2789: * Get back an exclusive (write) lock.
2790: */
2791: vm_map_upgrade(map);
2792: vm_map_unbusy(map);
2793:
2794: #ifdef DIAGNOSTIC
2795: if (timestamp_save != map->timestamp)
2796: panic("uvm_map_pageable_all: stale map");
2797: #endif
2798:
2799: /*
2800: * first drop the wiring count on all the entries
2801: * which haven't actually been wired yet.
2802: *
2803: * Skip VM_PROT_NONE entries like we did above.
2804: */
2805: failed_entry = entry;
2806: for (/* nothing */; entry != &map->header;
2807: entry = entry->next) {
2808: if (entry->protection == VM_PROT_NONE)
2809: continue;
2810: entry->wired_count--;
2811: }
2812:
2813: /*
2814: * now, unwire all the entries that were successfully
2815: * wired above.
2816: *
2817: * Skip VM_PROT_NONE entries like we did above.
2818: */
2819: for (entry = map->header.next; entry != failed_entry;
2820: entry = entry->next) {
2821: if (entry->protection == VM_PROT_NONE)
2822: continue;
2823: entry->wired_count--;
2824: if (VM_MAPENT_ISWIRED(entry))
2825: uvm_map_entry_unwire(map, entry);
2826: }
2827: vm_map_unlock(map);
2828: UVMHIST_LOG(maphist,"<- done (RV=%ld)", error,0,0,0);
2829: return (error);
2830: }
2831:
2832: /* We are holding a read lock here. */
2833: vm_map_unbusy(map);
2834: vm_map_unlock_read(map);
2835:
2836: UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
2837: return (0);
2838: }
2839:
2840: /*
2841: * uvm_map_clean: clean out a map range
2842: *
2843: * => valid flags:
2844: * if (flags & PGO_CLEANIT): dirty pages are cleaned first
2845: * if (flags & PGO_SYNCIO): dirty pages are written synchronously
2846: * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
2847: * if (flags & PGO_FREE): any cached pages are freed after clean
2848: * => returns an error if any part of the specified range isn't mapped
2849: * => never a need to flush amap layer since the anonymous memory has
2850: * no permanent home, but may deactivate pages there
2851: * => called from sys_msync() and sys_madvise()
2852: * => caller must not write-lock map (read OK).
2853: * => we may sleep while cleaning if SYNCIO [with map read-locked]
2854: */
2855:
2856: int amap_clean_works = 1; /* XXX for now, just in case... */
2857:
2858: int
2859: uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
2860: {
2861: struct vm_map_entry *current, *entry;
2862: struct uvm_object *uobj;
2863: struct vm_amap *amap;
2864: struct vm_anon *anon;
2865: struct vm_page *pg;
2866: vaddr_t offset;
2867: vsize_t size;
2868: int rv, error, refs;
2869: UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
2870:
2871: UVMHIST_LOG(maphist,"(map=%p,start=0x%lx,end=0x%lx,flags=0x%lx)",
2872: map, start, end, flags);
2873: KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) !=
2874: (PGO_FREE|PGO_DEACTIVATE));
2875:
2876: vm_map_lock_read(map);
2877: VM_MAP_RANGE_CHECK(map, start, end);
2878: if (uvm_map_lookup_entry(map, start, &entry) == FALSE) {
2879: vm_map_unlock_read(map);
2880: return (EFAULT);
2881: }
2882:
2883: /*
2884: * Make a first pass to check for holes.
2885: */
2886:
2887: for (current = entry; current->start < end; current = current->next) {
2888: if (UVM_ET_ISSUBMAP(current)) {
2889: vm_map_unlock_read(map);
2890: return (EINVAL);
2891: }
2892: if (end > current->end && (current->next == &map->header ||
2893: current->end != current->next->start)) {
2894: vm_map_unlock_read(map);
2895: return (EFAULT);
2896: }
2897: }
2898:
2899: error = 0;
2900:
2901: for (current = entry; current->start < end; current = current->next) {
2902: amap = current->aref.ar_amap; /* top layer */
2903: uobj = current->object.uvm_obj; /* bottom layer */
2904: KASSERT(start >= current->start);
2905:
2906: /*
2907: * No amap cleaning necessary if:
2908: *
2909: * (1) There's no amap.
2910: *
2911: * (2) We're not deactivating or freeing pages.
2912: */
2913:
2914: if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
2915: goto flush_object;
2916:
2917: /* XXX for now, just in case... */
2918: if (amap_clean_works == 0)
2919: goto flush_object;
2920:
2921: offset = start - current->start;
2922: size = MIN(end, current->end) - start;
2923: for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
2924: anon = amap_lookup(¤t->aref, offset);
2925: if (anon == NULL)
2926: continue;
2927:
2928: simple_lock(&anon->an_lock);
2929:
2930: pg = anon->an_page;
2931: if (pg == NULL) {
2932: simple_unlock(&anon->an_lock);
2933: continue;
2934: }
2935:
2936: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
2937:
2938: /*
2939: * XXX In these first 3 cases, we always just
2940: * XXX deactivate the page. We may want to
2941: * XXX handle the different cases more
2942: * XXX specifically, in the future.
2943: */
2944:
2945: case PGO_CLEANIT|PGO_FREE:
2946: case PGO_CLEANIT|PGO_DEACTIVATE:
2947: case PGO_DEACTIVATE:
2948: deactivate_it:
2949: /* skip the page if it's loaned or wired */
2950: if (pg->loan_count != 0 ||
2951: pg->wire_count != 0) {
2952: simple_unlock(&anon->an_lock);
2953: continue;
2954: }
2955:
2956: uvm_lock_pageq();
2957:
2958: /*
2959: * skip the page if it's not actually owned
2960: * by the anon (may simply be loaned to the
2961: * anon).
2962: */
2963:
2964: if ((pg->pg_flags & PQ_ANON) == 0) {
2965: KASSERT(pg->uobject == NULL);
2966: uvm_unlock_pageq();
2967: simple_unlock(&anon->an_lock);
2968: continue;
2969: }
2970: KASSERT(pg->uanon == anon);
2971:
2972: #ifdef UBC
2973: /* ...and deactivate the page. */
2974: pmap_clear_reference(pg);
2975: #else
2976: /* zap all mappings for the page. */
2977: pmap_page_protect(pg, VM_PROT_NONE);
2978:
2979: /* ...and deactivate the page. */
2980: #endif
2981: uvm_pagedeactivate(pg);
2982:
2983: uvm_unlock_pageq();
2984: simple_unlock(&anon->an_lock);
2985: continue;
2986:
2987: case PGO_FREE:
2988:
2989: /*
2990: * If there are multiple references to
2991: * the amap, just deactivate the page.
2992: */
2993:
2994: if (amap_refs(amap) > 1)
2995: goto deactivate_it;
2996:
2997: /* XXX skip the page if it's wired */
2998: if (pg->wire_count != 0) {
2999: simple_unlock(&anon->an_lock);
3000: continue;
3001: }
3002: amap_unadd(¤t->aref, offset);
3003: refs = --anon->an_ref;
3004: simple_unlock(&anon->an_lock);
3005: if (refs == 0)
3006: uvm_anfree(anon);
3007: continue;
3008:
3009: default:
3010: panic("uvm_map_clean: weird flags");
3011: }
3012: }
3013:
3014: flush_object:
3015: /*
3016: * flush pages if we've got a valid backing object.
3017: *
3018: * Don't PGO_FREE if we don't have write permission
3019: * and don't flush if this is a copy-on-write object
3020: * since we can't know our permissions on it.
3021: */
3022:
3023: offset = current->offset + (start - current->start);
3024: size = MIN(end, current->end) - start;
3025: if (uobj != NULL &&
3026: ((flags & PGO_FREE) == 0 ||
3027: ((entry->max_protection & VM_PROT_WRITE) != 0 &&
3028: (entry->etype & UVM_ET_COPYONWRITE) == 0))) {
3029: simple_lock(&uobj->vmobjlock);
3030: rv = uobj->pgops->pgo_flush(uobj, offset,
3031: offset + size, flags);
3032: simple_unlock(&uobj->vmobjlock);
3033:
3034: if (rv == FALSE)
3035: error = EFAULT;
3036: }
3037: start += size;
3038: }
3039: vm_map_unlock_read(map);
3040: return (error);
3041: }
3042:
3043:
3044: /*
3045: * uvm_map_checkprot: check protection in map
3046: *
3047: * => must allow specified protection in a fully allocated region.
3048: * => map must be read or write locked by caller.
3049: */
3050:
3051: boolean_t
3052: uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end,
3053: vm_prot_t protection)
3054: {
3055: struct vm_map_entry *entry;
3056: struct vm_map_entry *tmp_entry;
3057:
3058: if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
3059: return(FALSE);
3060: }
3061: entry = tmp_entry;
3062: while (start < end) {
3063: if (entry == &map->header) {
3064: return(FALSE);
3065: }
3066:
3067: /*
3068: * no holes allowed
3069: */
3070:
3071: if (start < entry->start) {
3072: return(FALSE);
3073: }
3074:
3075: /*
3076: * check protection associated with entry
3077: */
3078:
3079: if ((entry->protection & protection) != protection) {
3080: return(FALSE);
3081: }
3082:
3083: /* go to next entry */
3084:
3085: start = entry->end;
3086: entry = entry->next;
3087: }
3088: return(TRUE);
3089: }
3090:
3091: /*
3092: * uvmspace_alloc: allocate a vmspace structure.
3093: *
3094: * - structure includes vm_map and pmap
3095: * - XXX: no locking on this structure
3096: * - refcnt set to 1, rest must be init'd by caller
3097: */
3098: struct vmspace *
3099: uvmspace_alloc(vaddr_t min, vaddr_t max, int pageable)
3100: {
3101: struct vmspace *vm;
3102: UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
3103:
3104: vm = pool_get(&uvm_vmspace_pool, PR_WAITOK);
3105: uvmspace_init(vm, NULL, min, max, pageable);
3106: UVMHIST_LOG(maphist,"<- done (vm=%p)", vm,0,0,0);
3107: return (vm);
3108: }
3109:
3110: /*
3111: * uvmspace_init: initialize a vmspace structure.
3112: *
3113: * - XXX: no locking on this structure
3114: * - refcnt set to 1, rest must be init'd by caller
3115: */
3116: void
3117: uvmspace_init(vm, pmap, min, max, pageable)
3118: struct vmspace *vm;
3119: struct pmap *pmap;
3120: vaddr_t min, max;
3121: boolean_t pageable;
3122: {
3123: UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
3124:
3125: memset(vm, 0, sizeof(*vm));
3126:
3127: uvm_map_setup(&vm->vm_map, min, max, pageable ? VM_MAP_PAGEABLE : 0);
3128:
3129: if (pmap)
3130: pmap_reference(pmap);
3131: else
3132: pmap = pmap_create();
3133: vm->vm_map.pmap = pmap;
3134:
3135: vm->vm_refcnt = 1;
3136: UVMHIST_LOG(maphist,"<- done",0,0,0,0);
3137: }
3138:
3139: /*
3140: * uvmspace_share: share a vmspace between two proceses
3141: *
3142: * - XXX: no locking on vmspace
3143: * - used for vfork, threads(?)
3144: */
3145:
3146: void
3147: uvmspace_share(p1, p2)
3148: struct proc *p1, *p2;
3149: {
3150: p2->p_vmspace = p1->p_vmspace;
3151: p1->p_vmspace->vm_refcnt++;
3152: }
3153:
3154: /*
3155: * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
3156: *
3157: * - XXX: no locking on vmspace
3158: */
3159:
3160: void
3161: uvmspace_unshare(p)
3162: struct proc *p;
3163: {
3164: struct vmspace *nvm, *ovm = p->p_vmspace;
3165:
3166: if (ovm->vm_refcnt == 1)
3167: /* nothing to do: vmspace isn't shared in the first place */
3168: return;
3169:
3170: /* make a new vmspace, still holding old one */
3171: nvm = uvmspace_fork(ovm);
3172:
3173: pmap_deactivate(p); /* unbind old vmspace */
3174: p->p_vmspace = nvm;
3175: pmap_activate(p); /* switch to new vmspace */
3176:
3177: uvmspace_free(ovm); /* drop reference to old vmspace */
3178: }
3179:
3180: /*
3181: * uvmspace_exec: the process wants to exec a new program
3182: *
3183: * - XXX: no locking on vmspace
3184: */
3185:
3186: void
3187: uvmspace_exec(struct proc *p, vaddr_t start, vaddr_t end)
3188: {
3189: struct vmspace *nvm, *ovm = p->p_vmspace;
3190: struct vm_map *map = &ovm->vm_map;
3191:
3192: pmap_unuse_final(p); /* before stack addresses go away */
3193:
3194: /*
3195: * see if more than one process is using this vmspace...
3196: */
3197:
3198: if (ovm->vm_refcnt == 1) {
3199:
3200: /*
3201: * if p is the only process using its vmspace then we can safely
3202: * recycle that vmspace for the program that is being exec'd.
3203: */
3204:
3205: #ifdef SYSVSHM
3206: /*
3207: * SYSV SHM semantics require us to kill all segments on an exec
3208: */
3209: if (ovm->vm_shm)
3210: shmexit(ovm);
3211: #endif
3212:
3213: /*
3214: * POSIX 1003.1b -- "lock future mappings" is revoked
3215: * when a process execs another program image.
3216: */
3217: vm_map_lock(map);
3218: vm_map_modflags(map, 0, VM_MAP_WIREFUTURE);
3219: vm_map_unlock(map);
3220:
3221: /*
3222: * now unmap the old program
3223: */
3224: uvm_unmap(map, map->min_offset, map->max_offset);
3225:
3226: /*
3227: * resize the map
3228: */
3229: vm_map_lock(map);
3230: map->min_offset = start;
3231: uvm_tree_sanity(map, "resize enter");
3232: map->max_offset = end;
3233: if (map->header.prev != &map->header)
3234: uvm_rb_fixup(map, map->header.prev);
3235: uvm_tree_sanity(map, "resize leave");
3236: vm_map_unlock(map);
3237:
3238:
3239: } else {
3240:
3241: /*
3242: * p's vmspace is being shared, so we can't reuse it for p since
3243: * it is still being used for others. allocate a new vmspace
3244: * for p
3245: */
3246: nvm = uvmspace_alloc(start, end,
3247: (map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE);
3248:
3249: /*
3250: * install new vmspace and drop our ref to the old one.
3251: */
3252:
3253: pmap_deactivate(p);
3254: p->p_vmspace = nvm;
3255: pmap_activate(p);
3256:
3257: uvmspace_free(ovm);
3258: }
3259: }
3260:
3261: /*
3262: * uvmspace_free: free a vmspace data structure
3263: *
3264: * - XXX: no locking on vmspace
3265: */
3266:
3267: void
3268: uvmspace_free(struct vmspace *vm)
3269: {
3270: struct vm_map_entry *dead_entries;
3271: UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
3272:
3273: UVMHIST_LOG(maphist,"(vm=%p) ref=%ld", vm, vm->vm_refcnt,0,0);
3274: if (--vm->vm_refcnt == 0) {
3275: /*
3276: * lock the map, to wait out all other references to it. delete
3277: * all of the mappings and pages they hold, then call the pmap
3278: * module to reclaim anything left.
3279: */
3280: #ifdef SYSVSHM
3281: /* Get rid of any SYSV shared memory segments. */
3282: if (vm->vm_shm != NULL)
3283: shmexit(vm);
3284: #endif
3285: vm_map_lock(&vm->vm_map);
3286: if (vm->vm_map.nentries) {
3287: uvm_unmap_remove(&vm->vm_map,
3288: vm->vm_map.min_offset, vm->vm_map.max_offset,
3289: &dead_entries, NULL);
3290: if (dead_entries != NULL)
3291: uvm_unmap_detach(dead_entries, 0);
3292: }
3293: pmap_destroy(vm->vm_map.pmap);
3294: vm->vm_map.pmap = NULL;
3295: pool_put(&uvm_vmspace_pool, vm);
3296: }
3297: UVMHIST_LOG(maphist,"<- done", 0,0,0,0);
3298: }
3299:
3300: /*
3301: * F O R K - m a i n e n t r y p o i n t
3302: */
3303: /*
3304: * uvmspace_fork: fork a process' main map
3305: *
3306: * => create a new vmspace for child process from parent.
3307: * => parent's map must not be locked.
3308: */
3309:
3310: struct vmspace *
3311: uvmspace_fork(struct vmspace *vm1)
3312: {
3313: struct vmspace *vm2;
3314: struct vm_map *old_map = &vm1->vm_map;
3315: struct vm_map *new_map;
3316: struct vm_map_entry *old_entry;
3317: struct vm_map_entry *new_entry;
3318: pmap_t new_pmap;
3319: boolean_t protect_child;
3320: UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
3321:
3322: vm_map_lock(old_map);
3323:
3324: vm2 = uvmspace_alloc(old_map->min_offset, old_map->max_offset,
3325: (old_map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE);
3326: memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
3327: (caddr_t) (vm1 + 1) - (caddr_t) &vm1->vm_startcopy);
3328: new_map = &vm2->vm_map; /* XXX */
3329: new_pmap = new_map->pmap;
3330:
3331: old_entry = old_map->header.next;
3332:
3333: /*
3334: * go entry-by-entry
3335: */
3336:
3337: while (old_entry != &old_map->header) {
3338:
3339: /*
3340: * first, some sanity checks on the old entry
3341: */
3342: if (UVM_ET_ISSUBMAP(old_entry))
3343: panic("fork: encountered a submap during fork (illegal)");
3344:
3345: if (!UVM_ET_ISCOPYONWRITE(old_entry) &&
3346: UVM_ET_ISNEEDSCOPY(old_entry))
3347: panic("fork: non-copy_on_write map entry marked needs_copy (illegal)");
3348:
3349:
3350: switch (old_entry->inheritance) {
3351: case MAP_INHERIT_NONE:
3352: /*
3353: * drop the mapping
3354: */
3355: break;
3356:
3357: case MAP_INHERIT_SHARE:
3358: /*
3359: * share the mapping: this means we want the old and
3360: * new entries to share amaps and backing objects.
3361: */
3362:
3363: /*
3364: * if the old_entry needs a new amap (due to prev fork)
3365: * then we need to allocate it now so that we have
3366: * something we own to share with the new_entry. [in
3367: * other words, we need to clear needs_copy]
3368: */
3369:
3370: if (UVM_ET_ISNEEDSCOPY(old_entry)) {
3371: /* get our own amap, clears needs_copy */
3372: amap_copy(old_map, old_entry, M_WAITOK, FALSE,
3373: 0, 0);
3374: /* XXXCDC: WAITOK??? */
3375: }
3376:
3377: new_entry = uvm_mapent_alloc(new_map);
3378: /* old_entry -> new_entry */
3379: uvm_mapent_copy(old_entry, new_entry);
3380:
3381: /* new pmap has nothing wired in it */
3382: new_entry->wired_count = 0;
3383:
3384: /*
3385: * gain reference to object backing the map (can't
3386: * be a submap, already checked this case).
3387: */
3388: if (new_entry->aref.ar_amap)
3389: /* share reference */
3390: uvm_map_reference_amap(new_entry, AMAP_SHARED);
3391:
3392: if (new_entry->object.uvm_obj &&
3393: new_entry->object.uvm_obj->pgops->pgo_reference)
3394: new_entry->object.uvm_obj->
3395: pgops->pgo_reference(
3396: new_entry->object.uvm_obj);
3397:
3398: /* insert entry at end of new_map's entry list */
3399: uvm_map_entry_link(new_map, new_map->header.prev,
3400: new_entry);
3401:
3402: /*
3403: * pmap_copy the mappings: this routine is optional
3404: * but if it is there it will reduce the number of
3405: * page faults in the new proc.
3406: */
3407:
3408: pmap_copy(new_pmap, old_map->pmap, new_entry->start,
3409: (old_entry->end - old_entry->start),
3410: old_entry->start);
3411:
3412: break;
3413:
3414: case MAP_INHERIT_COPY:
3415:
3416: /*
3417: * copy-on-write the mapping (using mmap's
3418: * MAP_PRIVATE semantics)
3419: *
3420: * allocate new_entry, adjust reference counts.
3421: * (note that new references are read-only).
3422: */
3423:
3424: new_entry = uvm_mapent_alloc(new_map);
3425: /* old_entry -> new_entry */
3426: uvm_mapent_copy(old_entry, new_entry);
3427:
3428: if (new_entry->aref.ar_amap)
3429: uvm_map_reference_amap(new_entry, 0);
3430:
3431: if (new_entry->object.uvm_obj &&
3432: new_entry->object.uvm_obj->pgops->pgo_reference)
3433: new_entry->object.uvm_obj->pgops->pgo_reference
3434: (new_entry->object.uvm_obj);
3435:
3436: /* new pmap has nothing wired in it */
3437: new_entry->wired_count = 0;
3438:
3439: new_entry->etype |=
3440: (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
3441: uvm_map_entry_link(new_map, new_map->header.prev,
3442: new_entry);
3443:
3444: /*
3445: * the new entry will need an amap. it will either
3446: * need to be copied from the old entry or created
3447: * from scratch (if the old entry does not have an
3448: * amap). can we defer this process until later
3449: * (by setting "needs_copy") or do we need to copy
3450: * the amap now?
3451: *
3452: * we must copy the amap now if any of the following
3453: * conditions hold:
3454: * 1. the old entry has an amap and that amap is
3455: * being shared. this means that the old (parent)
3456: * process is sharing the amap with another
3457: * process. if we do not clear needs_copy here
3458: * we will end up in a situation where both the
3459: * parent and child process are referring to the
3460: * same amap with "needs_copy" set. if the
3461: * parent write-faults, the fault routine will
3462: * clear "needs_copy" in the parent by allocating
3463: * a new amap. this is wrong because the
3464: * parent is supposed to be sharing the old amap
3465: * and the new amap will break that.
3466: *
3467: * 2. if the old entry has an amap and a non-zero
3468: * wire count then we are going to have to call
3469: * amap_cow_now to avoid page faults in the
3470: * parent process. since amap_cow_now requires
3471: * "needs_copy" to be clear we might as well
3472: * clear it here as well.
3473: *
3474: */
3475:
3476: if (old_entry->aref.ar_amap != NULL) {
3477:
3478: if ((amap_flags(old_entry->aref.ar_amap) &
3479: AMAP_SHARED) != 0 ||
3480: VM_MAPENT_ISWIRED(old_entry)) {
3481:
3482: amap_copy(new_map, new_entry, M_WAITOK, FALSE,
3483: 0, 0);
3484: /* XXXCDC: M_WAITOK ... ok? */
3485: }
3486: }
3487:
3488: /*
3489: * if the parent's entry is wired down, then the
3490: * parent process does not want page faults on
3491: * access to that memory. this means that we
3492: * cannot do copy-on-write because we can't write
3493: * protect the old entry. in this case we
3494: * resolve all copy-on-write faults now, using
3495: * amap_cow_now. note that we have already
3496: * allocated any needed amap (above).
3497: */
3498:
3499: if (VM_MAPENT_ISWIRED(old_entry)) {
3500:
3501: /*
3502: * resolve all copy-on-write faults now
3503: * (note that there is nothing to do if
3504: * the old mapping does not have an amap).
3505: * XXX: is it worthwhile to bother with pmap_copy
3506: * in this case?
3507: */
3508: if (old_entry->aref.ar_amap)
3509: amap_cow_now(new_map, new_entry);
3510:
3511: } else {
3512:
3513: /*
3514: * setup mappings to trigger copy-on-write faults
3515: * we must write-protect the parent if it has
3516: * an amap and it is not already "needs_copy"...
3517: * if it is already "needs_copy" then the parent
3518: * has already been write-protected by a previous
3519: * fork operation.
3520: *
3521: * if we do not write-protect the parent, then
3522: * we must be sure to write-protect the child
3523: * after the pmap_copy() operation.
3524: *
3525: * XXX: pmap_copy should have some way of telling
3526: * us that it didn't do anything so we can avoid
3527: * calling pmap_protect needlessly.
3528: */
3529:
3530: if (old_entry->aref.ar_amap) {
3531:
3532: if (!UVM_ET_ISNEEDSCOPY(old_entry)) {
3533: if (old_entry->max_protection & VM_PROT_WRITE) {
3534: pmap_protect(old_map->pmap,
3535: old_entry->start,
3536: old_entry->end,
3537: old_entry->protection &
3538: ~VM_PROT_WRITE);
3539: pmap_update(old_map->pmap);
3540:
3541: }
3542: old_entry->etype |= UVM_ET_NEEDSCOPY;
3543: }
3544:
3545: /*
3546: * parent must now be write-protected
3547: */
3548: protect_child = FALSE;
3549: } else {
3550:
3551: /*
3552: * we only need to protect the child if the
3553: * parent has write access.
3554: */
3555: if (old_entry->max_protection & VM_PROT_WRITE)
3556: protect_child = TRUE;
3557: else
3558: protect_child = FALSE;
3559:
3560: }
3561:
3562: /*
3563: * copy the mappings
3564: * XXX: need a way to tell if this does anything
3565: */
3566:
3567: pmap_copy(new_pmap, old_map->pmap,
3568: new_entry->start,
3569: (old_entry->end - old_entry->start),
3570: old_entry->start);
3571:
3572: /*
3573: * protect the child's mappings if necessary
3574: */
3575: if (protect_child) {
3576: pmap_protect(new_pmap, new_entry->start,
3577: new_entry->end,
3578: new_entry->protection &
3579: ~VM_PROT_WRITE);
3580: }
3581:
3582: }
3583: break;
3584: } /* end of switch statement */
3585: old_entry = old_entry->next;
3586: }
3587:
3588: new_map->size = old_map->size;
3589: vm_map_unlock(old_map);
3590:
3591: #ifdef SYSVSHM
3592: if (vm1->vm_shm)
3593: shmfork(vm1, vm2);
3594: #endif
3595:
3596: #ifdef PMAP_FORK
3597: pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
3598: #endif
3599:
3600: UVMHIST_LOG(maphist,"<- done",0,0,0,0);
3601: return(vm2);
3602: }
3603:
3604: #if defined(DDB)
3605:
3606: /*
3607: * DDB hooks
3608: */
3609:
3610: /*
3611: * uvm_map_printit: actually prints the map
3612: */
3613:
3614: void
3615: uvm_map_printit(struct vm_map *map, boolean_t full,
3616: int (*pr)(const char *, ...))
3617: {
3618: struct vm_map_entry *entry;
3619:
3620: (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, map->min_offset,map->max_offset);
3621: (*pr)("\t#ent=%d, sz=%u, ref=%d, version=%u, flags=0x%x\n",
3622: map->nentries, map->size, map->ref_count, map->timestamp,
3623: map->flags);
3624: #ifdef pmap_resident_count
3625: (*pr)("\tpmap=%p(resident=%d)\n", map->pmap,
3626: pmap_resident_count(map->pmap));
3627: #else
3628: /* XXXCDC: this should be required ... */
3629: (*pr)("\tpmap=%p(resident=<<NOT SUPPORTED!!!>>)\n", map->pmap);
3630: #endif
3631: if (!full)
3632: return;
3633: for (entry = map->header.next; entry != &map->header;
3634: entry = entry->next) {
3635: (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n",
3636: entry, entry->start, entry->end, entry->object.uvm_obj,
3637: (long long)entry->offset, entry->aref.ar_amap,
3638: entry->aref.ar_pageoff);
3639: (*pr)(
3640: "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, "
3641: "wc=%d, adv=%d\n",
3642: (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
3643: (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
3644: (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
3645: entry->protection, entry->max_protection,
3646: entry->inheritance, entry->wired_count, entry->advice);
3647: }
3648: }
3649:
3650: /*
3651: * uvm_object_printit: actually prints the object
3652: */
3653:
3654: void
3655: uvm_object_printit(uobj, full, pr)
3656: struct uvm_object *uobj;
3657: boolean_t full;
3658: int (*pr)(const char *, ...);
3659: {
3660: struct vm_page *pg;
3661: int cnt = 0;
3662:
3663: (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
3664: uobj, uobj->vmobjlock.lock_data, uobj->pgops, uobj->uo_npages);
3665: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
3666: (*pr)("refs=<SYSTEM>\n");
3667: else
3668: (*pr)("refs=%d\n", uobj->uo_refs);
3669:
3670: if (!full) {
3671: return;
3672: }
3673: (*pr)(" PAGES <pg,offset>:\n ");
3674: for (pg = TAILQ_FIRST(&uobj->memq);
3675: pg != NULL;
3676: pg = TAILQ_NEXT(pg, listq), cnt++) {
3677: (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
3678: if ((cnt % 3) == 2) {
3679: (*pr)("\n ");
3680: }
3681: }
3682: if ((cnt % 3) != 2) {
3683: (*pr)("\n");
3684: }
3685: }
3686:
3687: /*
3688: * uvm_page_printit: actually print the page
3689: */
3690:
3691: static const char page_flagbits[] =
3692: "\20\1BUSY\2WANTED\3TABLED\4CLEAN\5CLEANCHK\6RELEASED\7FAKE\10RDONLY"
3693: "\11ZERO\15PAGER1";
3694: static const char page_pqflagbits[] =
3695: "\20\1FREE\2INACTIVE\3ACTIVE\4LAUNDRY\5ANON\6AOBJ";
3696:
3697: void
3698: uvm_page_printit(pg, full, pr)
3699: struct vm_page *pg;
3700: boolean_t full;
3701: int (*pr)(const char *, ...);
3702: {
3703: struct vm_page *tpg;
3704: struct uvm_object *uobj;
3705: struct pglist *pgl;
3706: char pgbuf[128];
3707: char pqbuf[128];
3708:
3709: (*pr)("PAGE %p:\n", pg);
3710: snprintf(pgbuf, sizeof(pgbuf), "%b", pg->pg_flags, page_flagbits);
3711: snprintf(pqbuf, sizeof(pqbuf), "%b", pg->pg_flags, page_pqflagbits);
3712: (*pr)(" flags=%s, pg_flags=%s, vers=%d, wire_count=%d, pa=0x%llx\n",
3713: pgbuf, pqbuf, pg->pg_version, pg->wire_count,
3714: (long long)pg->phys_addr);
3715: (*pr)(" uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n",
3716: pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count);
3717: #if defined(UVM_PAGE_TRKOWN)
3718: if (pg->pg_flags & PG_BUSY)
3719: (*pr)(" owning process = %d, tag=%s\n",
3720: pg->owner, pg->owner_tag);
3721: else
3722: (*pr)(" page not busy, no owner\n");
3723: #else
3724: (*pr)(" [page ownership tracking disabled]\n");
3725: #endif
3726:
3727: if (!full)
3728: return;
3729:
3730: /* cross-verify object/anon */
3731: if ((pg->pg_flags & PQ_FREE) == 0) {
3732: if (pg->pg_flags & PQ_ANON) {
3733: if (pg->uanon == NULL || pg->uanon->an_page != pg)
3734: (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n",
3735: (pg->uanon) ? pg->uanon->an_page : NULL);
3736: else
3737: (*pr)(" anon backpointer is OK\n");
3738: } else {
3739: uobj = pg->uobject;
3740: if (uobj) {
3741: (*pr)(" checking object list\n");
3742: TAILQ_FOREACH(tpg, &uobj->memq, listq) {
3743: if (tpg == pg) {
3744: break;
3745: }
3746: }
3747: if (tpg)
3748: (*pr)(" page found on object list\n");
3749: else
3750: (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
3751: }
3752: }
3753: }
3754:
3755: /* cross-verify page queue */
3756: if (pg->pg_flags & PQ_FREE) {
3757: int fl = uvm_page_lookup_freelist(pg);
3758: pgl = &uvm.page_free[fl].pgfl_queues[((pg)->pg_flags & PG_ZERO) ?
3759: PGFL_ZEROS : PGFL_UNKNOWN];
3760: } else if (pg->pg_flags & PQ_INACTIVE) {
3761: pgl = (pg->pg_flags & PQ_SWAPBACKED) ?
3762: &uvm.page_inactive_swp : &uvm.page_inactive_obj;
3763: } else if (pg->pg_flags & PQ_ACTIVE) {
3764: pgl = &uvm.page_active;
3765: } else {
3766: pgl = NULL;
3767: }
3768:
3769: if (pgl) {
3770: (*pr)(" checking pageq list\n");
3771: TAILQ_FOREACH(tpg, pgl, pageq) {
3772: if (tpg == pg) {
3773: break;
3774: }
3775: }
3776: if (tpg)
3777: (*pr)(" page found on pageq list\n");
3778: else
3779: (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
3780: }
3781: }
3782: #endif