![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_fault.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm|
Return to uvm_fault.c CVS log | Up to [local] / sys / uvm |
1.1 nbrk 1: /* $OpenBSD: uvm_fault.c,v 1.49 2007/06/18 21:51:15 pedro Exp $ */
2: /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
3:
4: /*
5: *
6: * Copyright (c) 1997 Charles D. Cranor and Washington University.
7: * All rights reserved.
8: *
9: * Redistribution and use in source and binary forms, with or without
10: * modification, are permitted provided that the following conditions
11: * are met:
12: * 1. Redistributions of source code must retain the above copyright
13: * notice, this list of conditions and the following disclaimer.
14: * 2. Redistributions in binary form must reproduce the above copyright
15: * notice, this list of conditions and the following disclaimer in the
16: * documentation and/or other materials provided with the distribution.
17: * 3. All advertising materials mentioning features or use of this software
18: * must display the following acknowledgement:
19: * This product includes software developed by Charles D. Cranor and
20: * Washington University.
21: * 4. The name of the author may not be used to endorse or promote products
22: * derived from this software without specific prior written permission.
23: *
24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34: *
35: * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
36: */
37:
38: /*
39: * uvm_fault.c: fault handler
40: */
41:
42: #include <sys/param.h>
43: #include <sys/systm.h>
44: #include <sys/kernel.h>
45: #include <sys/proc.h>
46: #include <sys/malloc.h>
47: #include <sys/mman.h>
48: #include <sys/user.h>
49:
50: #include <uvm/uvm.h>
51:
52: /*
53: *
54: * a word on page faults:
55: *
56: * types of page faults we handle:
57: *
58: * CASE 1: upper layer faults CASE 2: lower layer faults
59: *
60: * CASE 1A CASE 1B CASE 2A CASE 2B
61: * read/write1 write>1 read/write +-cow_write/zero
62: * | | | |
63: * +--|--+ +--|--+ +-----+ + | + | +-----+
64: * amap | V | | ----------->new| | | | ^ |
65: * +-----+ +-----+ +-----+ + | + | +--|--+
66: * | | |
67: * +-----+ +-----+ +--|--+ | +--|--+
68: * uobj | d/c | | d/c | | V | +----| |
69: * +-----+ +-----+ +-----+ +-----+
70: *
71: * d/c = don't care
72: *
73: * case [0]: layerless fault
74: * no amap or uobj is present. this is an error.
75: *
76: * case [1]: upper layer fault [anon active]
77: * 1A: [read] or [write with anon->an_ref == 1]
78: * I/O takes place in top level anon and uobj is not touched.
79: * 1B: [write with anon->an_ref > 1]
80: * new anon is alloc'd and data is copied off ["COW"]
81: *
82: * case [2]: lower layer fault [uobj]
83: * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
84: * I/O takes place directly in object.
85: * 2B: [write to copy_on_write] or [read on NULL uobj]
86: * data is "promoted" from uobj to a new anon.
87: * if uobj is null, then we zero fill.
88: *
89: * we follow the standard UVM locking protocol ordering:
90: *
91: * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
92: * we hold a PG_BUSY page if we unlock for I/O
93: *
94: *
95: * the code is structured as follows:
96: *
97: * - init the "IN" params in the ufi structure
98: * ReFault:
99: * - do lookups [locks maps], check protection, handle needs_copy
100: * - check for case 0 fault (error)
101: * - establish "range" of fault
102: * - if we have an amap lock it and extract the anons
103: * - if sequential advice deactivate pages behind us
104: * - at the same time check pmap for unmapped areas and anon for pages
105: * that we could map in (and do map it if found)
106: * - check object for resident pages that we could map in
107: * - if (case 2) goto Case2
108: * - >>> handle case 1
109: * - ensure source anon is resident in RAM
110: * - if case 1B alloc new anon and copy from source
111: * - map the correct page in
112: * Case2:
113: * - >>> handle case 2
114: * - ensure source page is resident (if uobj)
115: * - if case 2B alloc new anon and copy from source (could be zero
116: * fill if uobj == NULL)
117: * - map the correct page in
118: * - done!
119: *
120: * note on paging:
121: * if we have to do I/O we place a PG_BUSY page in the correct object,
122: * unlock everything, and do the I/O. when I/O is done we must reverify
123: * the state of the world before assuming that our data structures are
124: * valid. [because mappings could change while the map is unlocked]
125: *
126: * alternative 1: unbusy the page in question and restart the page fault
127: * from the top (ReFault). this is easy but does not take advantage
128: * of the information that we already have from our previous lookup,
129: * although it is possible that the "hints" in the vm_map will help here.
130: *
131: * alternative 2: the system already keeps track of a "version" number of
132: * a map. [i.e. every time you write-lock a map (e.g. to change a
133: * mapping) you bump the version number up by one...] so, we can save
134: * the version number of the map before we release the lock and start I/O.
135: * then when I/O is done we can relock and check the version numbers
136: * to see if anything changed. this might save us some over 1 because
137: * we don't have to unbusy the page and may be less compares(?).
138: *
139: * alternative 3: put in backpointers or a way to "hold" part of a map
140: * in place while I/O is in progress. this could be complex to
141: * implement (especially with structures like amap that can be referenced
142: * by multiple map entries, and figuring out what should wait could be
143: * complex as well...).
144: *
145: * given that we are not currently multiprocessor or multithreaded we might
146: * as well choose alternative 2 now. maybe alternative 3 would be useful
147: * in the future. XXX keep in mind for future consideration//rechecking.
148: */
149:
150: /*
151: * local data structures
152: */
153:
154: struct uvm_advice {
155: int advice;
156: int nback;
157: int nforw;
158: };
159:
160: /*
161: * page range array:
162: * note: index in array must match "advice" value
163: * XXX: borrowed numbers from freebsd. do they work well for us?
164: */
165:
166: static struct uvm_advice uvmadvice[] = {
167: { MADV_NORMAL, 3, 4 },
168: { MADV_RANDOM, 0, 0 },
169: { MADV_SEQUENTIAL, 8, 7},
170: };
171:
172: #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */
173:
174: /*
175: * private prototypes
176: */
177:
178: static void uvmfault_amapcopy(struct uvm_faultinfo *);
179: static __inline void uvmfault_anonflush(struct vm_anon **, int);
180:
181: /*
182: * inline functions
183: */
184:
185: /*
186: * uvmfault_anonflush: try and deactivate pages in specified anons
187: *
188: * => does not have to deactivate page if it is busy
189: */
190:
191: static __inline void
192: uvmfault_anonflush(anons, n)
193: struct vm_anon **anons;
194: int n;
195: {
196: int lcv;
197: struct vm_page *pg;
198:
199: for (lcv = 0 ; lcv < n ; lcv++) {
200: if (anons[lcv] == NULL)
201: continue;
202: simple_lock(&anons[lcv]->an_lock);
203: pg = anons[lcv]->an_page;
204: if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) {
205: uvm_lock_pageq();
206: if (pg->wire_count == 0) {
207: #ifdef UBC
208: pmap_clear_reference(pg);
209: #else
210: pmap_page_protect(pg, VM_PROT_NONE);
211: #endif
212: uvm_pagedeactivate(pg);
213: }
214: uvm_unlock_pageq();
215: }
216: simple_unlock(&anons[lcv]->an_lock);
217: }
218: }
219:
220: /*
221: * normal functions
222: */
223:
224: /*
225: * uvmfault_amapcopy: clear "needs_copy" in a map.
226: *
227: * => called with VM data structures unlocked (usually, see below)
228: * => we get a write lock on the maps and clear needs_copy for a VA
229: * => if we are out of RAM we sleep (waiting for more)
230: */
231:
232: static void
233: uvmfault_amapcopy(ufi)
234: struct uvm_faultinfo *ufi;
235: {
236:
237: /*
238: * while we haven't done the job
239: */
240:
241: while (1) {
242:
243: /*
244: * no mapping? give up.
245: */
246:
247: if (uvmfault_lookup(ufi, TRUE) == FALSE)
248: return;
249:
250: /*
251: * copy if needed.
252: */
253:
254: if (UVM_ET_ISNEEDSCOPY(ufi->entry))
255: amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE,
256: ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
257:
258: /*
259: * didn't work? must be out of RAM. unlock and sleep.
260: */
261:
262: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
263: uvmfault_unlockmaps(ufi, TRUE);
264: uvm_wait("fltamapcopy");
265: continue;
266: }
267:
268: /*
269: * got it! unlock and return.
270: */
271:
272: uvmfault_unlockmaps(ufi, TRUE);
273: return;
274: }
275: /*NOTREACHED*/
276: }
277:
278: /*
279: * uvmfault_anonget: get data in an anon into a non-busy, non-released
280: * page in that anon.
281: *
282: * => maps, amap, and anon locked by caller.
283: * => if we fail (result != VM_PAGER_OK) we unlock everything.
284: * => if we are successful, we return with everything still locked.
285: * => we don't move the page on the queues [gets moved later]
286: * => if we allocate a new page [we_own], it gets put on the queues.
287: * either way, the result is that the page is on the queues at return time
288: * => for pages which are on loan from a uvm_object (and thus are not
289: * owned by the anon): if successful, we return with the owning object
290: * locked. the caller must unlock this object when it unlocks everything
291: * else.
292: */
293:
294: int
295: uvmfault_anonget(ufi, amap, anon)
296: struct uvm_faultinfo *ufi;
297: struct vm_amap *amap;
298: struct vm_anon *anon;
299: {
300: boolean_t we_own; /* we own anon's page? */
301: boolean_t locked; /* did we relock? */
302: struct vm_page *pg;
303: int result;
304: UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
305:
306: result = 0; /* XXX shut up gcc */
307: uvmexp.fltanget++;
308: /* bump rusage counters */
309: if (anon->an_page)
310: curproc->p_addr->u_stats.p_ru.ru_minflt++;
311: else
312: curproc->p_addr->u_stats.p_ru.ru_majflt++;
313:
314: /*
315: * loop until we get it, or fail.
316: */
317:
318: while (1) {
319:
320: we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
321: pg = anon->an_page;
322:
323: /*
324: * if there is a resident page and it is loaned, then anon
325: * may not own it. call out to uvm_anon_lockpage() to ensure
326: * the real owner of the page has been identified and locked.
327: */
328:
329: if (pg && pg->loan_count)
330: pg = uvm_anon_lockloanpg(anon);
331:
332: /*
333: * page there? make sure it is not busy/released.
334: */
335:
336: if (pg) {
337:
338: /*
339: * at this point, if the page has a uobject [meaning
340: * we have it on loan], then that uobject is locked
341: * by us! if the page is busy, we drop all the
342: * locks (including uobject) and try again.
343: */
344:
345: if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
346: UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
347: return (VM_PAGER_OK);
348: }
349: atomic_setbits_int(&pg->pg_flags, PG_WANTED);
350: uvmexp.fltpgwait++;
351:
352: /*
353: * the last unlock must be an atomic unlock+wait on
354: * the owner of page
355: */
356: if (pg->uobject) { /* owner is uobject ? */
357: uvmfault_unlockall(ufi, amap, NULL, anon);
358: UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
359: 0,0,0);
360: UVM_UNLOCK_AND_WAIT(pg,
361: &pg->uobject->vmobjlock,
362: FALSE, "anonget1",0);
363: } else {
364: /* anon owns page */
365: uvmfault_unlockall(ufi, amap, NULL, NULL);
366: UVMHIST_LOG(maphist, " unlock+wait on anon",0,
367: 0,0,0);
368: UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
369: "anonget2",0);
370: }
371: /* ready to relock and try again */
372:
373: } else {
374:
375: /*
376: * no page, we must try and bring it in.
377: */
378: pg = uvm_pagealloc(NULL, 0, anon, 0);
379:
380: if (pg == NULL) { /* out of RAM. */
381:
382: uvmfault_unlockall(ufi, amap, NULL, anon);
383: uvmexp.fltnoram++;
384: UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
385: 0,0,0);
386: uvm_wait("flt_noram1");
387: /* ready to relock and try again */
388:
389: } else {
390:
391: /* we set the PG_BUSY bit */
392: we_own = TRUE;
393: uvmfault_unlockall(ufi, amap, NULL, anon);
394:
395: /*
396: * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
397: * page into the uvm_swap_get function with
398: * all data structures unlocked. note that
399: * it is ok to read an_swslot here because
400: * we hold PG_BUSY on the page.
401: */
402: uvmexp.pageins++;
403: result = uvm_swap_get(pg, anon->an_swslot,
404: PGO_SYNCIO);
405:
406: /*
407: * we clean up after the i/o below in the
408: * "we_own" case
409: */
410: /* ready to relock and try again */
411: }
412: }
413:
414: /*
415: * now relock and try again
416: */
417:
418: locked = uvmfault_relock(ufi);
419: if (locked || we_own)
420: simple_lock(&anon->an_lock);
421:
422: /*
423: * if we own the page (i.e. we set PG_BUSY), then we need
424: * to clean up after the I/O. there are three cases to
425: * consider:
426: * [1] page released during I/O: free anon and ReFault.
427: * [2] I/O not OK. free the page and cause the fault
428: * to fail.
429: * [3] I/O OK! activate the page and sync with the
430: * non-we_own case (i.e. drop anon lock if not locked).
431: */
432:
433: if (we_own) {
434:
435: if (pg->pg_flags & PG_WANTED) {
436: /* still holding object lock */
437: wakeup(pg);
438: }
439: /* un-busy! */
440: atomic_clearbits_int(&pg->pg_flags,
441: PG_WANTED|PG_BUSY|PG_FAKE);
442: UVM_PAGE_OWN(pg, NULL);
443:
444: /*
445: * if we were RELEASED during I/O, then our anon is
446: * no longer part of an amap. we need to free the
447: * anon and try again.
448: */
449: if (pg->pg_flags & PG_RELEASED) {
450: pmap_page_protect(pg, VM_PROT_NONE);
451: simple_unlock(&anon->an_lock);
452: uvm_anfree(anon); /* frees page for us */
453: if (locked)
454: uvmfault_unlockall(ufi, amap, NULL,
455: NULL);
456: uvmexp.fltpgrele++;
457: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
458: return (VM_PAGER_REFAULT); /* refault! */
459: }
460:
461: if (result != VM_PAGER_OK) {
462: KASSERT(result != VM_PAGER_PEND);
463:
464: /* remove page from anon */
465: anon->an_page = NULL;
466:
467: /*
468: * remove the swap slot from the anon
469: * and mark the anon as having no real slot.
470: * don't free the swap slot, thus preventing
471: * it from being used again.
472: */
473: uvm_swap_markbad(anon->an_swslot, 1);
474: anon->an_swslot = SWSLOT_BAD;
475:
476: /*
477: * note: page was never !PG_BUSY, so it
478: * can't be mapped and thus no need to
479: * pmap_page_protect it...
480: */
481: uvm_lock_pageq();
482: uvm_pagefree(pg);
483: uvm_unlock_pageq();
484:
485: if (locked)
486: uvmfault_unlockall(ufi, amap, NULL,
487: anon);
488: else
489: simple_unlock(&anon->an_lock);
490: UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
491: return (VM_PAGER_ERROR);
492: }
493:
494: /*
495: * must be OK, clear modify (already PG_CLEAN)
496: * and activate
497: */
498: pmap_clear_modify(pg);
499: uvm_lock_pageq();
500: uvm_pageactivate(pg);
501: uvm_unlock_pageq();
502: if (!locked)
503: simple_unlock(&anon->an_lock);
504: }
505:
506: /*
507: * we were not able to relock. restart fault.
508: */
509:
510: if (!locked) {
511: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
512: return (VM_PAGER_REFAULT);
513: }
514:
515: /*
516: * verify no one has touched the amap and moved the anon on us.
517: */
518:
519: if (ufi != NULL &&
520: amap_lookup(&ufi->entry->aref,
521: ufi->orig_rvaddr - ufi->entry->start) != anon) {
522:
523: uvmfault_unlockall(ufi, amap, NULL, anon);
524: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
525: return (VM_PAGER_REFAULT);
526: }
527:
528: /*
529: * try it again!
530: */
531:
532: uvmexp.fltanretry++;
533: continue;
534:
535: } /* while (1) */
536:
537: /*NOTREACHED*/
538: }
539:
540: /*
541: * F A U L T - m a i n e n t r y p o i n t
542: */
543:
544: /*
545: * uvm_fault: page fault handler
546: *
547: * => called from MD code to resolve a page fault
548: * => VM data structures usually should be unlocked. however, it is
549: * possible to call here with the main map locked if the caller
550: * gets a write lock, sets it recursive, and then calls us (c.f.
551: * uvm_map_pageable). this should be avoided because it keeps
552: * the map locked off during I/O.
553: */
554:
555: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
556: ~VM_PROT_WRITE : VM_PROT_ALL)
557:
558: int
559: uvm_fault(orig_map, vaddr, fault_type, access_type)
560: vm_map_t orig_map;
561: vaddr_t vaddr;
562: vm_fault_t fault_type;
563: vm_prot_t access_type;
564: {
565: struct uvm_faultinfo ufi;
566: vm_prot_t enter_prot;
567: boolean_t wired, narrow, promote, locked, shadowed;
568: int npages, nback, nforw, centeridx, result, lcv, gotpages;
569: vaddr_t startva, currva;
570: voff_t uoff;
571: paddr_t pa;
572: struct vm_amap *amap;
573: struct uvm_object *uobj;
574: struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
575: struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
576: UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
577:
578: UVMHIST_LOG(maphist, "(map=%p, vaddr=0x%lx, ft=%ld, at=%ld)",
579: orig_map, vaddr, fault_type, access_type);
580:
581: anon = NULL;
582: pg = NULL;
583:
584: uvmexp.faults++; /* XXX: locking? */
585:
586: /*
587: * init the IN parameters in the ufi
588: */
589:
590: ufi.orig_map = orig_map;
591: ufi.orig_rvaddr = trunc_page(vaddr);
592: ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
593: if (fault_type == VM_FAULT_WIRE)
594: narrow = TRUE; /* don't look for neighborhood
595: * pages on wire */
596: else
597: narrow = FALSE; /* normal fault */
598:
599: /*
600: * "goto ReFault" means restart the page fault from ground zero.
601: */
602: ReFault:
603:
604: /*
605: * lookup and lock the maps
606: */
607:
608: if (uvmfault_lookup(&ufi, FALSE) == FALSE) {
609: UVMHIST_LOG(maphist, "<- no mapping @ 0x%lx", vaddr, 0,0,0);
610: return (EFAULT);
611: }
612: /* locked: maps(read) */
613:
614: #ifdef DIAGNOSTIC
615: if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0)
616: panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)",
617: ufi.map, vaddr);
618: #endif
619:
620: /*
621: * check protection
622: */
623:
624: if ((ufi.entry->protection & access_type) != access_type) {
625: UVMHIST_LOG(maphist,
626: "<- protection failure (prot=0x%lx, access=0x%lx)",
627: ufi.entry->protection, access_type, 0, 0);
628: uvmfault_unlockmaps(&ufi, FALSE);
629: return (EACCES);
630: }
631:
632: /*
633: * "enter_prot" is the protection we want to enter the page in at.
634: * for certain pages (e.g. copy-on-write pages) this protection can
635: * be more strict than ufi.entry->protection. "wired" means either
636: * the entry is wired or we are fault-wiring the pg.
637: */
638:
639: enter_prot = ufi.entry->protection;
640: wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE);
641: if (wired)
642: access_type = enter_prot; /* full access for wired */
643:
644: /*
645: * handle "needs_copy" case. if we need to copy the amap we will
646: * have to drop our readlock and relock it with a write lock. (we
647: * need a write lock to change anything in a map entry [e.g.
648: * needs_copy]).
649: */
650:
651: if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
652: if ((access_type & VM_PROT_WRITE) ||
653: (ufi.entry->object.uvm_obj == NULL)) {
654: /* need to clear */
655: UVMHIST_LOG(maphist,
656: " need to clear needs_copy and refault",0,0,0,0);
657: uvmfault_unlockmaps(&ufi, FALSE);
658: uvmfault_amapcopy(&ufi);
659: uvmexp.fltamcopy++;
660: goto ReFault;
661:
662: } else {
663:
664: /*
665: * ensure that we pmap_enter page R/O since
666: * needs_copy is still true
667: */
668: enter_prot &= ~VM_PROT_WRITE;
669:
670: }
671: }
672:
673: /*
674: * identify the players
675: */
676:
677: amap = ufi.entry->aref.ar_amap; /* top layer */
678: uobj = ufi.entry->object.uvm_obj; /* bottom layer */
679:
680: /*
681: * check for a case 0 fault. if nothing backing the entry then
682: * error now.
683: */
684:
685: if (amap == NULL && uobj == NULL) {
686: uvmfault_unlockmaps(&ufi, FALSE);
687: UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
688: return (EFAULT);
689: }
690:
691: /*
692: * establish range of interest based on advice from mapper
693: * and then clip to fit map entry. note that we only want
694: * to do this the first time through the fault. if we
695: * ReFault we will disable this by setting "narrow" to true.
696: */
697:
698: if (narrow == FALSE) {
699:
700: /* wide fault (!narrow) */
701: KASSERT(uvmadvice[ufi.entry->advice].advice ==
702: ufi.entry->advice);
703: nback = min(uvmadvice[ufi.entry->advice].nback,
704: (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
705: startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
706: nforw = min(uvmadvice[ufi.entry->advice].nforw,
707: ((ufi.entry->end - ufi.orig_rvaddr) >>
708: PAGE_SHIFT) - 1);
709: /*
710: * note: "-1" because we don't want to count the
711: * faulting page as forw
712: */
713: npages = nback + nforw + 1;
714: centeridx = nback;
715:
716: narrow = TRUE; /* ensure only once per-fault */
717:
718: } else {
719:
720: /* narrow fault! */
721: nback = nforw = 0;
722: startva = ufi.orig_rvaddr;
723: npages = 1;
724: centeridx = 0;
725:
726: }
727:
728: /* locked: maps(read) */
729: UVMHIST_LOG(maphist, " narrow=%ld, back=%ld, forw=%ld, startva=0x%lx",
730: narrow, nback, nforw, startva);
731: UVMHIST_LOG(maphist, " entry=%p, amap=%p, obj=%p", ufi.entry,
732: amap, uobj, 0);
733:
734: /*
735: * if we've got an amap, lock it and extract current anons.
736: */
737:
738: if (amap) {
739: anons = anons_store;
740: amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
741: anons, npages);
742: } else {
743: anons = NULL; /* to be safe */
744: }
745:
746: /* locked: maps(read), amap(if there) */
747:
748: /*
749: * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
750: * now and then forget about them (for the rest of the fault).
751: */
752:
753: if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) {
754:
755: UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
756: 0,0,0,0);
757: /* flush back-page anons? */
758: if (amap)
759: uvmfault_anonflush(anons, nback);
760:
761: /* flush object? */
762: if (uobj) {
763: uoff = (startva - ufi.entry->start) + ufi.entry->offset;
764: simple_lock(&uobj->vmobjlock);
765: (void) uobj->pgops->pgo_flush(uobj, uoff, uoff +
766: (nback << PAGE_SHIFT), PGO_DEACTIVATE);
767: simple_unlock(&uobj->vmobjlock);
768: }
769:
770: /* now forget about the backpages */
771: if (amap)
772: anons += nback;
773: startva += (nback << PAGE_SHIFT);
774: npages -= nback;
775: nback = centeridx = 0;
776: }
777:
778: /* locked: maps(read), amap(if there) */
779:
780: /*
781: * map in the backpages and frontpages we found in the amap in hopes
782: * of preventing future faults. we also init the pages[] array as
783: * we go.
784: */
785:
786: currva = startva;
787: shadowed = FALSE;
788: for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
789:
790: /*
791: * dont play with VAs that are already mapped
792: * except for center)
793: */
794: if (lcv != centeridx &&
795: pmap_extract(ufi.orig_map->pmap, currva, &pa)) {
796: pages[lcv] = PGO_DONTCARE;
797: continue;
798: }
799:
800: /*
801: * unmapped or center page. check if any anon at this level.
802: */
803: if (amap == NULL || anons[lcv] == NULL) {
804: pages[lcv] = NULL;
805: continue;
806: }
807:
808: /*
809: * check for present page and map if possible. re-activate it.
810: */
811:
812: pages[lcv] = PGO_DONTCARE;
813: if (lcv == centeridx) { /* save center for later! */
814: shadowed = TRUE;
815: continue;
816: }
817: anon = anons[lcv];
818: simple_lock(&anon->an_lock);
819: /* ignore loaned pages */
820: if (anon->an_page && anon->an_page->loan_count == 0 &&
821: (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
822: uvm_lock_pageq();
823: uvm_pageactivate(anon->an_page); /* reactivate */
824: uvm_unlock_pageq();
825: UVMHIST_LOG(maphist,
826: " MAPPING: n anon: pm=%p, va=0x%lx, pg=%p",
827: ufi.orig_map->pmap, currva, anon->an_page, 0);
828: uvmexp.fltnamap++;
829:
830: /*
831: * Since this isn't the page that's actually faulting,
832: * ignore pmap_enter() failures; it's not critical
833: * that we enter these right now.
834: */
835:
836: (void) pmap_enter(ufi.orig_map->pmap, currva,
837: VM_PAGE_TO_PHYS(anon->an_page),
838: (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
839: enter_prot,
840: PMAP_CANFAIL |
841: (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
842: }
843: simple_unlock(&anon->an_lock);
844: pmap_update(ufi.orig_map->pmap);
845: }
846:
847: /* locked: maps(read), amap(if there) */
848: /* (shadowed == TRUE) if there is an anon at the faulting address */
849: UVMHIST_LOG(maphist, " shadowed=%ld, will_get=%ld", shadowed,
850: (uobj && shadowed == FALSE),0,0);
851:
852: /*
853: * note that if we are really short of RAM we could sleep in the above
854: * call to pmap_enter with everything locked. bad?
855: *
856: * XXX Actually, that is bad; pmap_enter() should just fail in that
857: * XXX case. --thorpej
858: */
859:
860: /*
861: * if the desired page is not shadowed by the amap and we have a
862: * backing object, then we check to see if the backing object would
863: * prefer to handle the fault itself (rather than letting us do it
864: * with the usual pgo_get hook). the backing object signals this by
865: * providing a pgo_fault routine.
866: */
867:
868: if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) {
869: simple_lock(&uobj->vmobjlock);
870:
871: /* locked: maps(read), amap (if there), uobj */
872: result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
873: centeridx, fault_type, access_type,
874: PGO_LOCKED);
875:
876: /* locked: nothing, pgo_fault has unlocked everything */
877:
878: if (result == VM_PAGER_OK)
879: return (0); /* pgo_fault did pmap enter */
880: else if (result == VM_PAGER_REFAULT)
881: goto ReFault; /* try again! */
882: else
883: return (EACCES);
884: }
885:
886: /*
887: * now, if the desired page is not shadowed by the amap and we have
888: * a backing object that does not have a special fault routine, then
889: * we ask (with pgo_get) the object for resident pages that we care
890: * about and attempt to map them in. we do not let pgo_get block
891: * (PGO_LOCKED).
892: *
893: * ("get" has the option of doing a pmap_enter for us)
894: */
895:
896: if (uobj && shadowed == FALSE) {
897: simple_lock(&uobj->vmobjlock);
898:
899: /* locked (!shadowed): maps(read), amap (if there), uobj */
900: /*
901: * the following call to pgo_get does _not_ change locking state
902: */
903:
904: uvmexp.fltlget++;
905: gotpages = npages;
906: (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
907: (startva - ufi.entry->start),
908: pages, &gotpages, centeridx,
909: access_type & MASK(ufi.entry),
910: ufi.entry->advice, PGO_LOCKED);
911:
912: /*
913: * check for pages to map, if we got any
914: */
915:
916: uobjpage = NULL;
917:
918: if (gotpages) {
919: currva = startva;
920: for (lcv = 0 ; lcv < npages ;
921: lcv++, currva += PAGE_SIZE) {
922:
923: if (pages[lcv] == NULL ||
924: pages[lcv] == PGO_DONTCARE)
925: continue;
926:
927: KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0);
928:
929: /*
930: * if center page is resident and not
931: * PG_BUSY|PG_RELEASED then pgo_get
932: * made it PG_BUSY for us and gave
933: * us a handle to it. remember this
934: * page as "uobjpage." (for later use).
935: */
936:
937: if (lcv == centeridx) {
938: uobjpage = pages[lcv];
939: UVMHIST_LOG(maphist, " got uobjpage "
940: "(%p) with locked get",
941: uobjpage, 0,0,0);
942: continue;
943: }
944:
945: /*
946: * note: calling pgo_get with locked data
947: * structures returns us pages which are
948: * neither busy nor released, so we don't
949: * need to check for this. we can just
950: * directly enter the page (after moving it
951: * to the head of the active queue [useful?]).
952: */
953:
954: uvm_lock_pageq();
955: uvm_pageactivate(pages[lcv]); /* reactivate */
956: uvm_unlock_pageq();
957: UVMHIST_LOG(maphist,
958: " MAPPING: n obj: pm=%p, va=0x%lx, pg=%p",
959: ufi.orig_map->pmap, currva, pages[lcv], 0);
960: uvmexp.fltnomap++;
961:
962: /*
963: * Since this page isn't the page that's
964: * actually fauling, ignore pmap_enter()
965: * failures; it's not critical that we
966: * enter these right now.
967: */
968:
969: (void) pmap_enter(ufi.orig_map->pmap, currva,
970: VM_PAGE_TO_PHYS(pages[lcv]),
971: enter_prot & MASK(ufi.entry),
972: PMAP_CANFAIL |
973: (wired ? PMAP_WIRED : 0));
974:
975: /*
976: * NOTE: page can't be PG_WANTED or PG_RELEASED
977: * because we've held the lock the whole time
978: * we've had the handle.
979: */
980:
981: atomic_clearbits_int(&pages[lcv]->pg_flags,
982: PG_BUSY);
983: UVM_PAGE_OWN(pages[lcv], NULL);
984: } /* for "lcv" loop */
985: pmap_update(ufi.orig_map->pmap);
986: } /* "gotpages" != 0 */
987: /* note: object still _locked_ */
988: } else {
989: uobjpage = NULL;
990: }
991:
992: /* locked (shadowed): maps(read), amap */
993: /* locked (!shadowed): maps(read), amap(if there),
994: uobj(if !null), uobjpage(if !null) */
995:
996: /*
997: * note that at this point we are done with any front or back pages.
998: * we are now going to focus on the center page (i.e. the one we've
999: * faulted on). if we have faulted on the top (anon) layer
1000: * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1001: * not touched it yet). if we have faulted on the bottom (uobj)
1002: * layer [i.e. case 2] and the page was both present and available,
1003: * then we've got a pointer to it as "uobjpage" and we've already
1004: * made it BUSY.
1005: */
1006:
1007: /*
1008: * there are four possible cases we must address: 1A, 1B, 2A, and 2B
1009: */
1010:
1011: /*
1012: * redirect case 2: if we are not shadowed, go to case 2.
1013: */
1014:
1015: if (shadowed == FALSE)
1016: goto Case2;
1017:
1018: /* locked: maps(read), amap */
1019:
1020: /*
1021: * handle case 1: fault on an anon in our amap
1022: */
1023:
1024: anon = anons[centeridx];
1025: UVMHIST_LOG(maphist, " case 1 fault: anon=%p", anon, 0,0,0);
1026: simple_lock(&anon->an_lock);
1027:
1028: /* locked: maps(read), amap, anon */
1029:
1030: /*
1031: * no matter if we have case 1A or case 1B we are going to need to
1032: * have the anon's memory resident. ensure that now.
1033: */
1034:
1035: /*
1036: * let uvmfault_anonget do the dirty work.
1037: * if it fails (!OK) it will unlock everything for us.
1038: * if it succeeds, locks are still valid and locked.
1039: * also, if it is OK, then the anon's page is on the queues.
1040: * if the page is on loan from a uvm_object, then anonget will
1041: * lock that object for us if it does not fail.
1042: */
1043:
1044: result = uvmfault_anonget(&ufi, amap, anon);
1045: switch (result) {
1046: case VM_PAGER_OK:
1047: break;
1048:
1049: case VM_PAGER_REFAULT:
1050: goto ReFault;
1051:
1052: case VM_PAGER_ERROR:
1053: /*
1054: * An error occured while trying to bring in the
1055: * page -- this is the only error we return right
1056: * now.
1057: */
1058: return (EACCES); /* XXX */
1059:
1060: default:
1061: #ifdef DIAGNOSTIC
1062: panic("uvm_fault: uvmfault_anonget -> %d", result);
1063: #else
1064: return (EACCES);
1065: #endif
1066: }
1067:
1068: /*
1069: * uobj is non null if the page is on loan from an object (i.e. uobj)
1070: */
1071:
1072: uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1073:
1074: /* locked: maps(read), amap, anon, uobj(if one) */
1075:
1076: /*
1077: * special handling for loaned pages
1078: */
1079:
1080: if (anon->an_page->loan_count) {
1081:
1082: if ((access_type & VM_PROT_WRITE) == 0) {
1083:
1084: /*
1085: * for read faults on loaned pages we just cap the
1086: * protection at read-only.
1087: */
1088:
1089: enter_prot = enter_prot & ~VM_PROT_WRITE;
1090:
1091: } else {
1092: /*
1093: * note that we can't allow writes into a loaned page!
1094: *
1095: * if we have a write fault on a loaned page in an
1096: * anon then we need to look at the anon's ref count.
1097: * if it is greater than one then we are going to do
1098: * a normal copy-on-write fault into a new anon (this
1099: * is not a problem). however, if the reference count
1100: * is one (a case where we would normally allow a
1101: * write directly to the page) then we need to kill
1102: * the loan before we continue.
1103: */
1104:
1105: /* >1 case is already ok */
1106: if (anon->an_ref == 1) {
1107:
1108: /* get new un-owned replacement page */
1109: pg = uvm_pagealloc(NULL, 0, NULL, 0);
1110: if (pg == NULL) {
1111: uvmfault_unlockall(&ufi, amap, uobj,
1112: anon);
1113: uvm_wait("flt_noram2");
1114: goto ReFault;
1115: }
1116:
1117: /*
1118: * copy data, kill loan, and drop uobj lock
1119: * (if any)
1120: */
1121: /* copy old -> new */
1122: uvm_pagecopy(anon->an_page, pg);
1123:
1124: /* force reload */
1125: pmap_page_protect(anon->an_page,
1126: VM_PROT_NONE);
1127: uvm_lock_pageq(); /* KILL loan */
1128: if (uobj)
1129: /* if we were loaning */
1130: anon->an_page->loan_count--;
1131: anon->an_page->uanon = NULL;
1132: /* in case we owned */
1133: atomic_clearbits_int(
1134: &anon->an_page->pg_flags, PQ_ANON);
1135: uvm_pageactivate(pg);
1136: uvm_unlock_pageq();
1137: if (uobj) {
1138: simple_unlock(&uobj->vmobjlock);
1139: uobj = NULL;
1140: }
1141:
1142: /* install new page in anon */
1143: anon->an_page = pg;
1144: pg->uanon = anon;
1145: atomic_setbits_int(&pg->pg_flags, PQ_ANON);
1146: atomic_clearbits_int(&pg->pg_flags,
1147: PG_BUSY|PG_FAKE);
1148: UVM_PAGE_OWN(pg, NULL);
1149:
1150: /* done! */
1151: } /* ref == 1 */
1152: } /* write fault */
1153: } /* loan count */
1154:
1155: /*
1156: * if we are case 1B then we will need to allocate a new blank
1157: * anon to transfer the data into. note that we have a lock
1158: * on anon, so no one can busy or release the page until we are done.
1159: * also note that the ref count can't drop to zero here because
1160: * it is > 1 and we are only dropping one ref.
1161: *
1162: * in the (hopefully very rare) case that we are out of RAM we
1163: * will unlock, wait for more RAM, and refault.
1164: *
1165: * if we are out of anon VM we kill the process (XXX: could wait?).
1166: */
1167:
1168: if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) {
1169:
1170: UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1171: uvmexp.flt_acow++;
1172: oanon = anon; /* oanon = old, locked anon */
1173: anon = uvm_analloc();
1174: if (anon) {
1175: pg = uvm_pagealloc(NULL, 0, anon, 0);
1176: }
1177:
1178: /* check for out of RAM */
1179: if (anon == NULL || pg == NULL) {
1180: if (anon)
1181: uvm_anfree(anon);
1182: uvmfault_unlockall(&ufi, amap, uobj, oanon);
1183: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1184: if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1185: UVMHIST_LOG(maphist,
1186: "<- failed. out of VM",0,0,0,0);
1187: uvmexp.fltnoanon++;
1188: return (ENOMEM);
1189: }
1190:
1191: uvmexp.fltnoram++;
1192: uvm_wait("flt_noram3"); /* out of RAM, wait for more */
1193: goto ReFault;
1194: }
1195:
1196: /* got all resources, replace anon with nanon */
1197:
1198: uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */
1199: /* un-busy! new page */
1200: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE);
1201: UVM_PAGE_OWN(pg, NULL);
1202: amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1203: anon, 1);
1204:
1205: /* deref: can not drop to zero here by defn! */
1206: oanon->an_ref--;
1207:
1208: /*
1209: * note: oanon still locked. anon is _not_ locked, but we
1210: * have the sole references to in from amap which _is_ locked.
1211: * thus, no one can get at it until we are done with it.
1212: */
1213:
1214: } else {
1215:
1216: uvmexp.flt_anon++;
1217: oanon = anon; /* old, locked anon is same as anon */
1218: pg = anon->an_page;
1219: if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1220: enter_prot = enter_prot & ~VM_PROT_WRITE;
1221:
1222: }
1223:
1224: /* locked: maps(read), amap, oanon */
1225:
1226: /*
1227: * now map the page in ...
1228: * XXX: old fault unlocks object before pmap_enter. this seems
1229: * suspect since some other thread could blast the page out from
1230: * under us between the unlock and the pmap_enter.
1231: */
1232:
1233: UVMHIST_LOG(maphist, " MAPPING: anon: pm=%p, va=0x%lx, pg=%p",
1234: ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
1235: if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1236: enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1237: != 0) {
1238: /*
1239: * No need to undo what we did; we can simply think of
1240: * this as the pmap throwing away the mapping information.
1241: *
1242: * We do, however, have to go through the ReFault path,
1243: * as the map may change while we're asleep.
1244: */
1245: uvmfault_unlockall(&ufi, amap, uobj, oanon);
1246: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1247: if (uvmexp.swpgonly == uvmexp.swpages) {
1248: UVMHIST_LOG(maphist,
1249: "<- failed. out of VM",0,0,0,0);
1250: /* XXX instrumentation */
1251: return (ENOMEM);
1252: }
1253: /* XXX instrumentation */
1254: uvm_wait("flt_pmfail1");
1255: goto ReFault;
1256: }
1257:
1258: /*
1259: * ... update the page queues.
1260: */
1261:
1262: uvm_lock_pageq();
1263:
1264: if (fault_type == VM_FAULT_WIRE) {
1265: uvm_pagewire(pg);
1266:
1267: /*
1268: * since the now-wired page cannot be paged out,
1269: * release its swap resources for others to use.
1270: * since an anon with no swap cannot be PG_CLEAN,
1271: * clear its clean flag now.
1272: */
1273: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
1274: uvm_anon_dropswap(anon);
1275: } else {
1276: /* activate it */
1277: uvm_pageactivate(pg);
1278: }
1279:
1280: uvm_unlock_pageq();
1281:
1282: /*
1283: * done case 1! finish up by unlocking everything and returning success
1284: */
1285:
1286: uvmfault_unlockall(&ufi, amap, uobj, oanon);
1287: pmap_update(ufi.orig_map->pmap);
1288: return (0);
1289:
1290:
1291: Case2:
1292: /*
1293: * handle case 2: faulting on backing object or zero fill
1294: */
1295:
1296: /*
1297: * locked:
1298: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1299: */
1300:
1301: /*
1302: * note that uobjpage can not be PGO_DONTCARE at this point. we now
1303: * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1304: * have a backing object, check and see if we are going to promote
1305: * the data up to an anon during the fault.
1306: */
1307:
1308: if (uobj == NULL) {
1309: uobjpage = PGO_DONTCARE;
1310: promote = TRUE; /* always need anon here */
1311: } else {
1312: KASSERT(uobjpage != PGO_DONTCARE);
1313: promote = (access_type & VM_PROT_WRITE) &&
1314: UVM_ET_ISCOPYONWRITE(ufi.entry);
1315: }
1316: UVMHIST_LOG(maphist, " case 2 fault: promote=%ld, zfill=%ld",
1317: promote, (uobj == NULL), 0,0);
1318:
1319: /*
1320: * if uobjpage is not null then we do not need to do I/O to get the
1321: * uobjpage.
1322: *
1323: * if uobjpage is null, then we need to unlock and ask the pager to
1324: * get the data for us. once we have the data, we need to reverify
1325: * the state the world. we are currently not holding any resources.
1326: */
1327:
1328: if (uobjpage) {
1329: /* update rusage counters */
1330: curproc->p_addr->u_stats.p_ru.ru_minflt++;
1331: } else {
1332: /* update rusage counters */
1333: curproc->p_addr->u_stats.p_ru.ru_majflt++;
1334:
1335: /* locked: maps(read), amap(if there), uobj */
1336: uvmfault_unlockall(&ufi, amap, NULL, NULL);
1337: /* locked: uobj */
1338:
1339: uvmexp.fltget++;
1340: gotpages = 1;
1341: uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
1342: result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1343: 0, access_type & MASK(ufi.entry), ufi.entry->advice,
1344: PGO_SYNCIO);
1345:
1346: /* locked: uobjpage(if result OK) */
1347:
1348: /*
1349: * recover from I/O
1350: */
1351:
1352: if (result != VM_PAGER_OK) {
1353: KASSERT(result != VM_PAGER_PEND);
1354:
1355: if (result == VM_PAGER_AGAIN) {
1356: UVMHIST_LOG(maphist,
1357: " pgo_get says TRY AGAIN!",0,0,0,0);
1358: tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0);
1359: goto ReFault;
1360: }
1361:
1362: UVMHIST_LOG(maphist, "<- pgo_get failed (code %ld)",
1363: result, 0,0,0);
1364: return (EACCES); /* XXX i/o error */
1365: }
1366:
1367: /* locked: uobjpage */
1368:
1369: /*
1370: * re-verify the state of the world by first trying to relock
1371: * the maps. always relock the object.
1372: */
1373:
1374: locked = uvmfault_relock(&ufi);
1375: simple_lock(&uobj->vmobjlock);
1376:
1377: /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
1378: /* locked(!locked): uobj, uobjpage */
1379:
1380: /*
1381: * verify that the page has not be released and re-verify
1382: * that amap slot is still free. if there is a problem,
1383: * we unlock and clean up.
1384: */
1385:
1386: if ((uobjpage->pg_flags & PG_RELEASED) != 0 ||
1387: (locked && amap &&
1388: amap_lookup(&ufi.entry->aref,
1389: ufi.orig_rvaddr - ufi.entry->start))) {
1390: if (locked)
1391: uvmfault_unlockall(&ufi, amap, NULL, NULL);
1392: locked = FALSE;
1393: }
1394:
1395: /*
1396: * didn't get the lock? release the page and retry.
1397: */
1398:
1399: if (locked == FALSE) {
1400:
1401: UVMHIST_LOG(maphist,
1402: " wasn't able to relock after fault: retry",
1403: 0,0,0,0);
1404: if (uobjpage->pg_flags & PG_WANTED)
1405: /* still holding object lock */
1406: wakeup(uobjpage);
1407:
1408: if (uobjpage->pg_flags & PG_RELEASED) {
1409: uvmexp.fltpgrele++;
1410: KASSERT(uobj->pgops->pgo_releasepg != NULL);
1411:
1412: /* frees page */
1413: if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
1414: /* unlock if still alive */
1415: simple_unlock(&uobj->vmobjlock);
1416: goto ReFault;
1417: }
1418:
1419: uvm_lock_pageq();
1420: /* make sure it is in queues */
1421: uvm_pageactivate(uobjpage);
1422:
1423: uvm_unlock_pageq();
1424: atomic_clearbits_int(&uobjpage->pg_flags,
1425: PG_BUSY|PG_WANTED);
1426: UVM_PAGE_OWN(uobjpage, NULL);
1427: simple_unlock(&uobj->vmobjlock);
1428: goto ReFault;
1429:
1430: }
1431:
1432: /*
1433: * we have the data in uobjpage which is PG_BUSY and
1434: * !PG_RELEASED. we are holding object lock (so the page
1435: * can't be released on us).
1436: */
1437:
1438: /* locked: maps(read), amap(if !null), uobj, uobjpage */
1439: }
1440:
1441: /*
1442: * locked:
1443: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1444: */
1445:
1446: /*
1447: * notes:
1448: * - at this point uobjpage can not be NULL
1449: * - at this point uobjpage can not be PG_RELEASED (since we checked
1450: * for it above)
1451: * - at this point uobjpage could be PG_WANTED (handle later)
1452: */
1453:
1454: if (promote == FALSE) {
1455:
1456: /*
1457: * we are not promoting. if the mapping is COW ensure that we
1458: * don't give more access than we should (e.g. when doing a read
1459: * fault on a COPYONWRITE mapping we want to map the COW page in
1460: * R/O even though the entry protection could be R/W).
1461: *
1462: * set "pg" to the page we want to map in (uobjpage, usually)
1463: */
1464:
1465: uvmexp.flt_obj++;
1466: if (UVM_ET_ISCOPYONWRITE(ufi.entry))
1467: enter_prot &= ~VM_PROT_WRITE;
1468: pg = uobjpage; /* map in the actual object */
1469:
1470: /* assert(uobjpage != PGO_DONTCARE) */
1471:
1472: /*
1473: * we are faulting directly on the page. be careful
1474: * about writing to loaned pages...
1475: */
1476: if (uobjpage->loan_count) {
1477:
1478: if ((access_type & VM_PROT_WRITE) == 0) {
1479: /* read fault: cap the protection at readonly */
1480: /* cap! */
1481: enter_prot = enter_prot & ~VM_PROT_WRITE;
1482: } else {
1483: /* write fault: must break the loan here */
1484:
1485: /* alloc new un-owned page */
1486: pg = uvm_pagealloc(NULL, 0, NULL, 0);
1487:
1488: if (pg == NULL) {
1489: /*
1490: * drop ownership of page, it can't
1491: * be released
1492: */
1493: if (uobjpage->pg_flags & PG_WANTED)
1494: wakeup(uobjpage);
1495: atomic_clearbits_int(
1496: &uobjpage->pg_flags,
1497: PG_BUSY|PG_WANTED);
1498: UVM_PAGE_OWN(uobjpage, NULL);
1499:
1500: uvm_lock_pageq();
1501: /* activate: we will need it later */
1502: uvm_pageactivate(uobjpage);
1503:
1504: uvm_unlock_pageq();
1505: uvmfault_unlockall(&ufi, amap, uobj,
1506: NULL);
1507: UVMHIST_LOG(maphist,
1508: " out of RAM breaking loan, waiting",
1509: 0,0,0,0);
1510: uvmexp.fltnoram++;
1511: uvm_wait("flt_noram4");
1512: goto ReFault;
1513: }
1514:
1515: /*
1516: * copy the data from the old page to the new
1517: * one and clear the fake/clean flags on the
1518: * new page (keep it busy). force a reload
1519: * of the old page by clearing it from all
1520: * pmaps. then lock the page queues to
1521: * rename the pages.
1522: */
1523: uvm_pagecopy(uobjpage, pg); /* old -> new */
1524: atomic_clearbits_int(&pg->pg_flags,
1525: PG_FAKE|PG_CLEAN);
1526: pmap_page_protect(uobjpage, VM_PROT_NONE);
1527: if (uobjpage->pg_flags & PG_WANTED)
1528: wakeup(uobjpage);
1529: /* uobj still locked */
1530: atomic_clearbits_int(&uobjpage->pg_flags,
1531: PG_BUSY|PG_WANTED);
1532: UVM_PAGE_OWN(uobjpage, NULL);
1533:
1534: uvm_lock_pageq();
1535: uoff = uobjpage->offset;
1536: /* remove old page */
1537: uvm_pagerealloc(uobjpage, NULL, 0);
1538:
1539: /*
1540: * at this point we have absolutely no
1541: * control over uobjpage
1542: */
1543: /* install new page */
1544: uvm_pagerealloc(pg, uobj, uoff);
1545: uvm_unlock_pageq();
1546:
1547: /*
1548: * done! loan is broken and "pg" is
1549: * PG_BUSY. it can now replace uobjpage.
1550: */
1551:
1552: uobjpage = pg;
1553:
1554: } /* write fault case */
1555: } /* if loan_count */
1556:
1557: } else {
1558:
1559: /*
1560: * if we are going to promote the data to an anon we
1561: * allocate a blank anon here and plug it into our amap.
1562: */
1563: #ifdef DIAGNOSTIC
1564: if (amap == NULL)
1565: panic("uvm_fault: want to promote data, but no anon");
1566: #endif
1567:
1568: anon = uvm_analloc();
1569: if (anon) {
1570: /*
1571: * In `Fill in data...' below, if
1572: * uobjpage == PGO_DONTCARE, we want
1573: * a zero'd, dirty page, so have
1574: * uvm_pagealloc() do that for us.
1575: */
1576: pg = uvm_pagealloc(NULL, 0, anon,
1577: (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);
1578: }
1579:
1580: /*
1581: * out of memory resources?
1582: */
1583: if (anon == NULL || pg == NULL) {
1584:
1585: /*
1586: * arg! must unbusy our page and fail or sleep.
1587: */
1588: if (uobjpage != PGO_DONTCARE) {
1589: if (uobjpage->pg_flags & PG_WANTED)
1590: /* still holding object lock */
1591: wakeup(uobjpage);
1592:
1593: uvm_lock_pageq();
1594: uvm_pageactivate(uobjpage);
1595: uvm_unlock_pageq();
1596: atomic_clearbits_int(&uobjpage->pg_flags,
1597: PG_BUSY|PG_WANTED);
1598: UVM_PAGE_OWN(uobjpage, NULL);
1599: }
1600:
1601: /* unlock and fail ... */
1602: uvmfault_unlockall(&ufi, amap, uobj, NULL);
1603: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1604: if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1605: UVMHIST_LOG(maphist, " promote: out of VM",
1606: 0,0,0,0);
1607: uvmexp.fltnoanon++;
1608: return (ENOMEM);
1609: }
1610:
1611: UVMHIST_LOG(maphist, " out of RAM, waiting for more",
1612: 0,0,0,0);
1613: uvm_anfree(anon);
1614: uvmexp.fltnoram++;
1615: uvm_wait("flt_noram5");
1616: goto ReFault;
1617: }
1618:
1619: /*
1620: * fill in the data
1621: */
1622:
1623: if (uobjpage != PGO_DONTCARE) {
1624: uvmexp.flt_prcopy++;
1625: /* copy page [pg now dirty] */
1626: uvm_pagecopy(uobjpage, pg);
1627:
1628: /*
1629: * promote to shared amap? make sure all sharing
1630: * procs see it
1631: */
1632: if ((amap_flags(amap) & AMAP_SHARED) != 0) {
1633: pmap_page_protect(uobjpage, VM_PROT_NONE);
1634: }
1635:
1636: /*
1637: * dispose of uobjpage. it can't be PG_RELEASED
1638: * since we still hold the object lock.
1639: * drop handle to uobj as well.
1640: */
1641:
1642: if (uobjpage->pg_flags & PG_WANTED)
1643: /* still have the obj lock */
1644: wakeup(uobjpage);
1645: atomic_clearbits_int(&uobjpage->pg_flags,
1646: PG_BUSY|PG_WANTED);
1647: UVM_PAGE_OWN(uobjpage, NULL);
1648: uvm_lock_pageq();
1649: uvm_pageactivate(uobjpage);
1650: uvm_unlock_pageq();
1651: simple_unlock(&uobj->vmobjlock);
1652: uobj = NULL;
1653:
1654: UVMHIST_LOG(maphist,
1655: " promote uobjpage %p to anon/page %p/%p",
1656: uobjpage, anon, pg, 0);
1657:
1658: } else {
1659: uvmexp.flt_przero++;
1660: /*
1661: * Page is zero'd and marked dirty by uvm_pagealloc()
1662: * above.
1663: */
1664: UVMHIST_LOG(maphist," zero fill anon/page %p/%p",
1665: anon, pg, 0, 0);
1666: }
1667:
1668: amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1669: anon, 0);
1670: }
1671:
1672: /*
1673: * locked:
1674: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1675: *
1676: * note: pg is either the uobjpage or the new page in the new anon
1677: */
1678:
1679: /*
1680: * all resources are present. we can now map it in and free our
1681: * resources.
1682: */
1683:
1684: UVMHIST_LOG(maphist,
1685: " MAPPING: case2: pm=%p, va=0x%lx, pg=%p, promote=%ld",
1686: ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
1687: if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1688: enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1689: != 0) {
1690:
1691: /*
1692: * No need to undo what we did; we can simply think of
1693: * this as the pmap throwing away the mapping information.
1694: *
1695: * We do, however, have to go through the ReFault path,
1696: * as the map may change while we're asleep.
1697: */
1698:
1699: if (pg->pg_flags & PG_WANTED)
1700: wakeup(pg); /* lock still held */
1701:
1702: /*
1703: * note that pg can't be PG_RELEASED since we did not drop
1704: * the object lock since the last time we checked.
1705: */
1706:
1707: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
1708: UVM_PAGE_OWN(pg, NULL);
1709: uvmfault_unlockall(&ufi, amap, uobj, NULL);
1710: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1711: if (uvmexp.swpgonly == uvmexp.swpages) {
1712: UVMHIST_LOG(maphist,
1713: "<- failed. out of VM",0,0,0,0);
1714: /* XXX instrumentation */
1715: return (ENOMEM);
1716: }
1717: /* XXX instrumentation */
1718: uvm_wait("flt_pmfail2");
1719: goto ReFault;
1720: }
1721:
1722: uvm_lock_pageq();
1723:
1724: if (fault_type == VM_FAULT_WIRE) {
1725: uvm_pagewire(pg);
1726: if (pg->pg_flags & PQ_AOBJ) {
1727:
1728: /*
1729: * since the now-wired page cannot be paged out,
1730: * release its swap resources for others to use.
1731: * since an aobj page with no swap cannot be PG_CLEAN,
1732: * clear its clean flag now.
1733: */
1734: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
1735: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
1736: }
1737: } else {
1738: /* activate it */
1739: uvm_pageactivate(pg);
1740: }
1741: uvm_unlock_pageq();
1742:
1743: if (pg->pg_flags & PG_WANTED)
1744: wakeup(pg); /* lock still held */
1745:
1746: /*
1747: * note that pg can't be PG_RELEASED since we did not drop the object
1748: * lock since the last time we checked.
1749: */
1750:
1751: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
1752: UVM_PAGE_OWN(pg, NULL);
1753: uvmfault_unlockall(&ufi, amap, uobj, NULL);
1754: pmap_update(ufi.orig_map->pmap);
1755:
1756: UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
1757: return (0);
1758: }
1759:
1760:
1761: /*
1762: * uvm_fault_wire: wire down a range of virtual addresses in a map.
1763: *
1764: * => map may be read-locked by caller, but MUST NOT be write-locked.
1765: * => if map is read-locked, any operations which may cause map to
1766: * be write-locked in uvm_fault() must be taken care of by
1767: * the caller. See uvm_map_pageable().
1768: */
1769:
1770: int
1771: uvm_fault_wire(map, start, end, access_type)
1772: vm_map_t map;
1773: vaddr_t start, end;
1774: vm_prot_t access_type;
1775: {
1776: vaddr_t va;
1777: pmap_t pmap;
1778: int rv;
1779:
1780: pmap = vm_map_pmap(map);
1781:
1782: /*
1783: * now fault it in a page at a time. if the fault fails then we have
1784: * to undo what we have done. note that in uvm_fault VM_PROT_NONE
1785: * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1786: */
1787:
1788: for (va = start ; va < end ; va += PAGE_SIZE) {
1789: rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type);
1790: if (rv) {
1791: if (va != start) {
1792: uvm_fault_unwire(map, start, va);
1793: }
1794: return (rv);
1795: }
1796: }
1797:
1798: return (0);
1799: }
1800:
1801: /*
1802: * uvm_fault_unwire(): unwire range of virtual space.
1803: */
1804:
1805: void
1806: uvm_fault_unwire(map, start, end)
1807: vm_map_t map;
1808: vaddr_t start, end;
1809: {
1810:
1811: vm_map_lock_read(map);
1812: uvm_fault_unwire_locked(map, start, end);
1813: vm_map_unlock_read(map);
1814: }
1815:
1816: /*
1817: * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
1818: *
1819: * => map must be at least read-locked.
1820: */
1821:
1822: void
1823: uvm_fault_unwire_locked(map, start, end)
1824: vm_map_t map;
1825: vaddr_t start, end;
1826: {
1827: vm_map_entry_t entry;
1828: pmap_t pmap = vm_map_pmap(map);
1829: vaddr_t va;
1830: paddr_t pa;
1831: struct vm_page *pg;
1832:
1833: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1834:
1835: /*
1836: * we assume that the area we are unwiring has actually been wired
1837: * in the first place. this means that we should be able to extract
1838: * the PAs from the pmap. we also lock out the page daemon so that
1839: * we can call uvm_pageunwire.
1840: */
1841:
1842: uvm_lock_pageq();
1843:
1844: /*
1845: * find the beginning map entry for the region.
1846: */
1847: KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1848: if (uvm_map_lookup_entry(map, start, &entry) == FALSE)
1849: panic("uvm_fault_unwire_locked: address not in map");
1850:
1851: for (va = start; va < end ; va += PAGE_SIZE) {
1852: if (pmap_extract(pmap, va, &pa) == FALSE)
1853: continue;
1854:
1855: /*
1856: * find the map entry for the current address.
1857: */
1858: KASSERT(va >= entry->start);
1859: while (va >= entry->end) {
1860: KASSERT(entry->next != &map->header &&
1861: entry->next->start <= entry->end);
1862: entry = entry->next;
1863: }
1864:
1865: /*
1866: * if the entry is no longer wired, tell the pmap.
1867: */
1868: if (VM_MAPENT_ISWIRED(entry) == 0)
1869: pmap_unwire(pmap, va);
1870:
1871: pg = PHYS_TO_VM_PAGE(pa);
1872: if (pg)
1873: uvm_pageunwire(pg);
1874: }
1875:
1876: uvm_unlock_pageq();
1877: }