Annotation of sys/uvm/uvm_fault.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: uvm_fault.c,v 1.49 2007/06/18 21:51:15 pedro Exp $ */
! 2: /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
! 3:
! 4: /*
! 5: *
! 6: * Copyright (c) 1997 Charles D. Cranor and Washington University.
! 7: * All rights reserved.
! 8: *
! 9: * Redistribution and use in source and binary forms, with or without
! 10: * modification, are permitted provided that the following conditions
! 11: * are met:
! 12: * 1. Redistributions of source code must retain the above copyright
! 13: * notice, this list of conditions and the following disclaimer.
! 14: * 2. Redistributions in binary form must reproduce the above copyright
! 15: * notice, this list of conditions and the following disclaimer in the
! 16: * documentation and/or other materials provided with the distribution.
! 17: * 3. All advertising materials mentioning features or use of this software
! 18: * must display the following acknowledgement:
! 19: * This product includes software developed by Charles D. Cranor and
! 20: * Washington University.
! 21: * 4. The name of the author may not be used to endorse or promote products
! 22: * derived from this software without specific prior written permission.
! 23: *
! 24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
! 25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
! 26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
! 27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
! 28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
! 29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
! 30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
! 31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
! 32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
! 33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
! 34: *
! 35: * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
! 36: */
! 37:
! 38: /*
! 39: * uvm_fault.c: fault handler
! 40: */
! 41:
! 42: #include <sys/param.h>
! 43: #include <sys/systm.h>
! 44: #include <sys/kernel.h>
! 45: #include <sys/proc.h>
! 46: #include <sys/malloc.h>
! 47: #include <sys/mman.h>
! 48: #include <sys/user.h>
! 49:
! 50: #include <uvm/uvm.h>
! 51:
! 52: /*
! 53: *
! 54: * a word on page faults:
! 55: *
! 56: * types of page faults we handle:
! 57: *
! 58: * CASE 1: upper layer faults CASE 2: lower layer faults
! 59: *
! 60: * CASE 1A CASE 1B CASE 2A CASE 2B
! 61: * read/write1 write>1 read/write +-cow_write/zero
! 62: * | | | |
! 63: * +--|--+ +--|--+ +-----+ + | + | +-----+
! 64: * amap | V | | ----------->new| | | | ^ |
! 65: * +-----+ +-----+ +-----+ + | + | +--|--+
! 66: * | | |
! 67: * +-----+ +-----+ +--|--+ | +--|--+
! 68: * uobj | d/c | | d/c | | V | +----| |
! 69: * +-----+ +-----+ +-----+ +-----+
! 70: *
! 71: * d/c = don't care
! 72: *
! 73: * case [0]: layerless fault
! 74: * no amap or uobj is present. this is an error.
! 75: *
! 76: * case [1]: upper layer fault [anon active]
! 77: * 1A: [read] or [write with anon->an_ref == 1]
! 78: * I/O takes place in top level anon and uobj is not touched.
! 79: * 1B: [write with anon->an_ref > 1]
! 80: * new anon is alloc'd and data is copied off ["COW"]
! 81: *
! 82: * case [2]: lower layer fault [uobj]
! 83: * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
! 84: * I/O takes place directly in object.
! 85: * 2B: [write to copy_on_write] or [read on NULL uobj]
! 86: * data is "promoted" from uobj to a new anon.
! 87: * if uobj is null, then we zero fill.
! 88: *
! 89: * we follow the standard UVM locking protocol ordering:
! 90: *
! 91: * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
! 92: * we hold a PG_BUSY page if we unlock for I/O
! 93: *
! 94: *
! 95: * the code is structured as follows:
! 96: *
! 97: * - init the "IN" params in the ufi structure
! 98: * ReFault:
! 99: * - do lookups [locks maps], check protection, handle needs_copy
! 100: * - check for case 0 fault (error)
! 101: * - establish "range" of fault
! 102: * - if we have an amap lock it and extract the anons
! 103: * - if sequential advice deactivate pages behind us
! 104: * - at the same time check pmap for unmapped areas and anon for pages
! 105: * that we could map in (and do map it if found)
! 106: * - check object for resident pages that we could map in
! 107: * - if (case 2) goto Case2
! 108: * - >>> handle case 1
! 109: * - ensure source anon is resident in RAM
! 110: * - if case 1B alloc new anon and copy from source
! 111: * - map the correct page in
! 112: * Case2:
! 113: * - >>> handle case 2
! 114: * - ensure source page is resident (if uobj)
! 115: * - if case 2B alloc new anon and copy from source (could be zero
! 116: * fill if uobj == NULL)
! 117: * - map the correct page in
! 118: * - done!
! 119: *
! 120: * note on paging:
! 121: * if we have to do I/O we place a PG_BUSY page in the correct object,
! 122: * unlock everything, and do the I/O. when I/O is done we must reverify
! 123: * the state of the world before assuming that our data structures are
! 124: * valid. [because mappings could change while the map is unlocked]
! 125: *
! 126: * alternative 1: unbusy the page in question and restart the page fault
! 127: * from the top (ReFault). this is easy but does not take advantage
! 128: * of the information that we already have from our previous lookup,
! 129: * although it is possible that the "hints" in the vm_map will help here.
! 130: *
! 131: * alternative 2: the system already keeps track of a "version" number of
! 132: * a map. [i.e. every time you write-lock a map (e.g. to change a
! 133: * mapping) you bump the version number up by one...] so, we can save
! 134: * the version number of the map before we release the lock and start I/O.
! 135: * then when I/O is done we can relock and check the version numbers
! 136: * to see if anything changed. this might save us some over 1 because
! 137: * we don't have to unbusy the page and may be less compares(?).
! 138: *
! 139: * alternative 3: put in backpointers or a way to "hold" part of a map
! 140: * in place while I/O is in progress. this could be complex to
! 141: * implement (especially with structures like amap that can be referenced
! 142: * by multiple map entries, and figuring out what should wait could be
! 143: * complex as well...).
! 144: *
! 145: * given that we are not currently multiprocessor or multithreaded we might
! 146: * as well choose alternative 2 now. maybe alternative 3 would be useful
! 147: * in the future. XXX keep in mind for future consideration//rechecking.
! 148: */
! 149:
! 150: /*
! 151: * local data structures
! 152: */
! 153:
! 154: struct uvm_advice {
! 155: int advice;
! 156: int nback;
! 157: int nforw;
! 158: };
! 159:
! 160: /*
! 161: * page range array:
! 162: * note: index in array must match "advice" value
! 163: * XXX: borrowed numbers from freebsd. do they work well for us?
! 164: */
! 165:
! 166: static struct uvm_advice uvmadvice[] = {
! 167: { MADV_NORMAL, 3, 4 },
! 168: { MADV_RANDOM, 0, 0 },
! 169: { MADV_SEQUENTIAL, 8, 7},
! 170: };
! 171:
! 172: #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */
! 173:
! 174: /*
! 175: * private prototypes
! 176: */
! 177:
! 178: static void uvmfault_amapcopy(struct uvm_faultinfo *);
! 179: static __inline void uvmfault_anonflush(struct vm_anon **, int);
! 180:
! 181: /*
! 182: * inline functions
! 183: */
! 184:
! 185: /*
! 186: * uvmfault_anonflush: try and deactivate pages in specified anons
! 187: *
! 188: * => does not have to deactivate page if it is busy
! 189: */
! 190:
! 191: static __inline void
! 192: uvmfault_anonflush(anons, n)
! 193: struct vm_anon **anons;
! 194: int n;
! 195: {
! 196: int lcv;
! 197: struct vm_page *pg;
! 198:
! 199: for (lcv = 0 ; lcv < n ; lcv++) {
! 200: if (anons[lcv] == NULL)
! 201: continue;
! 202: simple_lock(&anons[lcv]->an_lock);
! 203: pg = anons[lcv]->an_page;
! 204: if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) {
! 205: uvm_lock_pageq();
! 206: if (pg->wire_count == 0) {
! 207: #ifdef UBC
! 208: pmap_clear_reference(pg);
! 209: #else
! 210: pmap_page_protect(pg, VM_PROT_NONE);
! 211: #endif
! 212: uvm_pagedeactivate(pg);
! 213: }
! 214: uvm_unlock_pageq();
! 215: }
! 216: simple_unlock(&anons[lcv]->an_lock);
! 217: }
! 218: }
! 219:
! 220: /*
! 221: * normal functions
! 222: */
! 223:
! 224: /*
! 225: * uvmfault_amapcopy: clear "needs_copy" in a map.
! 226: *
! 227: * => called with VM data structures unlocked (usually, see below)
! 228: * => we get a write lock on the maps and clear needs_copy for a VA
! 229: * => if we are out of RAM we sleep (waiting for more)
! 230: */
! 231:
! 232: static void
! 233: uvmfault_amapcopy(ufi)
! 234: struct uvm_faultinfo *ufi;
! 235: {
! 236:
! 237: /*
! 238: * while we haven't done the job
! 239: */
! 240:
! 241: while (1) {
! 242:
! 243: /*
! 244: * no mapping? give up.
! 245: */
! 246:
! 247: if (uvmfault_lookup(ufi, TRUE) == FALSE)
! 248: return;
! 249:
! 250: /*
! 251: * copy if needed.
! 252: */
! 253:
! 254: if (UVM_ET_ISNEEDSCOPY(ufi->entry))
! 255: amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE,
! 256: ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
! 257:
! 258: /*
! 259: * didn't work? must be out of RAM. unlock and sleep.
! 260: */
! 261:
! 262: if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
! 263: uvmfault_unlockmaps(ufi, TRUE);
! 264: uvm_wait("fltamapcopy");
! 265: continue;
! 266: }
! 267:
! 268: /*
! 269: * got it! unlock and return.
! 270: */
! 271:
! 272: uvmfault_unlockmaps(ufi, TRUE);
! 273: return;
! 274: }
! 275: /*NOTREACHED*/
! 276: }
! 277:
! 278: /*
! 279: * uvmfault_anonget: get data in an anon into a non-busy, non-released
! 280: * page in that anon.
! 281: *
! 282: * => maps, amap, and anon locked by caller.
! 283: * => if we fail (result != VM_PAGER_OK) we unlock everything.
! 284: * => if we are successful, we return with everything still locked.
! 285: * => we don't move the page on the queues [gets moved later]
! 286: * => if we allocate a new page [we_own], it gets put on the queues.
! 287: * either way, the result is that the page is on the queues at return time
! 288: * => for pages which are on loan from a uvm_object (and thus are not
! 289: * owned by the anon): if successful, we return with the owning object
! 290: * locked. the caller must unlock this object when it unlocks everything
! 291: * else.
! 292: */
! 293:
! 294: int
! 295: uvmfault_anonget(ufi, amap, anon)
! 296: struct uvm_faultinfo *ufi;
! 297: struct vm_amap *amap;
! 298: struct vm_anon *anon;
! 299: {
! 300: boolean_t we_own; /* we own anon's page? */
! 301: boolean_t locked; /* did we relock? */
! 302: struct vm_page *pg;
! 303: int result;
! 304: UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
! 305:
! 306: result = 0; /* XXX shut up gcc */
! 307: uvmexp.fltanget++;
! 308: /* bump rusage counters */
! 309: if (anon->an_page)
! 310: curproc->p_addr->u_stats.p_ru.ru_minflt++;
! 311: else
! 312: curproc->p_addr->u_stats.p_ru.ru_majflt++;
! 313:
! 314: /*
! 315: * loop until we get it, or fail.
! 316: */
! 317:
! 318: while (1) {
! 319:
! 320: we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
! 321: pg = anon->an_page;
! 322:
! 323: /*
! 324: * if there is a resident page and it is loaned, then anon
! 325: * may not own it. call out to uvm_anon_lockpage() to ensure
! 326: * the real owner of the page has been identified and locked.
! 327: */
! 328:
! 329: if (pg && pg->loan_count)
! 330: pg = uvm_anon_lockloanpg(anon);
! 331:
! 332: /*
! 333: * page there? make sure it is not busy/released.
! 334: */
! 335:
! 336: if (pg) {
! 337:
! 338: /*
! 339: * at this point, if the page has a uobject [meaning
! 340: * we have it on loan], then that uobject is locked
! 341: * by us! if the page is busy, we drop all the
! 342: * locks (including uobject) and try again.
! 343: */
! 344:
! 345: if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
! 346: UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
! 347: return (VM_PAGER_OK);
! 348: }
! 349: atomic_setbits_int(&pg->pg_flags, PG_WANTED);
! 350: uvmexp.fltpgwait++;
! 351:
! 352: /*
! 353: * the last unlock must be an atomic unlock+wait on
! 354: * the owner of page
! 355: */
! 356: if (pg->uobject) { /* owner is uobject ? */
! 357: uvmfault_unlockall(ufi, amap, NULL, anon);
! 358: UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
! 359: 0,0,0);
! 360: UVM_UNLOCK_AND_WAIT(pg,
! 361: &pg->uobject->vmobjlock,
! 362: FALSE, "anonget1",0);
! 363: } else {
! 364: /* anon owns page */
! 365: uvmfault_unlockall(ufi, amap, NULL, NULL);
! 366: UVMHIST_LOG(maphist, " unlock+wait on anon",0,
! 367: 0,0,0);
! 368: UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
! 369: "anonget2",0);
! 370: }
! 371: /* ready to relock and try again */
! 372:
! 373: } else {
! 374:
! 375: /*
! 376: * no page, we must try and bring it in.
! 377: */
! 378: pg = uvm_pagealloc(NULL, 0, anon, 0);
! 379:
! 380: if (pg == NULL) { /* out of RAM. */
! 381:
! 382: uvmfault_unlockall(ufi, amap, NULL, anon);
! 383: uvmexp.fltnoram++;
! 384: UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
! 385: 0,0,0);
! 386: uvm_wait("flt_noram1");
! 387: /* ready to relock and try again */
! 388:
! 389: } else {
! 390:
! 391: /* we set the PG_BUSY bit */
! 392: we_own = TRUE;
! 393: uvmfault_unlockall(ufi, amap, NULL, anon);
! 394:
! 395: /*
! 396: * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
! 397: * page into the uvm_swap_get function with
! 398: * all data structures unlocked. note that
! 399: * it is ok to read an_swslot here because
! 400: * we hold PG_BUSY on the page.
! 401: */
! 402: uvmexp.pageins++;
! 403: result = uvm_swap_get(pg, anon->an_swslot,
! 404: PGO_SYNCIO);
! 405:
! 406: /*
! 407: * we clean up after the i/o below in the
! 408: * "we_own" case
! 409: */
! 410: /* ready to relock and try again */
! 411: }
! 412: }
! 413:
! 414: /*
! 415: * now relock and try again
! 416: */
! 417:
! 418: locked = uvmfault_relock(ufi);
! 419: if (locked || we_own)
! 420: simple_lock(&anon->an_lock);
! 421:
! 422: /*
! 423: * if we own the page (i.e. we set PG_BUSY), then we need
! 424: * to clean up after the I/O. there are three cases to
! 425: * consider:
! 426: * [1] page released during I/O: free anon and ReFault.
! 427: * [2] I/O not OK. free the page and cause the fault
! 428: * to fail.
! 429: * [3] I/O OK! activate the page and sync with the
! 430: * non-we_own case (i.e. drop anon lock if not locked).
! 431: */
! 432:
! 433: if (we_own) {
! 434:
! 435: if (pg->pg_flags & PG_WANTED) {
! 436: /* still holding object lock */
! 437: wakeup(pg);
! 438: }
! 439: /* un-busy! */
! 440: atomic_clearbits_int(&pg->pg_flags,
! 441: PG_WANTED|PG_BUSY|PG_FAKE);
! 442: UVM_PAGE_OWN(pg, NULL);
! 443:
! 444: /*
! 445: * if we were RELEASED during I/O, then our anon is
! 446: * no longer part of an amap. we need to free the
! 447: * anon and try again.
! 448: */
! 449: if (pg->pg_flags & PG_RELEASED) {
! 450: pmap_page_protect(pg, VM_PROT_NONE);
! 451: simple_unlock(&anon->an_lock);
! 452: uvm_anfree(anon); /* frees page for us */
! 453: if (locked)
! 454: uvmfault_unlockall(ufi, amap, NULL,
! 455: NULL);
! 456: uvmexp.fltpgrele++;
! 457: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
! 458: return (VM_PAGER_REFAULT); /* refault! */
! 459: }
! 460:
! 461: if (result != VM_PAGER_OK) {
! 462: KASSERT(result != VM_PAGER_PEND);
! 463:
! 464: /* remove page from anon */
! 465: anon->an_page = NULL;
! 466:
! 467: /*
! 468: * remove the swap slot from the anon
! 469: * and mark the anon as having no real slot.
! 470: * don't free the swap slot, thus preventing
! 471: * it from being used again.
! 472: */
! 473: uvm_swap_markbad(anon->an_swslot, 1);
! 474: anon->an_swslot = SWSLOT_BAD;
! 475:
! 476: /*
! 477: * note: page was never !PG_BUSY, so it
! 478: * can't be mapped and thus no need to
! 479: * pmap_page_protect it...
! 480: */
! 481: uvm_lock_pageq();
! 482: uvm_pagefree(pg);
! 483: uvm_unlock_pageq();
! 484:
! 485: if (locked)
! 486: uvmfault_unlockall(ufi, amap, NULL,
! 487: anon);
! 488: else
! 489: simple_unlock(&anon->an_lock);
! 490: UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
! 491: return (VM_PAGER_ERROR);
! 492: }
! 493:
! 494: /*
! 495: * must be OK, clear modify (already PG_CLEAN)
! 496: * and activate
! 497: */
! 498: pmap_clear_modify(pg);
! 499: uvm_lock_pageq();
! 500: uvm_pageactivate(pg);
! 501: uvm_unlock_pageq();
! 502: if (!locked)
! 503: simple_unlock(&anon->an_lock);
! 504: }
! 505:
! 506: /*
! 507: * we were not able to relock. restart fault.
! 508: */
! 509:
! 510: if (!locked) {
! 511: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
! 512: return (VM_PAGER_REFAULT);
! 513: }
! 514:
! 515: /*
! 516: * verify no one has touched the amap and moved the anon on us.
! 517: */
! 518:
! 519: if (ufi != NULL &&
! 520: amap_lookup(&ufi->entry->aref,
! 521: ufi->orig_rvaddr - ufi->entry->start) != anon) {
! 522:
! 523: uvmfault_unlockall(ufi, amap, NULL, anon);
! 524: UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
! 525: return (VM_PAGER_REFAULT);
! 526: }
! 527:
! 528: /*
! 529: * try it again!
! 530: */
! 531:
! 532: uvmexp.fltanretry++;
! 533: continue;
! 534:
! 535: } /* while (1) */
! 536:
! 537: /*NOTREACHED*/
! 538: }
! 539:
! 540: /*
! 541: * F A U L T - m a i n e n t r y p o i n t
! 542: */
! 543:
! 544: /*
! 545: * uvm_fault: page fault handler
! 546: *
! 547: * => called from MD code to resolve a page fault
! 548: * => VM data structures usually should be unlocked. however, it is
! 549: * possible to call here with the main map locked if the caller
! 550: * gets a write lock, sets it recursive, and then calls us (c.f.
! 551: * uvm_map_pageable). this should be avoided because it keeps
! 552: * the map locked off during I/O.
! 553: */
! 554:
! 555: #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
! 556: ~VM_PROT_WRITE : VM_PROT_ALL)
! 557:
! 558: int
! 559: uvm_fault(orig_map, vaddr, fault_type, access_type)
! 560: vm_map_t orig_map;
! 561: vaddr_t vaddr;
! 562: vm_fault_t fault_type;
! 563: vm_prot_t access_type;
! 564: {
! 565: struct uvm_faultinfo ufi;
! 566: vm_prot_t enter_prot;
! 567: boolean_t wired, narrow, promote, locked, shadowed;
! 568: int npages, nback, nforw, centeridx, result, lcv, gotpages;
! 569: vaddr_t startva, currva;
! 570: voff_t uoff;
! 571: paddr_t pa;
! 572: struct vm_amap *amap;
! 573: struct uvm_object *uobj;
! 574: struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
! 575: struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
! 576: UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
! 577:
! 578: UVMHIST_LOG(maphist, "(map=%p, vaddr=0x%lx, ft=%ld, at=%ld)",
! 579: orig_map, vaddr, fault_type, access_type);
! 580:
! 581: anon = NULL;
! 582: pg = NULL;
! 583:
! 584: uvmexp.faults++; /* XXX: locking? */
! 585:
! 586: /*
! 587: * init the IN parameters in the ufi
! 588: */
! 589:
! 590: ufi.orig_map = orig_map;
! 591: ufi.orig_rvaddr = trunc_page(vaddr);
! 592: ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
! 593: if (fault_type == VM_FAULT_WIRE)
! 594: narrow = TRUE; /* don't look for neighborhood
! 595: * pages on wire */
! 596: else
! 597: narrow = FALSE; /* normal fault */
! 598:
! 599: /*
! 600: * "goto ReFault" means restart the page fault from ground zero.
! 601: */
! 602: ReFault:
! 603:
! 604: /*
! 605: * lookup and lock the maps
! 606: */
! 607:
! 608: if (uvmfault_lookup(&ufi, FALSE) == FALSE) {
! 609: UVMHIST_LOG(maphist, "<- no mapping @ 0x%lx", vaddr, 0,0,0);
! 610: return (EFAULT);
! 611: }
! 612: /* locked: maps(read) */
! 613:
! 614: #ifdef DIAGNOSTIC
! 615: if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0)
! 616: panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)",
! 617: ufi.map, vaddr);
! 618: #endif
! 619:
! 620: /*
! 621: * check protection
! 622: */
! 623:
! 624: if ((ufi.entry->protection & access_type) != access_type) {
! 625: UVMHIST_LOG(maphist,
! 626: "<- protection failure (prot=0x%lx, access=0x%lx)",
! 627: ufi.entry->protection, access_type, 0, 0);
! 628: uvmfault_unlockmaps(&ufi, FALSE);
! 629: return (EACCES);
! 630: }
! 631:
! 632: /*
! 633: * "enter_prot" is the protection we want to enter the page in at.
! 634: * for certain pages (e.g. copy-on-write pages) this protection can
! 635: * be more strict than ufi.entry->protection. "wired" means either
! 636: * the entry is wired or we are fault-wiring the pg.
! 637: */
! 638:
! 639: enter_prot = ufi.entry->protection;
! 640: wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE);
! 641: if (wired)
! 642: access_type = enter_prot; /* full access for wired */
! 643:
! 644: /*
! 645: * handle "needs_copy" case. if we need to copy the amap we will
! 646: * have to drop our readlock and relock it with a write lock. (we
! 647: * need a write lock to change anything in a map entry [e.g.
! 648: * needs_copy]).
! 649: */
! 650:
! 651: if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
! 652: if ((access_type & VM_PROT_WRITE) ||
! 653: (ufi.entry->object.uvm_obj == NULL)) {
! 654: /* need to clear */
! 655: UVMHIST_LOG(maphist,
! 656: " need to clear needs_copy and refault",0,0,0,0);
! 657: uvmfault_unlockmaps(&ufi, FALSE);
! 658: uvmfault_amapcopy(&ufi);
! 659: uvmexp.fltamcopy++;
! 660: goto ReFault;
! 661:
! 662: } else {
! 663:
! 664: /*
! 665: * ensure that we pmap_enter page R/O since
! 666: * needs_copy is still true
! 667: */
! 668: enter_prot &= ~VM_PROT_WRITE;
! 669:
! 670: }
! 671: }
! 672:
! 673: /*
! 674: * identify the players
! 675: */
! 676:
! 677: amap = ufi.entry->aref.ar_amap; /* top layer */
! 678: uobj = ufi.entry->object.uvm_obj; /* bottom layer */
! 679:
! 680: /*
! 681: * check for a case 0 fault. if nothing backing the entry then
! 682: * error now.
! 683: */
! 684:
! 685: if (amap == NULL && uobj == NULL) {
! 686: uvmfault_unlockmaps(&ufi, FALSE);
! 687: UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
! 688: return (EFAULT);
! 689: }
! 690:
! 691: /*
! 692: * establish range of interest based on advice from mapper
! 693: * and then clip to fit map entry. note that we only want
! 694: * to do this the first time through the fault. if we
! 695: * ReFault we will disable this by setting "narrow" to true.
! 696: */
! 697:
! 698: if (narrow == FALSE) {
! 699:
! 700: /* wide fault (!narrow) */
! 701: KASSERT(uvmadvice[ufi.entry->advice].advice ==
! 702: ufi.entry->advice);
! 703: nback = min(uvmadvice[ufi.entry->advice].nback,
! 704: (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
! 705: startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
! 706: nforw = min(uvmadvice[ufi.entry->advice].nforw,
! 707: ((ufi.entry->end - ufi.orig_rvaddr) >>
! 708: PAGE_SHIFT) - 1);
! 709: /*
! 710: * note: "-1" because we don't want to count the
! 711: * faulting page as forw
! 712: */
! 713: npages = nback + nforw + 1;
! 714: centeridx = nback;
! 715:
! 716: narrow = TRUE; /* ensure only once per-fault */
! 717:
! 718: } else {
! 719:
! 720: /* narrow fault! */
! 721: nback = nforw = 0;
! 722: startva = ufi.orig_rvaddr;
! 723: npages = 1;
! 724: centeridx = 0;
! 725:
! 726: }
! 727:
! 728: /* locked: maps(read) */
! 729: UVMHIST_LOG(maphist, " narrow=%ld, back=%ld, forw=%ld, startva=0x%lx",
! 730: narrow, nback, nforw, startva);
! 731: UVMHIST_LOG(maphist, " entry=%p, amap=%p, obj=%p", ufi.entry,
! 732: amap, uobj, 0);
! 733:
! 734: /*
! 735: * if we've got an amap, lock it and extract current anons.
! 736: */
! 737:
! 738: if (amap) {
! 739: anons = anons_store;
! 740: amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
! 741: anons, npages);
! 742: } else {
! 743: anons = NULL; /* to be safe */
! 744: }
! 745:
! 746: /* locked: maps(read), amap(if there) */
! 747:
! 748: /*
! 749: * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
! 750: * now and then forget about them (for the rest of the fault).
! 751: */
! 752:
! 753: if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) {
! 754:
! 755: UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
! 756: 0,0,0,0);
! 757: /* flush back-page anons? */
! 758: if (amap)
! 759: uvmfault_anonflush(anons, nback);
! 760:
! 761: /* flush object? */
! 762: if (uobj) {
! 763: uoff = (startva - ufi.entry->start) + ufi.entry->offset;
! 764: simple_lock(&uobj->vmobjlock);
! 765: (void) uobj->pgops->pgo_flush(uobj, uoff, uoff +
! 766: (nback << PAGE_SHIFT), PGO_DEACTIVATE);
! 767: simple_unlock(&uobj->vmobjlock);
! 768: }
! 769:
! 770: /* now forget about the backpages */
! 771: if (amap)
! 772: anons += nback;
! 773: startva += (nback << PAGE_SHIFT);
! 774: npages -= nback;
! 775: nback = centeridx = 0;
! 776: }
! 777:
! 778: /* locked: maps(read), amap(if there) */
! 779:
! 780: /*
! 781: * map in the backpages and frontpages we found in the amap in hopes
! 782: * of preventing future faults. we also init the pages[] array as
! 783: * we go.
! 784: */
! 785:
! 786: currva = startva;
! 787: shadowed = FALSE;
! 788: for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
! 789:
! 790: /*
! 791: * dont play with VAs that are already mapped
! 792: * except for center)
! 793: */
! 794: if (lcv != centeridx &&
! 795: pmap_extract(ufi.orig_map->pmap, currva, &pa)) {
! 796: pages[lcv] = PGO_DONTCARE;
! 797: continue;
! 798: }
! 799:
! 800: /*
! 801: * unmapped or center page. check if any anon at this level.
! 802: */
! 803: if (amap == NULL || anons[lcv] == NULL) {
! 804: pages[lcv] = NULL;
! 805: continue;
! 806: }
! 807:
! 808: /*
! 809: * check for present page and map if possible. re-activate it.
! 810: */
! 811:
! 812: pages[lcv] = PGO_DONTCARE;
! 813: if (lcv == centeridx) { /* save center for later! */
! 814: shadowed = TRUE;
! 815: continue;
! 816: }
! 817: anon = anons[lcv];
! 818: simple_lock(&anon->an_lock);
! 819: /* ignore loaned pages */
! 820: if (anon->an_page && anon->an_page->loan_count == 0 &&
! 821: (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
! 822: uvm_lock_pageq();
! 823: uvm_pageactivate(anon->an_page); /* reactivate */
! 824: uvm_unlock_pageq();
! 825: UVMHIST_LOG(maphist,
! 826: " MAPPING: n anon: pm=%p, va=0x%lx, pg=%p",
! 827: ufi.orig_map->pmap, currva, anon->an_page, 0);
! 828: uvmexp.fltnamap++;
! 829:
! 830: /*
! 831: * Since this isn't the page that's actually faulting,
! 832: * ignore pmap_enter() failures; it's not critical
! 833: * that we enter these right now.
! 834: */
! 835:
! 836: (void) pmap_enter(ufi.orig_map->pmap, currva,
! 837: VM_PAGE_TO_PHYS(anon->an_page),
! 838: (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
! 839: enter_prot,
! 840: PMAP_CANFAIL |
! 841: (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
! 842: }
! 843: simple_unlock(&anon->an_lock);
! 844: pmap_update(ufi.orig_map->pmap);
! 845: }
! 846:
! 847: /* locked: maps(read), amap(if there) */
! 848: /* (shadowed == TRUE) if there is an anon at the faulting address */
! 849: UVMHIST_LOG(maphist, " shadowed=%ld, will_get=%ld", shadowed,
! 850: (uobj && shadowed == FALSE),0,0);
! 851:
! 852: /*
! 853: * note that if we are really short of RAM we could sleep in the above
! 854: * call to pmap_enter with everything locked. bad?
! 855: *
! 856: * XXX Actually, that is bad; pmap_enter() should just fail in that
! 857: * XXX case. --thorpej
! 858: */
! 859:
! 860: /*
! 861: * if the desired page is not shadowed by the amap and we have a
! 862: * backing object, then we check to see if the backing object would
! 863: * prefer to handle the fault itself (rather than letting us do it
! 864: * with the usual pgo_get hook). the backing object signals this by
! 865: * providing a pgo_fault routine.
! 866: */
! 867:
! 868: if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) {
! 869: simple_lock(&uobj->vmobjlock);
! 870:
! 871: /* locked: maps(read), amap (if there), uobj */
! 872: result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
! 873: centeridx, fault_type, access_type,
! 874: PGO_LOCKED);
! 875:
! 876: /* locked: nothing, pgo_fault has unlocked everything */
! 877:
! 878: if (result == VM_PAGER_OK)
! 879: return (0); /* pgo_fault did pmap enter */
! 880: else if (result == VM_PAGER_REFAULT)
! 881: goto ReFault; /* try again! */
! 882: else
! 883: return (EACCES);
! 884: }
! 885:
! 886: /*
! 887: * now, if the desired page is not shadowed by the amap and we have
! 888: * a backing object that does not have a special fault routine, then
! 889: * we ask (with pgo_get) the object for resident pages that we care
! 890: * about and attempt to map them in. we do not let pgo_get block
! 891: * (PGO_LOCKED).
! 892: *
! 893: * ("get" has the option of doing a pmap_enter for us)
! 894: */
! 895:
! 896: if (uobj && shadowed == FALSE) {
! 897: simple_lock(&uobj->vmobjlock);
! 898:
! 899: /* locked (!shadowed): maps(read), amap (if there), uobj */
! 900: /*
! 901: * the following call to pgo_get does _not_ change locking state
! 902: */
! 903:
! 904: uvmexp.fltlget++;
! 905: gotpages = npages;
! 906: (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
! 907: (startva - ufi.entry->start),
! 908: pages, &gotpages, centeridx,
! 909: access_type & MASK(ufi.entry),
! 910: ufi.entry->advice, PGO_LOCKED);
! 911:
! 912: /*
! 913: * check for pages to map, if we got any
! 914: */
! 915:
! 916: uobjpage = NULL;
! 917:
! 918: if (gotpages) {
! 919: currva = startva;
! 920: for (lcv = 0 ; lcv < npages ;
! 921: lcv++, currva += PAGE_SIZE) {
! 922:
! 923: if (pages[lcv] == NULL ||
! 924: pages[lcv] == PGO_DONTCARE)
! 925: continue;
! 926:
! 927: KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0);
! 928:
! 929: /*
! 930: * if center page is resident and not
! 931: * PG_BUSY|PG_RELEASED then pgo_get
! 932: * made it PG_BUSY for us and gave
! 933: * us a handle to it. remember this
! 934: * page as "uobjpage." (for later use).
! 935: */
! 936:
! 937: if (lcv == centeridx) {
! 938: uobjpage = pages[lcv];
! 939: UVMHIST_LOG(maphist, " got uobjpage "
! 940: "(%p) with locked get",
! 941: uobjpage, 0,0,0);
! 942: continue;
! 943: }
! 944:
! 945: /*
! 946: * note: calling pgo_get with locked data
! 947: * structures returns us pages which are
! 948: * neither busy nor released, so we don't
! 949: * need to check for this. we can just
! 950: * directly enter the page (after moving it
! 951: * to the head of the active queue [useful?]).
! 952: */
! 953:
! 954: uvm_lock_pageq();
! 955: uvm_pageactivate(pages[lcv]); /* reactivate */
! 956: uvm_unlock_pageq();
! 957: UVMHIST_LOG(maphist,
! 958: " MAPPING: n obj: pm=%p, va=0x%lx, pg=%p",
! 959: ufi.orig_map->pmap, currva, pages[lcv], 0);
! 960: uvmexp.fltnomap++;
! 961:
! 962: /*
! 963: * Since this page isn't the page that's
! 964: * actually fauling, ignore pmap_enter()
! 965: * failures; it's not critical that we
! 966: * enter these right now.
! 967: */
! 968:
! 969: (void) pmap_enter(ufi.orig_map->pmap, currva,
! 970: VM_PAGE_TO_PHYS(pages[lcv]),
! 971: enter_prot & MASK(ufi.entry),
! 972: PMAP_CANFAIL |
! 973: (wired ? PMAP_WIRED : 0));
! 974:
! 975: /*
! 976: * NOTE: page can't be PG_WANTED or PG_RELEASED
! 977: * because we've held the lock the whole time
! 978: * we've had the handle.
! 979: */
! 980:
! 981: atomic_clearbits_int(&pages[lcv]->pg_flags,
! 982: PG_BUSY);
! 983: UVM_PAGE_OWN(pages[lcv], NULL);
! 984: } /* for "lcv" loop */
! 985: pmap_update(ufi.orig_map->pmap);
! 986: } /* "gotpages" != 0 */
! 987: /* note: object still _locked_ */
! 988: } else {
! 989: uobjpage = NULL;
! 990: }
! 991:
! 992: /* locked (shadowed): maps(read), amap */
! 993: /* locked (!shadowed): maps(read), amap(if there),
! 994: uobj(if !null), uobjpage(if !null) */
! 995:
! 996: /*
! 997: * note that at this point we are done with any front or back pages.
! 998: * we are now going to focus on the center page (i.e. the one we've
! 999: * faulted on). if we have faulted on the top (anon) layer
! 1000: * [i.e. case 1], then the anon we want is anons[centeridx] (we have
! 1001: * not touched it yet). if we have faulted on the bottom (uobj)
! 1002: * layer [i.e. case 2] and the page was both present and available,
! 1003: * then we've got a pointer to it as "uobjpage" and we've already
! 1004: * made it BUSY.
! 1005: */
! 1006:
! 1007: /*
! 1008: * there are four possible cases we must address: 1A, 1B, 2A, and 2B
! 1009: */
! 1010:
! 1011: /*
! 1012: * redirect case 2: if we are not shadowed, go to case 2.
! 1013: */
! 1014:
! 1015: if (shadowed == FALSE)
! 1016: goto Case2;
! 1017:
! 1018: /* locked: maps(read), amap */
! 1019:
! 1020: /*
! 1021: * handle case 1: fault on an anon in our amap
! 1022: */
! 1023:
! 1024: anon = anons[centeridx];
! 1025: UVMHIST_LOG(maphist, " case 1 fault: anon=%p", anon, 0,0,0);
! 1026: simple_lock(&anon->an_lock);
! 1027:
! 1028: /* locked: maps(read), amap, anon */
! 1029:
! 1030: /*
! 1031: * no matter if we have case 1A or case 1B we are going to need to
! 1032: * have the anon's memory resident. ensure that now.
! 1033: */
! 1034:
! 1035: /*
! 1036: * let uvmfault_anonget do the dirty work.
! 1037: * if it fails (!OK) it will unlock everything for us.
! 1038: * if it succeeds, locks are still valid and locked.
! 1039: * also, if it is OK, then the anon's page is on the queues.
! 1040: * if the page is on loan from a uvm_object, then anonget will
! 1041: * lock that object for us if it does not fail.
! 1042: */
! 1043:
! 1044: result = uvmfault_anonget(&ufi, amap, anon);
! 1045: switch (result) {
! 1046: case VM_PAGER_OK:
! 1047: break;
! 1048:
! 1049: case VM_PAGER_REFAULT:
! 1050: goto ReFault;
! 1051:
! 1052: case VM_PAGER_ERROR:
! 1053: /*
! 1054: * An error occured while trying to bring in the
! 1055: * page -- this is the only error we return right
! 1056: * now.
! 1057: */
! 1058: return (EACCES); /* XXX */
! 1059:
! 1060: default:
! 1061: #ifdef DIAGNOSTIC
! 1062: panic("uvm_fault: uvmfault_anonget -> %d", result);
! 1063: #else
! 1064: return (EACCES);
! 1065: #endif
! 1066: }
! 1067:
! 1068: /*
! 1069: * uobj is non null if the page is on loan from an object (i.e. uobj)
! 1070: */
! 1071:
! 1072: uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
! 1073:
! 1074: /* locked: maps(read), amap, anon, uobj(if one) */
! 1075:
! 1076: /*
! 1077: * special handling for loaned pages
! 1078: */
! 1079:
! 1080: if (anon->an_page->loan_count) {
! 1081:
! 1082: if ((access_type & VM_PROT_WRITE) == 0) {
! 1083:
! 1084: /*
! 1085: * for read faults on loaned pages we just cap the
! 1086: * protection at read-only.
! 1087: */
! 1088:
! 1089: enter_prot = enter_prot & ~VM_PROT_WRITE;
! 1090:
! 1091: } else {
! 1092: /*
! 1093: * note that we can't allow writes into a loaned page!
! 1094: *
! 1095: * if we have a write fault on a loaned page in an
! 1096: * anon then we need to look at the anon's ref count.
! 1097: * if it is greater than one then we are going to do
! 1098: * a normal copy-on-write fault into a new anon (this
! 1099: * is not a problem). however, if the reference count
! 1100: * is one (a case where we would normally allow a
! 1101: * write directly to the page) then we need to kill
! 1102: * the loan before we continue.
! 1103: */
! 1104:
! 1105: /* >1 case is already ok */
! 1106: if (anon->an_ref == 1) {
! 1107:
! 1108: /* get new un-owned replacement page */
! 1109: pg = uvm_pagealloc(NULL, 0, NULL, 0);
! 1110: if (pg == NULL) {
! 1111: uvmfault_unlockall(&ufi, amap, uobj,
! 1112: anon);
! 1113: uvm_wait("flt_noram2");
! 1114: goto ReFault;
! 1115: }
! 1116:
! 1117: /*
! 1118: * copy data, kill loan, and drop uobj lock
! 1119: * (if any)
! 1120: */
! 1121: /* copy old -> new */
! 1122: uvm_pagecopy(anon->an_page, pg);
! 1123:
! 1124: /* force reload */
! 1125: pmap_page_protect(anon->an_page,
! 1126: VM_PROT_NONE);
! 1127: uvm_lock_pageq(); /* KILL loan */
! 1128: if (uobj)
! 1129: /* if we were loaning */
! 1130: anon->an_page->loan_count--;
! 1131: anon->an_page->uanon = NULL;
! 1132: /* in case we owned */
! 1133: atomic_clearbits_int(
! 1134: &anon->an_page->pg_flags, PQ_ANON);
! 1135: uvm_pageactivate(pg);
! 1136: uvm_unlock_pageq();
! 1137: if (uobj) {
! 1138: simple_unlock(&uobj->vmobjlock);
! 1139: uobj = NULL;
! 1140: }
! 1141:
! 1142: /* install new page in anon */
! 1143: anon->an_page = pg;
! 1144: pg->uanon = anon;
! 1145: atomic_setbits_int(&pg->pg_flags, PQ_ANON);
! 1146: atomic_clearbits_int(&pg->pg_flags,
! 1147: PG_BUSY|PG_FAKE);
! 1148: UVM_PAGE_OWN(pg, NULL);
! 1149:
! 1150: /* done! */
! 1151: } /* ref == 1 */
! 1152: } /* write fault */
! 1153: } /* loan count */
! 1154:
! 1155: /*
! 1156: * if we are case 1B then we will need to allocate a new blank
! 1157: * anon to transfer the data into. note that we have a lock
! 1158: * on anon, so no one can busy or release the page until we are done.
! 1159: * also note that the ref count can't drop to zero here because
! 1160: * it is > 1 and we are only dropping one ref.
! 1161: *
! 1162: * in the (hopefully very rare) case that we are out of RAM we
! 1163: * will unlock, wait for more RAM, and refault.
! 1164: *
! 1165: * if we are out of anon VM we kill the process (XXX: could wait?).
! 1166: */
! 1167:
! 1168: if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) {
! 1169:
! 1170: UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
! 1171: uvmexp.flt_acow++;
! 1172: oanon = anon; /* oanon = old, locked anon */
! 1173: anon = uvm_analloc();
! 1174: if (anon) {
! 1175: pg = uvm_pagealloc(NULL, 0, anon, 0);
! 1176: }
! 1177:
! 1178: /* check for out of RAM */
! 1179: if (anon == NULL || pg == NULL) {
! 1180: if (anon)
! 1181: uvm_anfree(anon);
! 1182: uvmfault_unlockall(&ufi, amap, uobj, oanon);
! 1183: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
! 1184: if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
! 1185: UVMHIST_LOG(maphist,
! 1186: "<- failed. out of VM",0,0,0,0);
! 1187: uvmexp.fltnoanon++;
! 1188: return (ENOMEM);
! 1189: }
! 1190:
! 1191: uvmexp.fltnoram++;
! 1192: uvm_wait("flt_noram3"); /* out of RAM, wait for more */
! 1193: goto ReFault;
! 1194: }
! 1195:
! 1196: /* got all resources, replace anon with nanon */
! 1197:
! 1198: uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */
! 1199: /* un-busy! new page */
! 1200: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE);
! 1201: UVM_PAGE_OWN(pg, NULL);
! 1202: amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
! 1203: anon, 1);
! 1204:
! 1205: /* deref: can not drop to zero here by defn! */
! 1206: oanon->an_ref--;
! 1207:
! 1208: /*
! 1209: * note: oanon still locked. anon is _not_ locked, but we
! 1210: * have the sole references to in from amap which _is_ locked.
! 1211: * thus, no one can get at it until we are done with it.
! 1212: */
! 1213:
! 1214: } else {
! 1215:
! 1216: uvmexp.flt_anon++;
! 1217: oanon = anon; /* old, locked anon is same as anon */
! 1218: pg = anon->an_page;
! 1219: if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
! 1220: enter_prot = enter_prot & ~VM_PROT_WRITE;
! 1221:
! 1222: }
! 1223:
! 1224: /* locked: maps(read), amap, oanon */
! 1225:
! 1226: /*
! 1227: * now map the page in ...
! 1228: * XXX: old fault unlocks object before pmap_enter. this seems
! 1229: * suspect since some other thread could blast the page out from
! 1230: * under us between the unlock and the pmap_enter.
! 1231: */
! 1232:
! 1233: UVMHIST_LOG(maphist, " MAPPING: anon: pm=%p, va=0x%lx, pg=%p",
! 1234: ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
! 1235: if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
! 1236: enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
! 1237: != 0) {
! 1238: /*
! 1239: * No need to undo what we did; we can simply think of
! 1240: * this as the pmap throwing away the mapping information.
! 1241: *
! 1242: * We do, however, have to go through the ReFault path,
! 1243: * as the map may change while we're asleep.
! 1244: */
! 1245: uvmfault_unlockall(&ufi, amap, uobj, oanon);
! 1246: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
! 1247: if (uvmexp.swpgonly == uvmexp.swpages) {
! 1248: UVMHIST_LOG(maphist,
! 1249: "<- failed. out of VM",0,0,0,0);
! 1250: /* XXX instrumentation */
! 1251: return (ENOMEM);
! 1252: }
! 1253: /* XXX instrumentation */
! 1254: uvm_wait("flt_pmfail1");
! 1255: goto ReFault;
! 1256: }
! 1257:
! 1258: /*
! 1259: * ... update the page queues.
! 1260: */
! 1261:
! 1262: uvm_lock_pageq();
! 1263:
! 1264: if (fault_type == VM_FAULT_WIRE) {
! 1265: uvm_pagewire(pg);
! 1266:
! 1267: /*
! 1268: * since the now-wired page cannot be paged out,
! 1269: * release its swap resources for others to use.
! 1270: * since an anon with no swap cannot be PG_CLEAN,
! 1271: * clear its clean flag now.
! 1272: */
! 1273: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
! 1274: uvm_anon_dropswap(anon);
! 1275: } else {
! 1276: /* activate it */
! 1277: uvm_pageactivate(pg);
! 1278: }
! 1279:
! 1280: uvm_unlock_pageq();
! 1281:
! 1282: /*
! 1283: * done case 1! finish up by unlocking everything and returning success
! 1284: */
! 1285:
! 1286: uvmfault_unlockall(&ufi, amap, uobj, oanon);
! 1287: pmap_update(ufi.orig_map->pmap);
! 1288: return (0);
! 1289:
! 1290:
! 1291: Case2:
! 1292: /*
! 1293: * handle case 2: faulting on backing object or zero fill
! 1294: */
! 1295:
! 1296: /*
! 1297: * locked:
! 1298: * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
! 1299: */
! 1300:
! 1301: /*
! 1302: * note that uobjpage can not be PGO_DONTCARE at this point. we now
! 1303: * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
! 1304: * have a backing object, check and see if we are going to promote
! 1305: * the data up to an anon during the fault.
! 1306: */
! 1307:
! 1308: if (uobj == NULL) {
! 1309: uobjpage = PGO_DONTCARE;
! 1310: promote = TRUE; /* always need anon here */
! 1311: } else {
! 1312: KASSERT(uobjpage != PGO_DONTCARE);
! 1313: promote = (access_type & VM_PROT_WRITE) &&
! 1314: UVM_ET_ISCOPYONWRITE(ufi.entry);
! 1315: }
! 1316: UVMHIST_LOG(maphist, " case 2 fault: promote=%ld, zfill=%ld",
! 1317: promote, (uobj == NULL), 0,0);
! 1318:
! 1319: /*
! 1320: * if uobjpage is not null then we do not need to do I/O to get the
! 1321: * uobjpage.
! 1322: *
! 1323: * if uobjpage is null, then we need to unlock and ask the pager to
! 1324: * get the data for us. once we have the data, we need to reverify
! 1325: * the state the world. we are currently not holding any resources.
! 1326: */
! 1327:
! 1328: if (uobjpage) {
! 1329: /* update rusage counters */
! 1330: curproc->p_addr->u_stats.p_ru.ru_minflt++;
! 1331: } else {
! 1332: /* update rusage counters */
! 1333: curproc->p_addr->u_stats.p_ru.ru_majflt++;
! 1334:
! 1335: /* locked: maps(read), amap(if there), uobj */
! 1336: uvmfault_unlockall(&ufi, amap, NULL, NULL);
! 1337: /* locked: uobj */
! 1338:
! 1339: uvmexp.fltget++;
! 1340: gotpages = 1;
! 1341: uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
! 1342: result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
! 1343: 0, access_type & MASK(ufi.entry), ufi.entry->advice,
! 1344: PGO_SYNCIO);
! 1345:
! 1346: /* locked: uobjpage(if result OK) */
! 1347:
! 1348: /*
! 1349: * recover from I/O
! 1350: */
! 1351:
! 1352: if (result != VM_PAGER_OK) {
! 1353: KASSERT(result != VM_PAGER_PEND);
! 1354:
! 1355: if (result == VM_PAGER_AGAIN) {
! 1356: UVMHIST_LOG(maphist,
! 1357: " pgo_get says TRY AGAIN!",0,0,0,0);
! 1358: tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0);
! 1359: goto ReFault;
! 1360: }
! 1361:
! 1362: UVMHIST_LOG(maphist, "<- pgo_get failed (code %ld)",
! 1363: result, 0,0,0);
! 1364: return (EACCES); /* XXX i/o error */
! 1365: }
! 1366:
! 1367: /* locked: uobjpage */
! 1368:
! 1369: /*
! 1370: * re-verify the state of the world by first trying to relock
! 1371: * the maps. always relock the object.
! 1372: */
! 1373:
! 1374: locked = uvmfault_relock(&ufi);
! 1375: simple_lock(&uobj->vmobjlock);
! 1376:
! 1377: /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
! 1378: /* locked(!locked): uobj, uobjpage */
! 1379:
! 1380: /*
! 1381: * verify that the page has not be released and re-verify
! 1382: * that amap slot is still free. if there is a problem,
! 1383: * we unlock and clean up.
! 1384: */
! 1385:
! 1386: if ((uobjpage->pg_flags & PG_RELEASED) != 0 ||
! 1387: (locked && amap &&
! 1388: amap_lookup(&ufi.entry->aref,
! 1389: ufi.orig_rvaddr - ufi.entry->start))) {
! 1390: if (locked)
! 1391: uvmfault_unlockall(&ufi, amap, NULL, NULL);
! 1392: locked = FALSE;
! 1393: }
! 1394:
! 1395: /*
! 1396: * didn't get the lock? release the page and retry.
! 1397: */
! 1398:
! 1399: if (locked == FALSE) {
! 1400:
! 1401: UVMHIST_LOG(maphist,
! 1402: " wasn't able to relock after fault: retry",
! 1403: 0,0,0,0);
! 1404: if (uobjpage->pg_flags & PG_WANTED)
! 1405: /* still holding object lock */
! 1406: wakeup(uobjpage);
! 1407:
! 1408: if (uobjpage->pg_flags & PG_RELEASED) {
! 1409: uvmexp.fltpgrele++;
! 1410: KASSERT(uobj->pgops->pgo_releasepg != NULL);
! 1411:
! 1412: /* frees page */
! 1413: if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
! 1414: /* unlock if still alive */
! 1415: simple_unlock(&uobj->vmobjlock);
! 1416: goto ReFault;
! 1417: }
! 1418:
! 1419: uvm_lock_pageq();
! 1420: /* make sure it is in queues */
! 1421: uvm_pageactivate(uobjpage);
! 1422:
! 1423: uvm_unlock_pageq();
! 1424: atomic_clearbits_int(&uobjpage->pg_flags,
! 1425: PG_BUSY|PG_WANTED);
! 1426: UVM_PAGE_OWN(uobjpage, NULL);
! 1427: simple_unlock(&uobj->vmobjlock);
! 1428: goto ReFault;
! 1429:
! 1430: }
! 1431:
! 1432: /*
! 1433: * we have the data in uobjpage which is PG_BUSY and
! 1434: * !PG_RELEASED. we are holding object lock (so the page
! 1435: * can't be released on us).
! 1436: */
! 1437:
! 1438: /* locked: maps(read), amap(if !null), uobj, uobjpage */
! 1439: }
! 1440:
! 1441: /*
! 1442: * locked:
! 1443: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
! 1444: */
! 1445:
! 1446: /*
! 1447: * notes:
! 1448: * - at this point uobjpage can not be NULL
! 1449: * - at this point uobjpage can not be PG_RELEASED (since we checked
! 1450: * for it above)
! 1451: * - at this point uobjpage could be PG_WANTED (handle later)
! 1452: */
! 1453:
! 1454: if (promote == FALSE) {
! 1455:
! 1456: /*
! 1457: * we are not promoting. if the mapping is COW ensure that we
! 1458: * don't give more access than we should (e.g. when doing a read
! 1459: * fault on a COPYONWRITE mapping we want to map the COW page in
! 1460: * R/O even though the entry protection could be R/W).
! 1461: *
! 1462: * set "pg" to the page we want to map in (uobjpage, usually)
! 1463: */
! 1464:
! 1465: uvmexp.flt_obj++;
! 1466: if (UVM_ET_ISCOPYONWRITE(ufi.entry))
! 1467: enter_prot &= ~VM_PROT_WRITE;
! 1468: pg = uobjpage; /* map in the actual object */
! 1469:
! 1470: /* assert(uobjpage != PGO_DONTCARE) */
! 1471:
! 1472: /*
! 1473: * we are faulting directly on the page. be careful
! 1474: * about writing to loaned pages...
! 1475: */
! 1476: if (uobjpage->loan_count) {
! 1477:
! 1478: if ((access_type & VM_PROT_WRITE) == 0) {
! 1479: /* read fault: cap the protection at readonly */
! 1480: /* cap! */
! 1481: enter_prot = enter_prot & ~VM_PROT_WRITE;
! 1482: } else {
! 1483: /* write fault: must break the loan here */
! 1484:
! 1485: /* alloc new un-owned page */
! 1486: pg = uvm_pagealloc(NULL, 0, NULL, 0);
! 1487:
! 1488: if (pg == NULL) {
! 1489: /*
! 1490: * drop ownership of page, it can't
! 1491: * be released
! 1492: */
! 1493: if (uobjpage->pg_flags & PG_WANTED)
! 1494: wakeup(uobjpage);
! 1495: atomic_clearbits_int(
! 1496: &uobjpage->pg_flags,
! 1497: PG_BUSY|PG_WANTED);
! 1498: UVM_PAGE_OWN(uobjpage, NULL);
! 1499:
! 1500: uvm_lock_pageq();
! 1501: /* activate: we will need it later */
! 1502: uvm_pageactivate(uobjpage);
! 1503:
! 1504: uvm_unlock_pageq();
! 1505: uvmfault_unlockall(&ufi, amap, uobj,
! 1506: NULL);
! 1507: UVMHIST_LOG(maphist,
! 1508: " out of RAM breaking loan, waiting",
! 1509: 0,0,0,0);
! 1510: uvmexp.fltnoram++;
! 1511: uvm_wait("flt_noram4");
! 1512: goto ReFault;
! 1513: }
! 1514:
! 1515: /*
! 1516: * copy the data from the old page to the new
! 1517: * one and clear the fake/clean flags on the
! 1518: * new page (keep it busy). force a reload
! 1519: * of the old page by clearing it from all
! 1520: * pmaps. then lock the page queues to
! 1521: * rename the pages.
! 1522: */
! 1523: uvm_pagecopy(uobjpage, pg); /* old -> new */
! 1524: atomic_clearbits_int(&pg->pg_flags,
! 1525: PG_FAKE|PG_CLEAN);
! 1526: pmap_page_protect(uobjpage, VM_PROT_NONE);
! 1527: if (uobjpage->pg_flags & PG_WANTED)
! 1528: wakeup(uobjpage);
! 1529: /* uobj still locked */
! 1530: atomic_clearbits_int(&uobjpage->pg_flags,
! 1531: PG_BUSY|PG_WANTED);
! 1532: UVM_PAGE_OWN(uobjpage, NULL);
! 1533:
! 1534: uvm_lock_pageq();
! 1535: uoff = uobjpage->offset;
! 1536: /* remove old page */
! 1537: uvm_pagerealloc(uobjpage, NULL, 0);
! 1538:
! 1539: /*
! 1540: * at this point we have absolutely no
! 1541: * control over uobjpage
! 1542: */
! 1543: /* install new page */
! 1544: uvm_pagerealloc(pg, uobj, uoff);
! 1545: uvm_unlock_pageq();
! 1546:
! 1547: /*
! 1548: * done! loan is broken and "pg" is
! 1549: * PG_BUSY. it can now replace uobjpage.
! 1550: */
! 1551:
! 1552: uobjpage = pg;
! 1553:
! 1554: } /* write fault case */
! 1555: } /* if loan_count */
! 1556:
! 1557: } else {
! 1558:
! 1559: /*
! 1560: * if we are going to promote the data to an anon we
! 1561: * allocate a blank anon here and plug it into our amap.
! 1562: */
! 1563: #ifdef DIAGNOSTIC
! 1564: if (amap == NULL)
! 1565: panic("uvm_fault: want to promote data, but no anon");
! 1566: #endif
! 1567:
! 1568: anon = uvm_analloc();
! 1569: if (anon) {
! 1570: /*
! 1571: * In `Fill in data...' below, if
! 1572: * uobjpage == PGO_DONTCARE, we want
! 1573: * a zero'd, dirty page, so have
! 1574: * uvm_pagealloc() do that for us.
! 1575: */
! 1576: pg = uvm_pagealloc(NULL, 0, anon,
! 1577: (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);
! 1578: }
! 1579:
! 1580: /*
! 1581: * out of memory resources?
! 1582: */
! 1583: if (anon == NULL || pg == NULL) {
! 1584:
! 1585: /*
! 1586: * arg! must unbusy our page and fail or sleep.
! 1587: */
! 1588: if (uobjpage != PGO_DONTCARE) {
! 1589: if (uobjpage->pg_flags & PG_WANTED)
! 1590: /* still holding object lock */
! 1591: wakeup(uobjpage);
! 1592:
! 1593: uvm_lock_pageq();
! 1594: uvm_pageactivate(uobjpage);
! 1595: uvm_unlock_pageq();
! 1596: atomic_clearbits_int(&uobjpage->pg_flags,
! 1597: PG_BUSY|PG_WANTED);
! 1598: UVM_PAGE_OWN(uobjpage, NULL);
! 1599: }
! 1600:
! 1601: /* unlock and fail ... */
! 1602: uvmfault_unlockall(&ufi, amap, uobj, NULL);
! 1603: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
! 1604: if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
! 1605: UVMHIST_LOG(maphist, " promote: out of VM",
! 1606: 0,0,0,0);
! 1607: uvmexp.fltnoanon++;
! 1608: return (ENOMEM);
! 1609: }
! 1610:
! 1611: UVMHIST_LOG(maphist, " out of RAM, waiting for more",
! 1612: 0,0,0,0);
! 1613: uvm_anfree(anon);
! 1614: uvmexp.fltnoram++;
! 1615: uvm_wait("flt_noram5");
! 1616: goto ReFault;
! 1617: }
! 1618:
! 1619: /*
! 1620: * fill in the data
! 1621: */
! 1622:
! 1623: if (uobjpage != PGO_DONTCARE) {
! 1624: uvmexp.flt_prcopy++;
! 1625: /* copy page [pg now dirty] */
! 1626: uvm_pagecopy(uobjpage, pg);
! 1627:
! 1628: /*
! 1629: * promote to shared amap? make sure all sharing
! 1630: * procs see it
! 1631: */
! 1632: if ((amap_flags(amap) & AMAP_SHARED) != 0) {
! 1633: pmap_page_protect(uobjpage, VM_PROT_NONE);
! 1634: }
! 1635:
! 1636: /*
! 1637: * dispose of uobjpage. it can't be PG_RELEASED
! 1638: * since we still hold the object lock.
! 1639: * drop handle to uobj as well.
! 1640: */
! 1641:
! 1642: if (uobjpage->pg_flags & PG_WANTED)
! 1643: /* still have the obj lock */
! 1644: wakeup(uobjpage);
! 1645: atomic_clearbits_int(&uobjpage->pg_flags,
! 1646: PG_BUSY|PG_WANTED);
! 1647: UVM_PAGE_OWN(uobjpage, NULL);
! 1648: uvm_lock_pageq();
! 1649: uvm_pageactivate(uobjpage);
! 1650: uvm_unlock_pageq();
! 1651: simple_unlock(&uobj->vmobjlock);
! 1652: uobj = NULL;
! 1653:
! 1654: UVMHIST_LOG(maphist,
! 1655: " promote uobjpage %p to anon/page %p/%p",
! 1656: uobjpage, anon, pg, 0);
! 1657:
! 1658: } else {
! 1659: uvmexp.flt_przero++;
! 1660: /*
! 1661: * Page is zero'd and marked dirty by uvm_pagealloc()
! 1662: * above.
! 1663: */
! 1664: UVMHIST_LOG(maphist," zero fill anon/page %p/%p",
! 1665: anon, pg, 0, 0);
! 1666: }
! 1667:
! 1668: amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
! 1669: anon, 0);
! 1670: }
! 1671:
! 1672: /*
! 1673: * locked:
! 1674: * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
! 1675: *
! 1676: * note: pg is either the uobjpage or the new page in the new anon
! 1677: */
! 1678:
! 1679: /*
! 1680: * all resources are present. we can now map it in and free our
! 1681: * resources.
! 1682: */
! 1683:
! 1684: UVMHIST_LOG(maphist,
! 1685: " MAPPING: case2: pm=%p, va=0x%lx, pg=%p, promote=%ld",
! 1686: ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
! 1687: if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
! 1688: enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
! 1689: != 0) {
! 1690:
! 1691: /*
! 1692: * No need to undo what we did; we can simply think of
! 1693: * this as the pmap throwing away the mapping information.
! 1694: *
! 1695: * We do, however, have to go through the ReFault path,
! 1696: * as the map may change while we're asleep.
! 1697: */
! 1698:
! 1699: if (pg->pg_flags & PG_WANTED)
! 1700: wakeup(pg); /* lock still held */
! 1701:
! 1702: /*
! 1703: * note that pg can't be PG_RELEASED since we did not drop
! 1704: * the object lock since the last time we checked.
! 1705: */
! 1706:
! 1707: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
! 1708: UVM_PAGE_OWN(pg, NULL);
! 1709: uvmfault_unlockall(&ufi, amap, uobj, NULL);
! 1710: KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
! 1711: if (uvmexp.swpgonly == uvmexp.swpages) {
! 1712: UVMHIST_LOG(maphist,
! 1713: "<- failed. out of VM",0,0,0,0);
! 1714: /* XXX instrumentation */
! 1715: return (ENOMEM);
! 1716: }
! 1717: /* XXX instrumentation */
! 1718: uvm_wait("flt_pmfail2");
! 1719: goto ReFault;
! 1720: }
! 1721:
! 1722: uvm_lock_pageq();
! 1723:
! 1724: if (fault_type == VM_FAULT_WIRE) {
! 1725: uvm_pagewire(pg);
! 1726: if (pg->pg_flags & PQ_AOBJ) {
! 1727:
! 1728: /*
! 1729: * since the now-wired page cannot be paged out,
! 1730: * release its swap resources for others to use.
! 1731: * since an aobj page with no swap cannot be PG_CLEAN,
! 1732: * clear its clean flag now.
! 1733: */
! 1734: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
! 1735: uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
! 1736: }
! 1737: } else {
! 1738: /* activate it */
! 1739: uvm_pageactivate(pg);
! 1740: }
! 1741: uvm_unlock_pageq();
! 1742:
! 1743: if (pg->pg_flags & PG_WANTED)
! 1744: wakeup(pg); /* lock still held */
! 1745:
! 1746: /*
! 1747: * note that pg can't be PG_RELEASED since we did not drop the object
! 1748: * lock since the last time we checked.
! 1749: */
! 1750:
! 1751: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
! 1752: UVM_PAGE_OWN(pg, NULL);
! 1753: uvmfault_unlockall(&ufi, amap, uobj, NULL);
! 1754: pmap_update(ufi.orig_map->pmap);
! 1755:
! 1756: UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
! 1757: return (0);
! 1758: }
! 1759:
! 1760:
! 1761: /*
! 1762: * uvm_fault_wire: wire down a range of virtual addresses in a map.
! 1763: *
! 1764: * => map may be read-locked by caller, but MUST NOT be write-locked.
! 1765: * => if map is read-locked, any operations which may cause map to
! 1766: * be write-locked in uvm_fault() must be taken care of by
! 1767: * the caller. See uvm_map_pageable().
! 1768: */
! 1769:
! 1770: int
! 1771: uvm_fault_wire(map, start, end, access_type)
! 1772: vm_map_t map;
! 1773: vaddr_t start, end;
! 1774: vm_prot_t access_type;
! 1775: {
! 1776: vaddr_t va;
! 1777: pmap_t pmap;
! 1778: int rv;
! 1779:
! 1780: pmap = vm_map_pmap(map);
! 1781:
! 1782: /*
! 1783: * now fault it in a page at a time. if the fault fails then we have
! 1784: * to undo what we have done. note that in uvm_fault VM_PROT_NONE
! 1785: * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
! 1786: */
! 1787:
! 1788: for (va = start ; va < end ; va += PAGE_SIZE) {
! 1789: rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type);
! 1790: if (rv) {
! 1791: if (va != start) {
! 1792: uvm_fault_unwire(map, start, va);
! 1793: }
! 1794: return (rv);
! 1795: }
! 1796: }
! 1797:
! 1798: return (0);
! 1799: }
! 1800:
! 1801: /*
! 1802: * uvm_fault_unwire(): unwire range of virtual space.
! 1803: */
! 1804:
! 1805: void
! 1806: uvm_fault_unwire(map, start, end)
! 1807: vm_map_t map;
! 1808: vaddr_t start, end;
! 1809: {
! 1810:
! 1811: vm_map_lock_read(map);
! 1812: uvm_fault_unwire_locked(map, start, end);
! 1813: vm_map_unlock_read(map);
! 1814: }
! 1815:
! 1816: /*
! 1817: * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
! 1818: *
! 1819: * => map must be at least read-locked.
! 1820: */
! 1821:
! 1822: void
! 1823: uvm_fault_unwire_locked(map, start, end)
! 1824: vm_map_t map;
! 1825: vaddr_t start, end;
! 1826: {
! 1827: vm_map_entry_t entry;
! 1828: pmap_t pmap = vm_map_pmap(map);
! 1829: vaddr_t va;
! 1830: paddr_t pa;
! 1831: struct vm_page *pg;
! 1832:
! 1833: KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
! 1834:
! 1835: /*
! 1836: * we assume that the area we are unwiring has actually been wired
! 1837: * in the first place. this means that we should be able to extract
! 1838: * the PAs from the pmap. we also lock out the page daemon so that
! 1839: * we can call uvm_pageunwire.
! 1840: */
! 1841:
! 1842: uvm_lock_pageq();
! 1843:
! 1844: /*
! 1845: * find the beginning map entry for the region.
! 1846: */
! 1847: KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
! 1848: if (uvm_map_lookup_entry(map, start, &entry) == FALSE)
! 1849: panic("uvm_fault_unwire_locked: address not in map");
! 1850:
! 1851: for (va = start; va < end ; va += PAGE_SIZE) {
! 1852: if (pmap_extract(pmap, va, &pa) == FALSE)
! 1853: continue;
! 1854:
! 1855: /*
! 1856: * find the map entry for the current address.
! 1857: */
! 1858: KASSERT(va >= entry->start);
! 1859: while (va >= entry->end) {
! 1860: KASSERT(entry->next != &map->header &&
! 1861: entry->next->start <= entry->end);
! 1862: entry = entry->next;
! 1863: }
! 1864:
! 1865: /*
! 1866: * if the entry is no longer wired, tell the pmap.
! 1867: */
! 1868: if (VM_MAPENT_ISWIRED(entry) == 0)
! 1869: pmap_unwire(pmap, va);
! 1870:
! 1871: pg = PHYS_TO_VM_PAGE(pa);
! 1872: if (pg)
! 1873: uvm_pageunwire(pg);
! 1874: }
! 1875:
! 1876: uvm_unlock_pageq();
! 1877: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_fault.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm