Annotation of sys/uvm/uvm_page.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: uvm_page.c,v 1.61 2007/06/18 21:51:15 pedro Exp $ */
! 2: /* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */
! 3:
! 4: /*
! 5: * Copyright (c) 1997 Charles D. Cranor and Washington University.
! 6: * Copyright (c) 1991, 1993, The Regents of the University of California.
! 7: *
! 8: * All rights reserved.
! 9: *
! 10: * This code is derived from software contributed to Berkeley by
! 11: * The Mach Operating System project at Carnegie-Mellon University.
! 12: *
! 13: * Redistribution and use in source and binary forms, with or without
! 14: * modification, are permitted provided that the following conditions
! 15: * are met:
! 16: * 1. Redistributions of source code must retain the above copyright
! 17: * notice, this list of conditions and the following disclaimer.
! 18: * 2. Redistributions in binary form must reproduce the above copyright
! 19: * notice, this list of conditions and the following disclaimer in the
! 20: * documentation and/or other materials provided with the distribution.
! 21: * 3. All advertising materials mentioning features or use of this software
! 22: * must display the following acknowledgement:
! 23: * This product includes software developed by Charles D. Cranor,
! 24: * Washington University, the University of California, Berkeley and
! 25: * its contributors.
! 26: * 4. Neither the name of the University nor the names of its contributors
! 27: * may be used to endorse or promote products derived from this software
! 28: * without specific prior written permission.
! 29: *
! 30: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
! 31: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
! 32: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
! 33: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
! 34: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
! 35: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
! 36: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
! 37: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
! 38: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
! 39: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
! 40: * SUCH DAMAGE.
! 41: *
! 42: * @(#)vm_page.c 8.3 (Berkeley) 3/21/94
! 43: * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
! 44: *
! 45: *
! 46: * Copyright (c) 1987, 1990 Carnegie-Mellon University.
! 47: * All rights reserved.
! 48: *
! 49: * Permission to use, copy, modify and distribute this software and
! 50: * its documentation is hereby granted, provided that both the copyright
! 51: * notice and this permission notice appear in all copies of the
! 52: * software, derivative works or modified versions, and any portions
! 53: * thereof, and that both notices appear in supporting documentation.
! 54: *
! 55: * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
! 56: * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
! 57: * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
! 58: *
! 59: * Carnegie Mellon requests users of this software to return to
! 60: *
! 61: * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
! 62: * School of Computer Science
! 63: * Carnegie Mellon University
! 64: * Pittsburgh PA 15213-3890
! 65: *
! 66: * any improvements or extensions that they make and grant Carnegie the
! 67: * rights to redistribute these changes.
! 68: */
! 69:
! 70: /*
! 71: * uvm_page.c: page ops.
! 72: */
! 73:
! 74: #define UVM_PAGE /* pull in uvm_page.h functions */
! 75: #include <sys/param.h>
! 76: #include <sys/systm.h>
! 77: #include <sys/malloc.h>
! 78: #include <sys/sched.h>
! 79: #include <sys/kernel.h>
! 80: #include <sys/vnode.h>
! 81:
! 82: #include <uvm/uvm.h>
! 83:
! 84: /*
! 85: * global vars... XXXCDC: move to uvm. structure.
! 86: */
! 87:
! 88: /*
! 89: * physical memory config is stored in vm_physmem.
! 90: */
! 91:
! 92: struct vm_physseg vm_physmem[VM_PHYSSEG_MAX]; /* XXXCDC: uvm.physmem */
! 93: int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
! 94:
! 95: /*
! 96: * Some supported CPUs in a given architecture don't support all
! 97: * of the things necessary to do idle page zero'ing efficiently.
! 98: * We therefore provide a way to disable it from machdep code here.
! 99: */
! 100:
! 101: /*
! 102: * XXX disabled until we can find a way to do this without causing
! 103: * problems for either cpu caches or DMA latency.
! 104: */
! 105: boolean_t vm_page_zero_enable = FALSE;
! 106:
! 107: /*
! 108: * local variables
! 109: */
! 110:
! 111: /*
! 112: * these variables record the values returned by vm_page_bootstrap,
! 113: * for debugging purposes. The implementation of uvm_pageboot_alloc
! 114: * and pmap_startup here also uses them internally.
! 115: */
! 116:
! 117: static vaddr_t virtual_space_start;
! 118: static vaddr_t virtual_space_end;
! 119:
! 120: /*
! 121: * we use a hash table with only one bucket during bootup. we will
! 122: * later rehash (resize) the hash table once the allocator is ready.
! 123: * we static allocate the one bootstrap bucket below...
! 124: */
! 125:
! 126: static struct pglist uvm_bootbucket;
! 127:
! 128: /*
! 129: * History
! 130: */
! 131: UVMHIST_DECL(pghist);
! 132:
! 133: /*
! 134: * local prototypes
! 135: */
! 136:
! 137: static void uvm_pageinsert(struct vm_page *);
! 138: static void uvm_pageremove(struct vm_page *);
! 139:
! 140: /*
! 141: * inline functions
! 142: */
! 143:
! 144: /*
! 145: * uvm_pageinsert: insert a page in the object and the hash table
! 146: *
! 147: * => caller must lock object
! 148: * => caller must lock page queues
! 149: * => call should have already set pg's object and offset pointers
! 150: * and bumped the version counter
! 151: */
! 152:
! 153: __inline static void
! 154: uvm_pageinsert(struct vm_page *pg)
! 155: {
! 156: struct pglist *buck;
! 157: int s;
! 158: UVMHIST_FUNC("uvm_pageinsert"); UVMHIST_CALLED(pghist);
! 159:
! 160: KASSERT((pg->pg_flags & PG_TABLED) == 0);
! 161: buck = &uvm.page_hash[uvm_pagehash(pg->uobject,pg->offset)];
! 162: s = splvm();
! 163: simple_lock(&uvm.hashlock);
! 164: TAILQ_INSERT_TAIL(buck, pg, hashq); /* put in hash */
! 165: simple_unlock(&uvm.hashlock);
! 166: splx(s);
! 167:
! 168: TAILQ_INSERT_TAIL(&pg->uobject->memq, pg, listq); /* put in object */
! 169: atomic_setbits_int(&pg->pg_flags, PG_TABLED);
! 170: pg->uobject->uo_npages++;
! 171: }
! 172:
! 173: /*
! 174: * uvm_page_remove: remove page from object and hash
! 175: *
! 176: * => caller must lock object
! 177: * => caller must lock page queues
! 178: */
! 179:
! 180: static __inline void
! 181: uvm_pageremove(struct vm_page *pg)
! 182: {
! 183: struct pglist *buck;
! 184: int s;
! 185: UVMHIST_FUNC("uvm_pageremove"); UVMHIST_CALLED(pghist);
! 186:
! 187: KASSERT(pg->pg_flags & PG_TABLED);
! 188: buck = &uvm.page_hash[uvm_pagehash(pg->uobject,pg->offset)];
! 189: s = splvm();
! 190: simple_lock(&uvm.hashlock);
! 191: TAILQ_REMOVE(buck, pg, hashq);
! 192: simple_unlock(&uvm.hashlock);
! 193: splx(s);
! 194:
! 195: #ifdef UBC
! 196: if (pg->uobject->pgops == &uvm_vnodeops) {
! 197: uvm_pgcnt_vnode--;
! 198: }
! 199: #endif
! 200:
! 201: /* object should be locked */
! 202: TAILQ_REMOVE(&pg->uobject->memq, pg, listq);
! 203:
! 204: atomic_clearbits_int(&pg->pg_flags, PG_TABLED);
! 205: pg->uobject->uo_npages--;
! 206: pg->uobject = NULL;
! 207: pg->pg_version++;
! 208: }
! 209:
! 210: /*
! 211: * uvm_page_init: init the page system. called from uvm_init().
! 212: *
! 213: * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
! 214: */
! 215:
! 216: void
! 217: uvm_page_init(kvm_startp, kvm_endp)
! 218: vaddr_t *kvm_startp, *kvm_endp;
! 219: {
! 220: vsize_t freepages, pagecount, n;
! 221: vm_page_t pagearray;
! 222: int lcv, i;
! 223: paddr_t paddr;
! 224: #if defined(UVMHIST)
! 225: static struct uvm_history_ent pghistbuf[100];
! 226: #endif
! 227:
! 228: UVMHIST_FUNC("uvm_page_init");
! 229: UVMHIST_INIT_STATIC(pghist, pghistbuf);
! 230: UVMHIST_CALLED(pghist);
! 231:
! 232: /*
! 233: * init the page queues and page queue locks
! 234: */
! 235:
! 236: for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
! 237: for (i = 0; i < PGFL_NQUEUES; i++)
! 238: TAILQ_INIT(&uvm.page_free[lcv].pgfl_queues[i]);
! 239: }
! 240: TAILQ_INIT(&uvm.page_active);
! 241: TAILQ_INIT(&uvm.page_inactive_swp);
! 242: TAILQ_INIT(&uvm.page_inactive_obj);
! 243: simple_lock_init(&uvm.pageqlock);
! 244: simple_lock_init(&uvm.fpageqlock);
! 245:
! 246: /*
! 247: * init the <obj,offset> => <page> hash table. for now
! 248: * we just have one bucket (the bootstrap bucket). later on we
! 249: * will allocate new buckets as we dynamically resize the hash table.
! 250: */
! 251:
! 252: uvm.page_nhash = 1; /* 1 bucket */
! 253: uvm.page_hashmask = 0; /* mask for hash function */
! 254: uvm.page_hash = &uvm_bootbucket; /* install bootstrap bucket */
! 255: TAILQ_INIT(uvm.page_hash); /* init hash table */
! 256: simple_lock_init(&uvm.hashlock); /* init hash table lock */
! 257:
! 258: /*
! 259: * allocate vm_page structures.
! 260: */
! 261:
! 262: /*
! 263: * sanity check:
! 264: * before calling this function the MD code is expected to register
! 265: * some free RAM with the uvm_page_physload() function. our job
! 266: * now is to allocate vm_page structures for this memory.
! 267: */
! 268:
! 269: if (vm_nphysseg == 0)
! 270: panic("uvm_page_bootstrap: no memory pre-allocated");
! 271:
! 272: /*
! 273: * first calculate the number of free pages...
! 274: *
! 275: * note that we use start/end rather than avail_start/avail_end.
! 276: * this allows us to allocate extra vm_page structures in case we
! 277: * want to return some memory to the pool after booting.
! 278: */
! 279:
! 280: freepages = 0;
! 281: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 282: freepages += (vm_physmem[lcv].end - vm_physmem[lcv].start);
! 283:
! 284: /*
! 285: * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
! 286: * use. for each page of memory we use we need a vm_page structure.
! 287: * thus, the total number of pages we can use is the total size of
! 288: * the memory divided by the PAGE_SIZE plus the size of the vm_page
! 289: * structure. we add one to freepages as a fudge factor to avoid
! 290: * truncation errors (since we can only allocate in terms of whole
! 291: * pages).
! 292: */
! 293:
! 294: pagecount = (((paddr_t)freepages + 1) << PAGE_SHIFT) /
! 295: (PAGE_SIZE + sizeof(struct vm_page));
! 296: pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount *
! 297: sizeof(struct vm_page));
! 298: memset(pagearray, 0, pagecount * sizeof(struct vm_page));
! 299:
! 300: /*
! 301: * init the vm_page structures and put them in the correct place.
! 302: */
! 303:
! 304: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
! 305: n = vm_physmem[lcv].end - vm_physmem[lcv].start;
! 306: if (n > pagecount) {
! 307: printf("uvm_page_init: lost %ld page(s) in init\n",
! 308: (long)(n - pagecount));
! 309: panic("uvm_page_init"); /* XXXCDC: shouldn't happen? */
! 310: /* n = pagecount; */
! 311: }
! 312:
! 313: /* set up page array pointers */
! 314: vm_physmem[lcv].pgs = pagearray;
! 315: pagearray += n;
! 316: pagecount -= n;
! 317: vm_physmem[lcv].lastpg = vm_physmem[lcv].pgs + (n - 1);
! 318:
! 319: /* init and free vm_pages (we've already zeroed them) */
! 320: paddr = ptoa(vm_physmem[lcv].start);
! 321: for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
! 322: vm_physmem[lcv].pgs[i].phys_addr = paddr;
! 323: #ifdef __HAVE_VM_PAGE_MD
! 324: VM_MDPAGE_INIT(&vm_physmem[lcv].pgs[i]);
! 325: #endif
! 326: if (atop(paddr) >= vm_physmem[lcv].avail_start &&
! 327: atop(paddr) <= vm_physmem[lcv].avail_end) {
! 328: uvmexp.npages++;
! 329: /* add page to free pool */
! 330: uvm_pagefree(&vm_physmem[lcv].pgs[i]);
! 331: }
! 332: }
! 333: }
! 334:
! 335: /*
! 336: * pass up the values of virtual_space_start and
! 337: * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
! 338: * layers of the VM.
! 339: */
! 340:
! 341: *kvm_startp = round_page(virtual_space_start);
! 342: *kvm_endp = trunc_page(virtual_space_end);
! 343:
! 344: /*
! 345: * init locks for kernel threads
! 346: */
! 347:
! 348: simple_lock_init(&uvm.pagedaemon_lock);
! 349: simple_lock_init(&uvm.aiodoned_lock);
! 350:
! 351: /*
! 352: * init reserve thresholds
! 353: * XXXCDC - values may need adjusting
! 354: */
! 355: uvmexp.reserve_pagedaemon = 4;
! 356: uvmexp.reserve_kernel = 6;
! 357: uvmexp.anonminpct = 10;
! 358: uvmexp.vnodeminpct = 10;
! 359: uvmexp.vtextminpct = 5;
! 360: uvmexp.anonmin = uvmexp.anonminpct * 256 / 100;
! 361: uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100;
! 362: uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100;
! 363:
! 364: /*
! 365: * determine if we should zero pages in the idle loop.
! 366: */
! 367:
! 368: uvm.page_idle_zero = vm_page_zero_enable;
! 369:
! 370: /*
! 371: * done!
! 372: */
! 373:
! 374: uvm.page_init_done = TRUE;
! 375: }
! 376:
! 377: /*
! 378: * uvm_setpagesize: set the page size
! 379: *
! 380: * => sets page_shift and page_mask from uvmexp.pagesize.
! 381: */
! 382:
! 383: void
! 384: uvm_setpagesize()
! 385: {
! 386: if (uvmexp.pagesize == 0)
! 387: uvmexp.pagesize = DEFAULT_PAGE_SIZE;
! 388: uvmexp.pagemask = uvmexp.pagesize - 1;
! 389: if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
! 390: panic("uvm_setpagesize: page size not a power of two");
! 391: for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
! 392: if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
! 393: break;
! 394: }
! 395:
! 396: /*
! 397: * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
! 398: */
! 399:
! 400: vaddr_t
! 401: uvm_pageboot_alloc(size)
! 402: vsize_t size;
! 403: {
! 404: #if defined(PMAP_STEAL_MEMORY)
! 405: vaddr_t addr;
! 406:
! 407: /*
! 408: * defer bootstrap allocation to MD code (it may want to allocate
! 409: * from a direct-mapped segment). pmap_steal_memory should round
! 410: * off virtual_space_start/virtual_space_end.
! 411: */
! 412:
! 413: addr = pmap_steal_memory(size, &virtual_space_start,
! 414: &virtual_space_end);
! 415:
! 416: return(addr);
! 417:
! 418: #else /* !PMAP_STEAL_MEMORY */
! 419:
! 420: static boolean_t initialized = FALSE;
! 421: vaddr_t addr, vaddr;
! 422: paddr_t paddr;
! 423:
! 424: /* round to page size */
! 425: size = round_page(size);
! 426:
! 427: /*
! 428: * on first call to this function, initialize ourselves.
! 429: */
! 430: if (initialized == FALSE) {
! 431: pmap_virtual_space(&virtual_space_start, &virtual_space_end);
! 432:
! 433: /* round it the way we like it */
! 434: virtual_space_start = round_page(virtual_space_start);
! 435: virtual_space_end = trunc_page(virtual_space_end);
! 436:
! 437: initialized = TRUE;
! 438: }
! 439:
! 440: /*
! 441: * allocate virtual memory for this request
! 442: */
! 443: if (virtual_space_start == virtual_space_end ||
! 444: (virtual_space_end - virtual_space_start) < size)
! 445: panic("uvm_pageboot_alloc: out of virtual space");
! 446:
! 447: addr = virtual_space_start;
! 448:
! 449: #ifdef PMAP_GROWKERNEL
! 450: /*
! 451: * If the kernel pmap can't map the requested space,
! 452: * then allocate more resources for it.
! 453: */
! 454: if (uvm_maxkaddr < (addr + size)) {
! 455: uvm_maxkaddr = pmap_growkernel(addr + size);
! 456: if (uvm_maxkaddr < (addr + size))
! 457: panic("uvm_pageboot_alloc: pmap_growkernel() failed");
! 458: }
! 459: #endif
! 460:
! 461: virtual_space_start += size;
! 462:
! 463: /*
! 464: * allocate and mapin physical pages to back new virtual pages
! 465: */
! 466:
! 467: for (vaddr = round_page(addr) ; vaddr < addr + size ;
! 468: vaddr += PAGE_SIZE) {
! 469:
! 470: if (!uvm_page_physget(&paddr))
! 471: panic("uvm_pageboot_alloc: out of memory");
! 472:
! 473: /*
! 474: * Note this memory is no longer managed, so using
! 475: * pmap_kenter is safe.
! 476: */
! 477: pmap_kenter_pa(vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE);
! 478: }
! 479: pmap_update(pmap_kernel());
! 480: return(addr);
! 481: #endif /* PMAP_STEAL_MEMORY */
! 482: }
! 483:
! 484: #if !defined(PMAP_STEAL_MEMORY)
! 485: /*
! 486: * uvm_page_physget: "steal" one page from the vm_physmem structure.
! 487: *
! 488: * => attempt to allocate it off the end of a segment in which the "avail"
! 489: * values match the start/end values. if we can't do that, then we
! 490: * will advance both values (making them equal, and removing some
! 491: * vm_page structures from the non-avail area).
! 492: * => return false if out of memory.
! 493: */
! 494:
! 495: /* subroutine: try to allocate from memory chunks on the specified freelist */
! 496: static boolean_t uvm_page_physget_freelist(paddr_t *, int);
! 497:
! 498: static boolean_t
! 499: uvm_page_physget_freelist(paddrp, freelist)
! 500: paddr_t *paddrp;
! 501: int freelist;
! 502: {
! 503: int lcv, x;
! 504: UVMHIST_FUNC("uvm_page_physget_freelist"); UVMHIST_CALLED(pghist);
! 505:
! 506: /* pass 1: try allocating from a matching end */
! 507: #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) || \
! 508: (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
! 509: for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
! 510: #else
! 511: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 512: #endif
! 513: {
! 514:
! 515: if (uvm.page_init_done == TRUE)
! 516: panic("uvm_page_physget: called _after_ bootstrap");
! 517:
! 518: if (vm_physmem[lcv].free_list != freelist)
! 519: continue;
! 520:
! 521: /* try from front */
! 522: if (vm_physmem[lcv].avail_start == vm_physmem[lcv].start &&
! 523: vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
! 524: *paddrp = ptoa(vm_physmem[lcv].avail_start);
! 525: vm_physmem[lcv].avail_start++;
! 526: vm_physmem[lcv].start++;
! 527: /* nothing left? nuke it */
! 528: if (vm_physmem[lcv].avail_start ==
! 529: vm_physmem[lcv].end) {
! 530: if (vm_nphysseg == 1)
! 531: panic("uvm_page_physget: out of memory!");
! 532: vm_nphysseg--;
! 533: for (x = lcv ; x < vm_nphysseg ; x++)
! 534: /* structure copy */
! 535: vm_physmem[x] = vm_physmem[x+1];
! 536: }
! 537: return (TRUE);
! 538: }
! 539:
! 540: /* try from rear */
! 541: if (vm_physmem[lcv].avail_end == vm_physmem[lcv].end &&
! 542: vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
! 543: *paddrp = ptoa(vm_physmem[lcv].avail_end - 1);
! 544: vm_physmem[lcv].avail_end--;
! 545: vm_physmem[lcv].end--;
! 546: /* nothing left? nuke it */
! 547: if (vm_physmem[lcv].avail_end ==
! 548: vm_physmem[lcv].start) {
! 549: if (vm_nphysseg == 1)
! 550: panic("uvm_page_physget: out of memory!");
! 551: vm_nphysseg--;
! 552: for (x = lcv ; x < vm_nphysseg ; x++)
! 553: /* structure copy */
! 554: vm_physmem[x] = vm_physmem[x+1];
! 555: }
! 556: return (TRUE);
! 557: }
! 558: }
! 559:
! 560: /* pass2: forget about matching ends, just allocate something */
! 561: #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) || \
! 562: (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
! 563: for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
! 564: #else
! 565: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 566: #endif
! 567: {
! 568:
! 569: /* any room in this bank? */
! 570: if (vm_physmem[lcv].avail_start >= vm_physmem[lcv].avail_end)
! 571: continue; /* nope */
! 572:
! 573: *paddrp = ptoa(vm_physmem[lcv].avail_start);
! 574: vm_physmem[lcv].avail_start++;
! 575: /* truncate! */
! 576: vm_physmem[lcv].start = vm_physmem[lcv].avail_start;
! 577:
! 578: /* nothing left? nuke it */
! 579: if (vm_physmem[lcv].avail_start == vm_physmem[lcv].end) {
! 580: if (vm_nphysseg == 1)
! 581: panic("uvm_page_physget: out of memory!");
! 582: vm_nphysseg--;
! 583: for (x = lcv ; x < vm_nphysseg ; x++)
! 584: /* structure copy */
! 585: vm_physmem[x] = vm_physmem[x+1];
! 586: }
! 587: return (TRUE);
! 588: }
! 589:
! 590: return (FALSE); /* whoops! */
! 591: }
! 592:
! 593: boolean_t
! 594: uvm_page_physget(paddrp)
! 595: paddr_t *paddrp;
! 596: {
! 597: int i;
! 598: UVMHIST_FUNC("uvm_page_physget"); UVMHIST_CALLED(pghist);
! 599:
! 600: /* try in the order of freelist preference */
! 601: for (i = 0; i < VM_NFREELIST; i++)
! 602: if (uvm_page_physget_freelist(paddrp, i) == TRUE)
! 603: return (TRUE);
! 604: return (FALSE);
! 605: }
! 606: #endif /* PMAP_STEAL_MEMORY */
! 607:
! 608: /*
! 609: * uvm_page_physload: load physical memory into VM system
! 610: *
! 611: * => all args are PFs
! 612: * => all pages in start/end get vm_page structures
! 613: * => areas marked by avail_start/avail_end get added to the free page pool
! 614: * => we are limited to VM_PHYSSEG_MAX physical memory segments
! 615: */
! 616:
! 617: void
! 618: uvm_page_physload(start, end, avail_start, avail_end, free_list)
! 619: paddr_t start, end, avail_start, avail_end;
! 620: int free_list;
! 621: {
! 622: int preload, lcv;
! 623: psize_t npages;
! 624: struct vm_page *pgs;
! 625: struct vm_physseg *ps;
! 626:
! 627: if (uvmexp.pagesize == 0)
! 628: panic("uvm_page_physload: page size not set!");
! 629:
! 630: if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
! 631: panic("uvm_page_physload: bad free list %d", free_list);
! 632:
! 633: if (start >= end)
! 634: panic("uvm_page_physload: start >= end");
! 635:
! 636: /*
! 637: * do we have room?
! 638: */
! 639: if (vm_nphysseg == VM_PHYSSEG_MAX) {
! 640: printf("uvm_page_physload: unable to load physical memory "
! 641: "segment\n");
! 642: printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
! 643: VM_PHYSSEG_MAX, (long long)start, (long long)end);
! 644: printf("\tincrease VM_PHYSSEG_MAX\n");
! 645: return;
! 646: }
! 647:
! 648: /*
! 649: * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
! 650: * called yet, so malloc is not available).
! 651: */
! 652: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
! 653: if (vm_physmem[lcv].pgs)
! 654: break;
! 655: }
! 656: preload = (lcv == vm_nphysseg);
! 657:
! 658: /*
! 659: * if VM is already running, attempt to malloc() vm_page structures
! 660: */
! 661: if (!preload) {
! 662: #if defined(VM_PHYSSEG_NOADD)
! 663: panic("uvm_page_physload: tried to add RAM after vm_mem_init");
! 664: #else
! 665: /* XXXCDC: need some sort of lockout for this case */
! 666: paddr_t paddr;
! 667: npages = end - start; /* # of pages */
! 668: pgs = (vm_page *)uvm_km_alloc(kernel_map,
! 669: sizeof(struct vm_page) * npages);
! 670: if (pgs == NULL) {
! 671: printf("uvm_page_physload: can not malloc vm_page "
! 672: "structs for segment\n");
! 673: printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
! 674: return;
! 675: }
! 676: /* zero data, init phys_addr and free_list, and free pages */
! 677: memset(pgs, 0, sizeof(struct vm_page) * npages);
! 678: for (lcv = 0, paddr = ptoa(start) ;
! 679: lcv < npages ; lcv++, paddr += PAGE_SIZE) {
! 680: pgs[lcv].phys_addr = paddr;
! 681: pgs[lcv].free_list = free_list;
! 682: if (atop(paddr) >= avail_start &&
! 683: atop(paddr) <= avail_end)
! 684: uvm_pagefree(&pgs[lcv]);
! 685: }
! 686: /* XXXCDC: incomplete: need to update uvmexp.free, what else? */
! 687: /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
! 688: #endif
! 689: } else {
! 690:
! 691: /* gcc complains if these don't get init'd */
! 692: pgs = NULL;
! 693: npages = 0;
! 694:
! 695: }
! 696:
! 697: /*
! 698: * now insert us in the proper place in vm_physmem[]
! 699: */
! 700:
! 701: #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
! 702:
! 703: /* random: put it at the end (easy!) */
! 704: ps = &vm_physmem[vm_nphysseg];
! 705:
! 706: #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
! 707:
! 708: {
! 709: int x;
! 710: /* sort by address for binary search */
! 711: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 712: if (start < vm_physmem[lcv].start)
! 713: break;
! 714: ps = &vm_physmem[lcv];
! 715: /* move back other entries, if necessary ... */
! 716: for (x = vm_nphysseg ; x > lcv ; x--)
! 717: /* structure copy */
! 718: vm_physmem[x] = vm_physmem[x - 1];
! 719: }
! 720:
! 721: #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
! 722:
! 723: {
! 724: int x;
! 725: /* sort by largest segment first */
! 726: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 727: if ((end - start) >
! 728: (vm_physmem[lcv].end - vm_physmem[lcv].start))
! 729: break;
! 730: ps = &vm_physmem[lcv];
! 731: /* move back other entries, if necessary ... */
! 732: for (x = vm_nphysseg ; x > lcv ; x--)
! 733: /* structure copy */
! 734: vm_physmem[x] = vm_physmem[x - 1];
! 735: }
! 736:
! 737: #else
! 738:
! 739: panic("uvm_page_physload: unknown physseg strategy selected!");
! 740:
! 741: #endif
! 742:
! 743: ps->start = start;
! 744: ps->end = end;
! 745: ps->avail_start = avail_start;
! 746: ps->avail_end = avail_end;
! 747: if (preload) {
! 748: ps->pgs = NULL;
! 749: } else {
! 750: ps->pgs = pgs;
! 751: ps->lastpg = pgs + npages - 1;
! 752: }
! 753: ps->free_list = free_list;
! 754: vm_nphysseg++;
! 755:
! 756: /*
! 757: * done!
! 758: */
! 759:
! 760: if (!preload)
! 761: uvm_page_rehash();
! 762:
! 763: return;
! 764: }
! 765:
! 766: /*
! 767: * uvm_page_rehash: reallocate hash table based on number of free pages.
! 768: */
! 769:
! 770: void
! 771: uvm_page_rehash()
! 772: {
! 773: int freepages, lcv, bucketcount, s, oldcount;
! 774: struct pglist *newbuckets, *oldbuckets;
! 775: struct vm_page *pg;
! 776: size_t newsize, oldsize;
! 777:
! 778: /*
! 779: * compute number of pages that can go in the free pool
! 780: */
! 781:
! 782: freepages = 0;
! 783: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 784: freepages +=
! 785: (vm_physmem[lcv].avail_end - vm_physmem[lcv].avail_start);
! 786:
! 787: /*
! 788: * compute number of buckets needed for this number of pages
! 789: */
! 790:
! 791: bucketcount = 1;
! 792: while (bucketcount < freepages)
! 793: bucketcount = bucketcount * 2;
! 794:
! 795: /*
! 796: * compute the size of the current table and new table.
! 797: */
! 798:
! 799: oldbuckets = uvm.page_hash;
! 800: oldcount = uvm.page_nhash;
! 801: oldsize = round_page(sizeof(struct pglist) * oldcount);
! 802: newsize = round_page(sizeof(struct pglist) * bucketcount);
! 803:
! 804: /*
! 805: * allocate the new buckets
! 806: */
! 807:
! 808: newbuckets = (struct pglist *) uvm_km_alloc(kernel_map, newsize);
! 809: if (newbuckets == NULL) {
! 810: printf("uvm_page_physrehash: WARNING: could not grow page "
! 811: "hash table\n");
! 812: return;
! 813: }
! 814: for (lcv = 0 ; lcv < bucketcount ; lcv++)
! 815: TAILQ_INIT(&newbuckets[lcv]);
! 816:
! 817: /*
! 818: * now replace the old buckets with the new ones and rehash everything
! 819: */
! 820:
! 821: s = splvm();
! 822: simple_lock(&uvm.hashlock);
! 823: uvm.page_hash = newbuckets;
! 824: uvm.page_nhash = bucketcount;
! 825: uvm.page_hashmask = bucketcount - 1; /* power of 2 */
! 826:
! 827: /* ... and rehash */
! 828: for (lcv = 0 ; lcv < oldcount ; lcv++) {
! 829: while ((pg = TAILQ_FIRST(&oldbuckets[lcv])) != NULL) {
! 830: TAILQ_REMOVE(&oldbuckets[lcv], pg, hashq);
! 831: TAILQ_INSERT_TAIL(
! 832: &uvm.page_hash[uvm_pagehash(pg->uobject, pg->offset)],
! 833: pg, hashq);
! 834: }
! 835: }
! 836: simple_unlock(&uvm.hashlock);
! 837: splx(s);
! 838:
! 839: /*
! 840: * free old bucket array if is not the boot-time table
! 841: */
! 842:
! 843: if (oldbuckets != &uvm_bootbucket)
! 844: uvm_km_free(kernel_map, (vaddr_t) oldbuckets, oldsize);
! 845:
! 846: /*
! 847: * done
! 848: */
! 849: return;
! 850: }
! 851:
! 852:
! 853: #if 1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */
! 854:
! 855: void uvm_page_physdump(void); /* SHUT UP GCC */
! 856:
! 857: /* call from DDB */
! 858: void
! 859: uvm_page_physdump()
! 860: {
! 861: int lcv;
! 862:
! 863: printf("rehash: physical memory config [segs=%d of %d]:\n",
! 864: vm_nphysseg, VM_PHYSSEG_MAX);
! 865: for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
! 866: printf("0x%llx->0x%llx [0x%llx->0x%llx]\n",
! 867: (long long)vm_physmem[lcv].start,
! 868: (long long)vm_physmem[lcv].end,
! 869: (long long)vm_physmem[lcv].avail_start,
! 870: (long long)vm_physmem[lcv].avail_end);
! 871: printf("STRATEGY = ");
! 872: switch (VM_PHYSSEG_STRAT) {
! 873: case VM_PSTRAT_RANDOM: printf("RANDOM\n"); break;
! 874: case VM_PSTRAT_BSEARCH: printf("BSEARCH\n"); break;
! 875: case VM_PSTRAT_BIGFIRST: printf("BIGFIRST\n"); break;
! 876: default: printf("<<UNKNOWN>>!!!!\n");
! 877: }
! 878: printf("number of buckets = %d\n", uvm.page_nhash);
! 879: }
! 880: #endif
! 881:
! 882: /*
! 883: * uvm_pagealloc_strat: allocate vm_page from a particular free list.
! 884: *
! 885: * => return null if no pages free
! 886: * => wake up pagedaemon if number of free pages drops below low water mark
! 887: * => if obj != NULL, obj must be locked (to put in hash)
! 888: * => if anon != NULL, anon must be locked (to put in anon)
! 889: * => only one of obj or anon can be non-null
! 890: * => caller must activate/deactivate page if it is not wired.
! 891: * => free_list is ignored if strat == UVM_PGA_STRAT_NORMAL.
! 892: * => policy decision: it is more important to pull a page off of the
! 893: * appropriate priority free list than it is to get a zero'd or
! 894: * unknown contents page. This is because we live with the
! 895: * consequences of a bad free list decision for the entire
! 896: * lifetime of the page, e.g. if the page comes from memory that
! 897: * is slower to access.
! 898: */
! 899:
! 900: struct vm_page *
! 901: uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
! 902: struct uvm_object *obj;
! 903: voff_t off;
! 904: int flags;
! 905: struct vm_anon *anon;
! 906: int strat, free_list;
! 907: {
! 908: int lcv, try1, try2, s, zeroit = 0;
! 909: struct vm_page *pg;
! 910: struct pglist *freeq;
! 911: struct pgfreelist *pgfl;
! 912: boolean_t use_reserve;
! 913: UVMHIST_FUNC("uvm_pagealloc_strat"); UVMHIST_CALLED(pghist);
! 914:
! 915: KASSERT(obj == NULL || anon == NULL);
! 916: KASSERT(off == trunc_page(off));
! 917: s = uvm_lock_fpageq();
! 918:
! 919: /*
! 920: * check to see if we need to generate some free pages waking
! 921: * the pagedaemon.
! 922: */
! 923:
! 924: #ifdef UBC
! 925: if (uvmexp.free + uvmexp.paging < uvmexp.freemin ||
! 926: (uvmexp.free + uvmexp.paging < uvmexp.freetarg &&
! 927: uvmexp.inactive < uvmexp.inactarg)) {
! 928: wakeup(&uvm.pagedaemon);
! 929: }
! 930: #else
! 931: if (uvmexp.free < uvmexp.freemin || (uvmexp.free < uvmexp.freetarg &&
! 932: uvmexp.inactive < uvmexp.inactarg))
! 933: wakeup(&uvm.pagedaemon);
! 934: #endif
! 935:
! 936: /*
! 937: * fail if any of these conditions is true:
! 938: * [1] there really are no free pages, or
! 939: * [2] only kernel "reserved" pages remain and
! 940: * the page isn't being allocated to a kernel object.
! 941: * [3] only pagedaemon "reserved" pages remain and
! 942: * the requestor isn't the pagedaemon.
! 943: */
! 944:
! 945: use_reserve = (flags & UVM_PGA_USERESERVE) ||
! 946: (obj && UVM_OBJ_IS_KERN_OBJECT(obj));
! 947: if ((uvmexp.free <= uvmexp.reserve_kernel && !use_reserve) ||
! 948: (uvmexp.free <= uvmexp.reserve_pagedaemon &&
! 949: !(use_reserve && (curproc == uvm.pagedaemon_proc ||
! 950: curproc == syncerproc))))
! 951: goto fail;
! 952:
! 953: #if PGFL_NQUEUES != 2
! 954: #error uvm_pagealloc_strat needs to be updated
! 955: #endif
! 956:
! 957: /*
! 958: * If we want a zero'd page, try the ZEROS queue first, otherwise
! 959: * we try the UNKNOWN queue first.
! 960: */
! 961: if (flags & UVM_PGA_ZERO) {
! 962: try1 = PGFL_ZEROS;
! 963: try2 = PGFL_UNKNOWN;
! 964: } else {
! 965: try1 = PGFL_UNKNOWN;
! 966: try2 = PGFL_ZEROS;
! 967: }
! 968:
! 969: UVMHIST_LOG(pghist, "obj=%p off=%lx anon=%p flags=%lx",
! 970: obj, (u_long)off, anon, flags);
! 971: UVMHIST_LOG(pghist, "strat=%ld free_list=%ld", strat, free_list, 0, 0);
! 972: again:
! 973: switch (strat) {
! 974: case UVM_PGA_STRAT_NORMAL:
! 975: /* Check all freelists in descending priority order. */
! 976: for (lcv = 0; lcv < VM_NFREELIST; lcv++) {
! 977: pgfl = &uvm.page_free[lcv];
! 978: if ((pg = TAILQ_FIRST((freeq =
! 979: &pgfl->pgfl_queues[try1]))) != NULL ||
! 980: (pg = TAILQ_FIRST((freeq =
! 981: &pgfl->pgfl_queues[try2]))) != NULL)
! 982: goto gotit;
! 983: }
! 984:
! 985: /* No pages free! */
! 986: goto fail;
! 987:
! 988: case UVM_PGA_STRAT_ONLY:
! 989: case UVM_PGA_STRAT_FALLBACK:
! 990: /* Attempt to allocate from the specified free list. */
! 991: KASSERT(free_list >= 0 && free_list < VM_NFREELIST);
! 992: pgfl = &uvm.page_free[free_list];
! 993: if ((pg = TAILQ_FIRST((freeq =
! 994: &pgfl->pgfl_queues[try1]))) != NULL ||
! 995: (pg = TAILQ_FIRST((freeq =
! 996: &pgfl->pgfl_queues[try2]))) != NULL)
! 997: goto gotit;
! 998:
! 999: /* Fall back, if possible. */
! 1000: if (strat == UVM_PGA_STRAT_FALLBACK) {
! 1001: strat = UVM_PGA_STRAT_NORMAL;
! 1002: goto again;
! 1003: }
! 1004:
! 1005: /* No pages free! */
! 1006: goto fail;
! 1007:
! 1008: default:
! 1009: panic("uvm_pagealloc_strat: bad strat %d", strat);
! 1010: /* NOTREACHED */
! 1011: }
! 1012:
! 1013: gotit:
! 1014: TAILQ_REMOVE(freeq, pg, pageq);
! 1015: uvmexp.free--;
! 1016:
! 1017: /* update zero'd page count */
! 1018: if (pg->pg_flags & PG_ZERO)
! 1019: uvmexp.zeropages--;
! 1020:
! 1021: /*
! 1022: * update allocation statistics and remember if we have to
! 1023: * zero the page
! 1024: */
! 1025: if (flags & UVM_PGA_ZERO) {
! 1026: if (pg->pg_flags & PG_ZERO) {
! 1027: uvmexp.pga_zerohit++;
! 1028: zeroit = 0;
! 1029: } else {
! 1030: uvmexp.pga_zeromiss++;
! 1031: zeroit = 1;
! 1032: }
! 1033: }
! 1034:
! 1035: uvm_unlock_fpageq(s); /* unlock free page queue */
! 1036:
! 1037: pg->offset = off;
! 1038: pg->uobject = obj;
! 1039: pg->uanon = anon;
! 1040: pg->pg_flags = PG_BUSY|PG_CLEAN|PG_FAKE;
! 1041: pg->pg_version++;
! 1042: if (anon) {
! 1043: anon->an_page = pg;
! 1044: atomic_setbits_int(&pg->pg_flags, PQ_ANON);
! 1045: #ifdef UBC
! 1046: uvm_pgcnt_anon++;
! 1047: #endif
! 1048: } else {
! 1049: if (obj)
! 1050: uvm_pageinsert(pg);
! 1051: }
! 1052: #if defined(UVM_PAGE_TRKOWN)
! 1053: pg->owner_tag = NULL;
! 1054: #endif
! 1055: UVM_PAGE_OWN(pg, "new alloc");
! 1056:
! 1057: if (flags & UVM_PGA_ZERO) {
! 1058: /*
! 1059: * A zero'd page is not clean. If we got a page not already
! 1060: * zero'd, then we have to zero it ourselves.
! 1061: */
! 1062: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
! 1063: if (zeroit)
! 1064: pmap_zero_page(pg);
! 1065: }
! 1066:
! 1067: UVMHIST_LOG(pghist, "allocated pg %p/%lx", pg,
! 1068: (u_long)VM_PAGE_TO_PHYS(pg), 0, 0);
! 1069: return(pg);
! 1070:
! 1071: fail:
! 1072: uvm_unlock_fpageq(s);
! 1073: UVMHIST_LOG(pghist, "failed!", 0, 0, 0, 0);
! 1074: return (NULL);
! 1075: }
! 1076:
! 1077: /*
! 1078: * uvm_pagerealloc: reallocate a page from one object to another
! 1079: *
! 1080: * => both objects must be locked
! 1081: */
! 1082:
! 1083: void
! 1084: uvm_pagerealloc(pg, newobj, newoff)
! 1085: struct vm_page *pg;
! 1086: struct uvm_object *newobj;
! 1087: voff_t newoff;
! 1088: {
! 1089:
! 1090: UVMHIST_FUNC("uvm_pagerealloc"); UVMHIST_CALLED(pghist);
! 1091:
! 1092: /*
! 1093: * remove it from the old object
! 1094: */
! 1095:
! 1096: if (pg->uobject) {
! 1097: uvm_pageremove(pg);
! 1098: }
! 1099:
! 1100: /*
! 1101: * put it in the new object
! 1102: */
! 1103:
! 1104: if (newobj) {
! 1105: pg->uobject = newobj;
! 1106: pg->offset = newoff;
! 1107: pg->pg_version++;
! 1108: uvm_pageinsert(pg);
! 1109: }
! 1110: }
! 1111:
! 1112:
! 1113: /*
! 1114: * uvm_pagefree: free page
! 1115: *
! 1116: * => erase page's identity (i.e. remove from hash/object)
! 1117: * => put page on free list
! 1118: * => caller must lock owning object (either anon or uvm_object)
! 1119: * => caller must lock page queues
! 1120: * => assumes all valid mappings of pg are gone
! 1121: */
! 1122:
! 1123: void
! 1124: uvm_pagefree(struct vm_page *pg)
! 1125: {
! 1126: int s;
! 1127: int saved_loan_count = pg->loan_count;
! 1128: UVMHIST_FUNC("uvm_pagefree"); UVMHIST_CALLED(pghist);
! 1129:
! 1130: #ifdef DEBUG
! 1131: if (pg->uobject == (void *)0xdeadbeef &&
! 1132: pg->uanon == (void *)0xdeadbeef) {
! 1133: panic("uvm_pagefree: freeing free page %p", pg);
! 1134: }
! 1135: #endif
! 1136:
! 1137: UVMHIST_LOG(pghist, "freeing pg %p/%lx", pg,
! 1138: (u_long)VM_PAGE_TO_PHYS(pg), 0, 0);
! 1139:
! 1140: /*
! 1141: * if the page was an object page (and thus "TABLED"), remove it
! 1142: * from the object.
! 1143: */
! 1144:
! 1145: if (pg->pg_flags & PG_TABLED) {
! 1146:
! 1147: /*
! 1148: * if the object page is on loan we are going to drop ownership.
! 1149: * it is possible that an anon will take over as owner for this
! 1150: * page later on. the anon will want a !PG_CLEAN page so that
! 1151: * it knows it needs to allocate swap if it wants to page the
! 1152: * page out.
! 1153: */
! 1154:
! 1155: /* in case an anon takes over */
! 1156: if (saved_loan_count)
! 1157: atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
! 1158: uvm_pageremove(pg);
! 1159:
! 1160: /*
! 1161: * if our page was on loan, then we just lost control over it
! 1162: * (in fact, if it was loaned to an anon, the anon may have
! 1163: * already taken over ownership of the page by now and thus
! 1164: * changed the loan_count [e.g. in uvmfault_anonget()]) we just
! 1165: * return (when the last loan is dropped, then the page can be
! 1166: * freed by whatever was holding the last loan).
! 1167: */
! 1168:
! 1169: if (saved_loan_count)
! 1170: return;
! 1171: } else if (saved_loan_count && pg->uanon) {
! 1172: /*
! 1173: * if our page is owned by an anon and is loaned out to the
! 1174: * kernel then we just want to drop ownership and return.
! 1175: * the kernel must free the page when all its loans clear ...
! 1176: * note that the kernel can't change the loan status of our
! 1177: * page as long as we are holding PQ lock.
! 1178: */
! 1179: atomic_clearbits_int(&pg->pg_flags, PQ_ANON);
! 1180: pg->uanon->an_page = NULL;
! 1181: pg->uanon = NULL;
! 1182: return;
! 1183: }
! 1184: KASSERT(saved_loan_count == 0);
! 1185:
! 1186: /*
! 1187: * now remove the page from the queues
! 1188: */
! 1189:
! 1190: if (pg->pg_flags & PQ_ACTIVE) {
! 1191: TAILQ_REMOVE(&uvm.page_active, pg, pageq);
! 1192: atomic_clearbits_int(&pg->pg_flags, PQ_ACTIVE);
! 1193: uvmexp.active--;
! 1194: }
! 1195: if (pg->pg_flags & PQ_INACTIVE) {
! 1196: if (pg->pg_flags & PQ_SWAPBACKED)
! 1197: TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq);
! 1198: else
! 1199: TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq);
! 1200: atomic_clearbits_int(&pg->pg_flags, PQ_INACTIVE);
! 1201: uvmexp.inactive--;
! 1202: }
! 1203:
! 1204: /*
! 1205: * if the page was wired, unwire it now.
! 1206: */
! 1207:
! 1208: if (pg->wire_count) {
! 1209: pg->wire_count = 0;
! 1210: uvmexp.wired--;
! 1211: }
! 1212: if (pg->uanon) {
! 1213: pg->uanon->an_page = NULL;
! 1214: #ifdef UBC
! 1215: uvm_pgcnt_anon--;
! 1216: #endif
! 1217: }
! 1218:
! 1219: /*
! 1220: * and put on free queue
! 1221: */
! 1222:
! 1223: atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
! 1224:
! 1225: s = uvm_lock_fpageq();
! 1226: TAILQ_INSERT_TAIL(&uvm.page_free[
! 1227: uvm_page_lookup_freelist(pg)].pgfl_queues[PGFL_UNKNOWN], pg, pageq);
! 1228: atomic_clearbits_int(&pg->pg_flags, PQ_MASK);
! 1229: atomic_setbits_int(&pg->pg_flags, PQ_FREE);
! 1230: #ifdef DEBUG
! 1231: pg->uobject = (void *)0xdeadbeef;
! 1232: pg->offset = 0xdeadbeef;
! 1233: pg->uanon = (void *)0xdeadbeef;
! 1234: #endif
! 1235: uvmexp.free++;
! 1236:
! 1237: if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
! 1238: uvm.page_idle_zero = vm_page_zero_enable;
! 1239:
! 1240: uvm_unlock_fpageq(s);
! 1241: }
! 1242:
! 1243: /*
! 1244: * uvm_page_unbusy: unbusy an array of pages.
! 1245: *
! 1246: * => pages must either all belong to the same object, or all belong to anons.
! 1247: * => if pages are object-owned, object must be locked.
! 1248: * => if pages are anon-owned, anons must be unlockd and have 0 refcount.
! 1249: */
! 1250:
! 1251: void
! 1252: uvm_page_unbusy(pgs, npgs)
! 1253: struct vm_page **pgs;
! 1254: int npgs;
! 1255: {
! 1256: struct vm_page *pg;
! 1257: struct uvm_object *uobj;
! 1258: int i;
! 1259: UVMHIST_FUNC("uvm_page_unbusy"); UVMHIST_CALLED(pdhist);
! 1260:
! 1261: for (i = 0; i < npgs; i++) {
! 1262: pg = pgs[i];
! 1263:
! 1264: if (pg == NULL || pg == PGO_DONTCARE) {
! 1265: continue;
! 1266: }
! 1267: if (pg->pg_flags & PG_WANTED) {
! 1268: wakeup(pg);
! 1269: }
! 1270: if (pg->pg_flags & PG_RELEASED) {
! 1271: UVMHIST_LOG(pdhist, "releasing pg %p", pg,0,0,0);
! 1272: uobj = pg->uobject;
! 1273: if (uobj != NULL) {
! 1274: uobj->pgops->pgo_releasepg(pg, NULL);
! 1275: } else {
! 1276: atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
! 1277: UVM_PAGE_OWN(pg, NULL);
! 1278: uvm_anfree(pg->uanon);
! 1279: }
! 1280: } else {
! 1281: UVMHIST_LOG(pdhist, "unbusying pg %p", pg,0,0,0);
! 1282: atomic_clearbits_int(&pg->pg_flags, PG_WANTED|PG_BUSY);
! 1283: UVM_PAGE_OWN(pg, NULL);
! 1284: }
! 1285: }
! 1286: }
! 1287:
! 1288: #if defined(UVM_PAGE_TRKOWN)
! 1289: /*
! 1290: * uvm_page_own: set or release page ownership
! 1291: *
! 1292: * => this is a debugging function that keeps track of who sets PG_BUSY
! 1293: * and where they do it. it can be used to track down problems
! 1294: * such a process setting "PG_BUSY" and never releasing it.
! 1295: * => page's object [if any] must be locked
! 1296: * => if "tag" is NULL then we are releasing page ownership
! 1297: */
! 1298: void
! 1299: uvm_page_own(pg, tag)
! 1300: struct vm_page *pg;
! 1301: char *tag;
! 1302: {
! 1303: /* gain ownership? */
! 1304: if (tag) {
! 1305: if (pg->owner_tag) {
! 1306: printf("uvm_page_own: page %p already owned "
! 1307: "by proc %d [%s]\n", pg,
! 1308: pg->owner, pg->owner_tag);
! 1309: panic("uvm_page_own");
! 1310: }
! 1311: pg->owner = (curproc) ? curproc->p_pid : (pid_t) -1;
! 1312: pg->owner_tag = tag;
! 1313: return;
! 1314: }
! 1315:
! 1316: /* drop ownership */
! 1317: if (pg->owner_tag == NULL) {
! 1318: printf("uvm_page_own: dropping ownership of an non-owned "
! 1319: "page (%p)\n", pg);
! 1320: panic("uvm_page_own");
! 1321: }
! 1322: pg->owner_tag = NULL;
! 1323: return;
! 1324: }
! 1325: #endif
! 1326:
! 1327: /*
! 1328: * uvm_pageidlezero: zero free pages while the system is idle.
! 1329: *
! 1330: * => we do at least one iteration per call, if we are below the target.
! 1331: * => we loop until we either reach the target or whichqs indicates that
! 1332: * there is a process ready to run.
! 1333: */
! 1334: void
! 1335: uvm_pageidlezero()
! 1336: {
! 1337: struct vm_page *pg;
! 1338: struct pgfreelist *pgfl;
! 1339: int free_list, s;
! 1340: UVMHIST_FUNC("uvm_pageidlezero"); UVMHIST_CALLED(pghist);
! 1341:
! 1342: do {
! 1343: s = uvm_lock_fpageq();
! 1344:
! 1345: if (uvmexp.zeropages >= UVM_PAGEZERO_TARGET) {
! 1346: uvm.page_idle_zero = FALSE;
! 1347: uvm_unlock_fpageq(s);
! 1348: return;
! 1349: }
! 1350:
! 1351: for (free_list = 0; free_list < VM_NFREELIST; free_list++) {
! 1352: pgfl = &uvm.page_free[free_list];
! 1353: if ((pg = TAILQ_FIRST(&pgfl->pgfl_queues[
! 1354: PGFL_UNKNOWN])) != NULL)
! 1355: break;
! 1356: }
! 1357:
! 1358: if (pg == NULL) {
! 1359: /*
! 1360: * No non-zero'd pages; don't bother trying again
! 1361: * until we know we have non-zero'd pages free.
! 1362: */
! 1363: uvm.page_idle_zero = FALSE;
! 1364: uvm_unlock_fpageq(s);
! 1365: return;
! 1366: }
! 1367:
! 1368: TAILQ_REMOVE(&pgfl->pgfl_queues[PGFL_UNKNOWN], pg, pageq);
! 1369: uvmexp.free--;
! 1370: uvm_unlock_fpageq(s);
! 1371:
! 1372: #ifdef PMAP_PAGEIDLEZERO
! 1373: if (PMAP_PAGEIDLEZERO(pg) == FALSE) {
! 1374: /*
! 1375: * The machine-dependent code detected some
! 1376: * reason for us to abort zeroing pages,
! 1377: * probably because there is a process now
! 1378: * ready to run.
! 1379: */
! 1380: s = uvm_lock_fpageq();
! 1381: TAILQ_INSERT_HEAD(&pgfl->pgfl_queues[PGFL_UNKNOWN],
! 1382: pg, pageq);
! 1383: uvmexp.free++;
! 1384: uvmexp.zeroaborts++;
! 1385: uvm_unlock_fpageq(s);
! 1386: return;
! 1387: }
! 1388: #else
! 1389: /*
! 1390: * XXX This will toast the cache unless the pmap_zero_page()
! 1391: * XXX implementation does uncached access.
! 1392: */
! 1393: pmap_zero_page(pg);
! 1394: #endif
! 1395: atomic_setbits_int(&pg->pg_flags, PG_ZERO);
! 1396:
! 1397: s = uvm_lock_fpageq();
! 1398: TAILQ_INSERT_HEAD(&pgfl->pgfl_queues[PGFL_ZEROS], pg, pageq);
! 1399: uvmexp.free++;
! 1400: uvmexp.zeropages++;
! 1401: uvm_unlock_fpageq(s);
! 1402: } while (sched_is_idle());
! 1403: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_page.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm