Annotation of sys/uvm/uvm_aobj.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: uvm_aobj.c,v 1.34 2007/04/13 18:57:49 art Exp $ */
! 2: /* $NetBSD: uvm_aobj.c,v 1.39 2001/02/18 21:19:08 chs Exp $ */
! 3:
! 4: /*
! 5: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
! 6: * Washington University.
! 7: * All rights reserved.
! 8: *
! 9: * Redistribution and use in source and binary forms, with or without
! 10: * modification, are permitted provided that the following conditions
! 11: * are met:
! 12: * 1. Redistributions of source code must retain the above copyright
! 13: * notice, this list of conditions and the following disclaimer.
! 14: * 2. Redistributions in binary form must reproduce the above copyright
! 15: * notice, this list of conditions and the following disclaimer in the
! 16: * documentation and/or other materials provided with the distribution.
! 17: * 3. All advertising materials mentioning features or use of this software
! 18: * must display the following acknowledgement:
! 19: * This product includes software developed by Charles D. Cranor and
! 20: * Washington University.
! 21: * 4. The name of the author may not be used to endorse or promote products
! 22: * derived from this software without specific prior written permission.
! 23: *
! 24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
! 25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
! 26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
! 27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
! 28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
! 29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
! 30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
! 31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
! 32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
! 33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
! 34: *
! 35: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
! 36: */
! 37: /*
! 38: * uvm_aobj.c: anonymous memory uvm_object pager
! 39: *
! 40: * author: Chuck Silvers <chuq@chuq.com>
! 41: * started: Jan-1998
! 42: *
! 43: * - design mostly from Chuck Cranor
! 44: */
! 45:
! 46: #include <sys/param.h>
! 47: #include <sys/systm.h>
! 48: #include <sys/proc.h>
! 49: #include <sys/malloc.h>
! 50: #include <sys/kernel.h>
! 51: #include <sys/pool.h>
! 52: #include <sys/kernel.h>
! 53:
! 54: #include <uvm/uvm.h>
! 55:
! 56: /*
! 57: * an aobj manages anonymous-memory backed uvm_objects. in addition
! 58: * to keeping the list of resident pages, it also keeps a list of
! 59: * allocated swap blocks. depending on the size of the aobj this list
! 60: * of allocated swap blocks is either stored in an array (small objects)
! 61: * or in a hash table (large objects).
! 62: */
! 63:
! 64: /*
! 65: * local structures
! 66: */
! 67:
! 68: /*
! 69: * for hash tables, we break the address space of the aobj into blocks
! 70: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
! 71: * be a power of two.
! 72: */
! 73:
! 74: #define UAO_SWHASH_CLUSTER_SHIFT 4
! 75: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
! 76:
! 77: /* get the "tag" for this page index */
! 78: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
! 79: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
! 80:
! 81: /* given an ELT and a page index, find the swap slot */
! 82: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
! 83: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
! 84:
! 85: /* given an ELT, return its pageidx base */
! 86: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
! 87: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
! 88:
! 89: /*
! 90: * the swhash hash function
! 91: */
! 92: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
! 93: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
! 94: & (AOBJ)->u_swhashmask)])
! 95:
! 96: /*
! 97: * the swhash threshold determines if we will use an array or a
! 98: * hash table to store the list of allocated swap blocks.
! 99: */
! 100:
! 101: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
! 102: #define UAO_USES_SWHASH(AOBJ) \
! 103: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
! 104:
! 105: /*
! 106: * the number of buckets in a swhash, with an upper bound
! 107: */
! 108: #define UAO_SWHASH_MAXBUCKETS 256
! 109: #define UAO_SWHASH_BUCKETS(AOBJ) \
! 110: (min((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
! 111: UAO_SWHASH_MAXBUCKETS))
! 112:
! 113:
! 114: /*
! 115: * uao_swhash_elt: when a hash table is being used, this structure defines
! 116: * the format of an entry in the bucket list.
! 117: */
! 118:
! 119: struct uao_swhash_elt {
! 120: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
! 121: voff_t tag; /* our 'tag' */
! 122: int count; /* our number of active slots */
! 123: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
! 124: };
! 125:
! 126: /*
! 127: * uao_swhash: the swap hash table structure
! 128: */
! 129:
! 130: LIST_HEAD(uao_swhash, uao_swhash_elt);
! 131:
! 132: /*
! 133: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
! 134: */
! 135:
! 136: struct pool uao_swhash_elt_pool;
! 137:
! 138: /*
! 139: * uvm_aobj: the actual anon-backed uvm_object
! 140: *
! 141: * => the uvm_object is at the top of the structure, this allows
! 142: * (struct uvm_device *) == (struct uvm_object *)
! 143: * => only one of u_swslots and u_swhash is used in any given aobj
! 144: */
! 145:
! 146: struct uvm_aobj {
! 147: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
! 148: int u_pages; /* number of pages in entire object */
! 149: int u_flags; /* the flags (see uvm_aobj.h) */
! 150: int *u_swslots; /* array of offset->swapslot mappings */
! 151: /*
! 152: * hashtable of offset->swapslot mappings
! 153: * (u_swhash is an array of bucket heads)
! 154: */
! 155: struct uao_swhash *u_swhash;
! 156: u_long u_swhashmask; /* mask for hashtable */
! 157: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
! 158: };
! 159:
! 160: /*
! 161: * uvm_aobj_pool: pool of uvm_aobj structures
! 162: */
! 163:
! 164: struct pool uvm_aobj_pool;
! 165:
! 166: /*
! 167: * local functions
! 168: */
! 169:
! 170: static struct uao_swhash_elt *uao_find_swhash_elt(struct uvm_aobj *,
! 171: int, boolean_t);
! 172: static int uao_find_swslot(struct uvm_aobj *, int);
! 173: static boolean_t uao_flush(struct uvm_object *,
! 174: voff_t, voff_t, int);
! 175: static void uao_free(struct uvm_aobj *);
! 176: static int uao_get(struct uvm_object *, voff_t,
! 177: vm_page_t *, int *, int,
! 178: vm_prot_t, int, int);
! 179: static boolean_t uao_releasepg(struct vm_page *,
! 180: struct vm_page **);
! 181: static boolean_t uao_pagein(struct uvm_aobj *, int, int);
! 182: static boolean_t uao_pagein_page(struct uvm_aobj *, int);
! 183:
! 184: /*
! 185: * aobj_pager
! 186: *
! 187: * note that some functions (e.g. put) are handled elsewhere
! 188: */
! 189:
! 190: struct uvm_pagerops aobj_pager = {
! 191: NULL, /* init */
! 192: uao_reference, /* reference */
! 193: uao_detach, /* detach */
! 194: NULL, /* fault */
! 195: uao_flush, /* flush */
! 196: uao_get, /* get */
! 197: NULL, /* put (done by pagedaemon) */
! 198: NULL, /* cluster */
! 199: NULL, /* mk_pcluster */
! 200: uao_releasepg /* releasepg */
! 201: };
! 202:
! 203: /*
! 204: * uao_list: global list of active aobjs, locked by uao_list_lock
! 205: */
! 206:
! 207: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
! 208: static simple_lock_data_t uao_list_lock;
! 209:
! 210:
! 211: /*
! 212: * functions
! 213: */
! 214:
! 215: /*
! 216: * hash table/array related functions
! 217: */
! 218:
! 219: /*
! 220: * uao_find_swhash_elt: find (or create) a hash table entry for a page
! 221: * offset.
! 222: *
! 223: * => the object should be locked by the caller
! 224: */
! 225:
! 226: static struct uao_swhash_elt *
! 227: uao_find_swhash_elt(aobj, pageidx, create)
! 228: struct uvm_aobj *aobj;
! 229: int pageidx;
! 230: boolean_t create;
! 231: {
! 232: struct uao_swhash *swhash;
! 233: struct uao_swhash_elt *elt;
! 234: voff_t page_tag;
! 235:
! 236: swhash = UAO_SWHASH_HASH(aobj, pageidx); /* first hash to get bucket */
! 237: page_tag = UAO_SWHASH_ELT_TAG(pageidx); /* tag to search for */
! 238:
! 239: /*
! 240: * now search the bucket for the requested tag
! 241: */
! 242: LIST_FOREACH(elt, swhash, list) {
! 243: if (elt->tag == page_tag)
! 244: return(elt);
! 245: }
! 246:
! 247: /* fail now if we are not allowed to create a new entry in the bucket */
! 248: if (!create)
! 249: return NULL;
! 250:
! 251:
! 252: /*
! 253: * allocate a new entry for the bucket and init/insert it in
! 254: */
! 255: elt = pool_get(&uao_swhash_elt_pool, PR_WAITOK);
! 256: LIST_INSERT_HEAD(swhash, elt, list);
! 257: elt->tag = page_tag;
! 258: elt->count = 0;
! 259: memset(elt->slots, 0, sizeof(elt->slots));
! 260:
! 261: return(elt);
! 262: }
! 263:
! 264: /*
! 265: * uao_find_swslot: find the swap slot number for an aobj/pageidx
! 266: *
! 267: * => object must be locked by caller
! 268: */
! 269: __inline static int
! 270: uao_find_swslot(aobj, pageidx)
! 271: struct uvm_aobj *aobj;
! 272: int pageidx;
! 273: {
! 274:
! 275: /*
! 276: * if noswap flag is set, then we never return a slot
! 277: */
! 278:
! 279: if (aobj->u_flags & UAO_FLAG_NOSWAP)
! 280: return(0);
! 281:
! 282: /*
! 283: * if hashing, look in hash table.
! 284: */
! 285:
! 286: if (UAO_USES_SWHASH(aobj)) {
! 287: struct uao_swhash_elt *elt =
! 288: uao_find_swhash_elt(aobj, pageidx, FALSE);
! 289:
! 290: if (elt)
! 291: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
! 292: else
! 293: return(0);
! 294: }
! 295:
! 296: /*
! 297: * otherwise, look in the array
! 298: */
! 299: return(aobj->u_swslots[pageidx]);
! 300: }
! 301:
! 302: /*
! 303: * uao_set_swslot: set the swap slot for a page in an aobj.
! 304: *
! 305: * => setting a slot to zero frees the slot
! 306: * => object must be locked by caller
! 307: */
! 308: int
! 309: uao_set_swslot(uobj, pageidx, slot)
! 310: struct uvm_object *uobj;
! 311: int pageidx, slot;
! 312: {
! 313: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
! 314: int oldslot;
! 315: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
! 316: UVMHIST_LOG(pdhist, "aobj %p pageidx %ld slot %ld",
! 317: aobj, pageidx, slot, 0);
! 318:
! 319: /*
! 320: * if noswap flag is set, then we can't set a slot
! 321: */
! 322:
! 323: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
! 324:
! 325: if (slot == 0)
! 326: return(0); /* a clear is ok */
! 327:
! 328: /* but a set is not */
! 329: printf("uao_set_swslot: uobj = %p\n", uobj);
! 330: panic("uao_set_swslot: attempt to set a slot on a NOSWAP object");
! 331: }
! 332:
! 333: /*
! 334: * are we using a hash table? if so, add it in the hash.
! 335: */
! 336:
! 337: if (UAO_USES_SWHASH(aobj)) {
! 338:
! 339: /*
! 340: * Avoid allocating an entry just to free it again if
! 341: * the page had not swap slot in the first place, and
! 342: * we are freeing.
! 343: */
! 344:
! 345: struct uao_swhash_elt *elt =
! 346: uao_find_swhash_elt(aobj, pageidx, slot ? TRUE : FALSE);
! 347: if (elt == NULL) {
! 348: KASSERT(slot == 0);
! 349: return (0);
! 350: }
! 351:
! 352: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
! 353: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
! 354:
! 355: /*
! 356: * now adjust the elt's reference counter and free it if we've
! 357: * dropped it to zero.
! 358: */
! 359:
! 360: /* an allocation? */
! 361: if (slot) {
! 362: if (oldslot == 0)
! 363: elt->count++;
! 364: } else { /* freeing slot ... */
! 365: if (oldslot) /* to be safe */
! 366: elt->count--;
! 367:
! 368: if (elt->count == 0) {
! 369: LIST_REMOVE(elt, list);
! 370: pool_put(&uao_swhash_elt_pool, elt);
! 371: }
! 372: }
! 373: } else {
! 374: /* we are using an array */
! 375: oldslot = aobj->u_swslots[pageidx];
! 376: aobj->u_swslots[pageidx] = slot;
! 377: }
! 378: return (oldslot);
! 379: }
! 380:
! 381: /*
! 382: * end of hash/array functions
! 383: */
! 384:
! 385: /*
! 386: * uao_free: free all resources held by an aobj, and then free the aobj
! 387: *
! 388: * => the aobj should be dead
! 389: */
! 390: static void
! 391: uao_free(aobj)
! 392: struct uvm_aobj *aobj;
! 393: {
! 394:
! 395: simple_unlock(&aobj->u_obj.vmobjlock);
! 396:
! 397: if (UAO_USES_SWHASH(aobj)) {
! 398: int i, hashbuckets = aobj->u_swhashmask + 1;
! 399:
! 400: /*
! 401: * free the swslots from each hash bucket,
! 402: * then the hash bucket, and finally the hash table itself.
! 403: */
! 404: for (i = 0; i < hashbuckets; i++) {
! 405: struct uao_swhash_elt *elt, *next;
! 406:
! 407: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
! 408: elt != NULL;
! 409: elt = next) {
! 410: int j;
! 411:
! 412: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
! 413: int slot = elt->slots[j];
! 414:
! 415: if (slot == 0) {
! 416: continue;
! 417: }
! 418: uvm_swap_free(slot, 1);
! 419:
! 420: /*
! 421: * this page is no longer
! 422: * only in swap.
! 423: */
! 424: simple_lock(&uvm.swap_data_lock);
! 425: uvmexp.swpgonly--;
! 426: simple_unlock(&uvm.swap_data_lock);
! 427: }
! 428:
! 429: next = LIST_NEXT(elt, list);
! 430: pool_put(&uao_swhash_elt_pool, elt);
! 431: }
! 432: }
! 433: free(aobj->u_swhash, M_UVMAOBJ);
! 434: } else {
! 435: int i;
! 436:
! 437: /*
! 438: * free the array
! 439: */
! 440:
! 441: for (i = 0; i < aobj->u_pages; i++) {
! 442: int slot = aobj->u_swslots[i];
! 443:
! 444: if (slot) {
! 445: uvm_swap_free(slot, 1);
! 446:
! 447: /* this page is no longer only in swap. */
! 448: simple_lock(&uvm.swap_data_lock);
! 449: uvmexp.swpgonly--;
! 450: simple_unlock(&uvm.swap_data_lock);
! 451: }
! 452: }
! 453: free(aobj->u_swslots, M_UVMAOBJ);
! 454: }
! 455:
! 456: /*
! 457: * finally free the aobj itself
! 458: */
! 459: pool_put(&uvm_aobj_pool, aobj);
! 460: }
! 461:
! 462: /*
! 463: * pager functions
! 464: */
! 465:
! 466: /*
! 467: * uao_create: create an aobj of the given size and return its uvm_object.
! 468: *
! 469: * => for normal use, flags are always zero
! 470: * => for the kernel object, the flags are:
! 471: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
! 472: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
! 473: */
! 474: struct uvm_object *
! 475: uao_create(size, flags)
! 476: vsize_t size;
! 477: int flags;
! 478: {
! 479: static struct uvm_aobj kernel_object_store; /* home of kernel_object */
! 480: static int kobj_alloced = 0; /* not allocated yet */
! 481: int pages = round_page(size) >> PAGE_SHIFT;
! 482: struct uvm_aobj *aobj;
! 483:
! 484: /*
! 485: * malloc a new aobj unless we are asked for the kernel object
! 486: */
! 487: if (flags & UAO_FLAG_KERNOBJ) { /* want kernel object? */
! 488: if (kobj_alloced)
! 489: panic("uao_create: kernel object already allocated");
! 490:
! 491: aobj = &kernel_object_store;
! 492: aobj->u_pages = pages;
! 493: aobj->u_flags = UAO_FLAG_NOSWAP; /* no swap to start */
! 494: /* we are special, we never die */
! 495: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
! 496: kobj_alloced = UAO_FLAG_KERNOBJ;
! 497: } else if (flags & UAO_FLAG_KERNSWAP) {
! 498: aobj = &kernel_object_store;
! 499: if (kobj_alloced != UAO_FLAG_KERNOBJ)
! 500: panic("uao_create: asked to enable swap on kernel object");
! 501: kobj_alloced = UAO_FLAG_KERNSWAP;
! 502: } else { /* normal object */
! 503: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
! 504: aobj->u_pages = pages;
! 505: aobj->u_flags = 0; /* normal object */
! 506: aobj->u_obj.uo_refs = 1; /* start with 1 reference */
! 507: }
! 508:
! 509: /*
! 510: * allocate hash/array if necessary
! 511: *
! 512: * note: in the KERNSWAP case no need to worry about locking since
! 513: * we are still booting we should be the only thread around.
! 514: */
! 515: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
! 516: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
! 517: M_NOWAIT : M_WAITOK;
! 518:
! 519: /* allocate hash table or array depending on object size */
! 520: if (UAO_USES_SWHASH(aobj)) {
! 521: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
! 522: M_UVMAOBJ, mflags, &aobj->u_swhashmask);
! 523: if (aobj->u_swhash == NULL)
! 524: panic("uao_create: hashinit swhash failed");
! 525: } else {
! 526: aobj->u_swslots = malloc(pages * sizeof(int),
! 527: M_UVMAOBJ, mflags);
! 528: if (aobj->u_swslots == NULL)
! 529: panic("uao_create: malloc swslots failed");
! 530: memset(aobj->u_swslots, 0, pages * sizeof(int));
! 531: }
! 532:
! 533: if (flags) {
! 534: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
! 535: return(&aobj->u_obj);
! 536: /* done! */
! 537: }
! 538: }
! 539:
! 540: /*
! 541: * init aobj fields
! 542: */
! 543: simple_lock_init(&aobj->u_obj.vmobjlock);
! 544: aobj->u_obj.pgops = &aobj_pager;
! 545: TAILQ_INIT(&aobj->u_obj.memq);
! 546: aobj->u_obj.uo_npages = 0;
! 547:
! 548: /*
! 549: * now that aobj is ready, add it to the global list
! 550: */
! 551: simple_lock(&uao_list_lock);
! 552: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
! 553: simple_unlock(&uao_list_lock);
! 554:
! 555: /*
! 556: * done!
! 557: */
! 558: return(&aobj->u_obj);
! 559: }
! 560:
! 561:
! 562:
! 563: /*
! 564: * uao_init: set up aobj pager subsystem
! 565: *
! 566: * => called at boot time from uvm_pager_init()
! 567: */
! 568: void
! 569: uao_init()
! 570: {
! 571: static int uao_initialized;
! 572:
! 573: if (uao_initialized)
! 574: return;
! 575: uao_initialized = TRUE;
! 576:
! 577: LIST_INIT(&uao_list);
! 578: simple_lock_init(&uao_list_lock);
! 579:
! 580: /*
! 581: * NOTE: Pages fror this pool must not come from a pageable
! 582: * kernel map!
! 583: */
! 584: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
! 585: 0, 0, 0, "uaoeltpl", &pool_allocator_nointr);
! 586:
! 587: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
! 588: "aobjpl", &pool_allocator_nointr);
! 589: }
! 590:
! 591: /*
! 592: * uao_reference: add a ref to an aobj
! 593: *
! 594: * => aobj must be unlocked
! 595: * => just lock it and call the locked version
! 596: */
! 597: void
! 598: uao_reference(uobj)
! 599: struct uvm_object *uobj;
! 600: {
! 601: simple_lock(&uobj->vmobjlock);
! 602: uao_reference_locked(uobj);
! 603: simple_unlock(&uobj->vmobjlock);
! 604: }
! 605:
! 606: /*
! 607: * uao_reference_locked: add a ref to an aobj that is already locked
! 608: *
! 609: * => aobj must be locked
! 610: * this needs to be separate from the normal routine
! 611: * since sometimes we need to add a reference to an aobj when
! 612: * it's already locked.
! 613: */
! 614: void
! 615: uao_reference_locked(uobj)
! 616: struct uvm_object *uobj;
! 617: {
! 618: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
! 619:
! 620: /*
! 621: * kernel_object already has plenty of references, leave it alone.
! 622: */
! 623:
! 624: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
! 625: return;
! 626:
! 627: uobj->uo_refs++; /* bump! */
! 628: UVMHIST_LOG(maphist, "<- done (uobj=%p, ref = %ld)",
! 629: uobj, uobj->uo_refs,0,0);
! 630: }
! 631:
! 632:
! 633: /*
! 634: * uao_detach: drop a reference to an aobj
! 635: *
! 636: * => aobj must be unlocked
! 637: * => just lock it and call the locked version
! 638: */
! 639: void
! 640: uao_detach(uobj)
! 641: struct uvm_object *uobj;
! 642: {
! 643: simple_lock(&uobj->vmobjlock);
! 644: uao_detach_locked(uobj);
! 645: }
! 646:
! 647:
! 648: /*
! 649: * uao_detach_locked: drop a reference to an aobj
! 650: *
! 651: * => aobj must be locked, and is unlocked (or freed) upon return.
! 652: * this needs to be separate from the normal routine
! 653: * since sometimes we need to detach from an aobj when
! 654: * it's already locked.
! 655: */
! 656: void
! 657: uao_detach_locked(uobj)
! 658: struct uvm_object *uobj;
! 659: {
! 660: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
! 661: struct vm_page *pg, *next;
! 662: boolean_t busybody;
! 663: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
! 664:
! 665: /*
! 666: * detaching from kernel_object is a noop.
! 667: */
! 668: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
! 669: simple_unlock(&uobj->vmobjlock);
! 670: return;
! 671: }
! 672:
! 673: UVMHIST_LOG(maphist," (uobj=%p) ref=%ld", uobj,uobj->uo_refs,0,0);
! 674: uobj->uo_refs--; /* drop ref! */
! 675: if (uobj->uo_refs) { /* still more refs? */
! 676: simple_unlock(&uobj->vmobjlock);
! 677: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
! 678: return;
! 679: }
! 680:
! 681: /*
! 682: * remove the aobj from the global list.
! 683: */
! 684: simple_lock(&uao_list_lock);
! 685: LIST_REMOVE(aobj, u_list);
! 686: simple_unlock(&uao_list_lock);
! 687:
! 688: /*
! 689: * free all the pages that aren't PG_BUSY,
! 690: * mark for release any that are.
! 691: */
! 692: busybody = FALSE;
! 693: for (pg = TAILQ_FIRST(&uobj->memq); pg != NULL; pg = next) {
! 694: next = TAILQ_NEXT(pg, listq);
! 695: if (pg->pg_flags & PG_BUSY) {
! 696: atomic_setbits_int(&pg->pg_flags, PG_RELEASED);
! 697: busybody = TRUE;
! 698: continue;
! 699: }
! 700:
! 701: /* zap the mappings, free the swap slot, free the page */
! 702: pmap_page_protect(pg, VM_PROT_NONE);
! 703: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
! 704: uvm_lock_pageq();
! 705: uvm_pagefree(pg);
! 706: uvm_unlock_pageq();
! 707: }
! 708:
! 709: /*
! 710: * if we found any busy pages, we're done for now.
! 711: * mark the aobj for death, releasepg will finish up for us.
! 712: */
! 713: if (busybody) {
! 714: aobj->u_flags |= UAO_FLAG_KILLME;
! 715: simple_unlock(&aobj->u_obj.vmobjlock);
! 716: return;
! 717: }
! 718:
! 719: /*
! 720: * finally, free the rest.
! 721: */
! 722: uao_free(aobj);
! 723: }
! 724:
! 725: /*
! 726: * uao_flush: "flush" pages out of a uvm object
! 727: *
! 728: * => object should be locked by caller. we may _unlock_ the object
! 729: * if (and only if) we need to clean a page (PGO_CLEANIT).
! 730: * XXXJRT Currently, however, we don't. In the case of cleaning
! 731: * XXXJRT a page, we simply just deactivate it. Should probably
! 732: * XXXJRT handle this better, in the future (although "flushing"
! 733: * XXXJRT anonymous memory isn't terribly important).
! 734: * => if PGO_CLEANIT is not set, then we will neither unlock the object
! 735: * or block.
! 736: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
! 737: * for flushing.
! 738: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
! 739: * that new pages are inserted on the tail end of the list. thus,
! 740: * we can make a complete pass through the object in one go by starting
! 741: * at the head and working towards the tail (new pages are put in
! 742: * front of us).
! 743: * => NOTE: we are allowed to lock the page queues, so the caller
! 744: * must not be holding the lock on them [e.g. pagedaemon had
! 745: * better not call us with the queues locked]
! 746: * => we return TRUE unless we encountered some sort of I/O error
! 747: * XXXJRT currently never happens, as we never directly initiate
! 748: * XXXJRT I/O
! 749: *
! 750: * comment on "cleaning" object and PG_BUSY pages:
! 751: * this routine is holding the lock on the object. the only time
! 752: * that is can run into a PG_BUSY page that it does not own is if
! 753: * some other process has started I/O on the page (e.g. either
! 754: * a pagein or a pageout). if the PG_BUSY page is being paged
! 755: * in, then it can not be dirty (!PG_CLEAN) because no one has
! 756: * had a change to modify it yet. if the PG_BUSY page is being
! 757: * paged out then it means that someone else has already started
! 758: * cleaning the page for us (how nice!). in this case, if we
! 759: * have syncio specified, then after we make our pass through the
! 760: * object we need to wait for the other PG_BUSY pages to clear
! 761: * off (i.e. we need to do an iosync). also note that once a
! 762: * page is PG_BUSY is must stary in its object until it is un-busyed.
! 763: * XXXJRT We never actually do this, as we are "flushing" anonymous
! 764: * XXXJRT memory, which doesn't have persistent backing store.
! 765: *
! 766: * note on page traversal:
! 767: * we can traverse the pages in an object either by going down the
! 768: * linked list in "uobj->memq", or we can go over the address range
! 769: * by page doing hash table lookups for each address. depending
! 770: * on how many pages are in the object it may be cheaper to do one
! 771: * or the other. we set "by_list" to true if we are using memq.
! 772: * if the cost of a hash lookup was equal to the cost of the list
! 773: * traversal we could compare the number of pages in the start->stop
! 774: * range to the total number of pages in the object. however, it
! 775: * seems that a hash table lookup is more expensive than the linked
! 776: * list traversal, so we multiply the number of pages in the
! 777: * start->stop range by a penalty which we define below.
! 778: */
! 779:
! 780: #define UAO_HASH_PENALTY 4 /* XXX: a guess */
! 781:
! 782: boolean_t
! 783: uao_flush(uobj, start, stop, flags)
! 784: struct uvm_object *uobj;
! 785: voff_t start, stop;
! 786: int flags;
! 787: {
! 788: struct uvm_aobj *aobj = (struct uvm_aobj *) uobj;
! 789: struct vm_page *pp, *ppnext;
! 790: boolean_t retval, by_list;
! 791: voff_t curoff;
! 792: UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist);
! 793:
! 794: curoff = 0; /* XXX: shut up gcc */
! 795:
! 796: retval = TRUE; /* default to success */
! 797:
! 798: if (flags & PGO_ALLPAGES) {
! 799: start = 0;
! 800: stop = aobj->u_pages << PAGE_SHIFT;
! 801: by_list = TRUE; /* always go by the list */
! 802: } else {
! 803: start = trunc_page(start);
! 804: stop = round_page(stop);
! 805: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
! 806: printf("uao_flush: strange, got an out of range "
! 807: "flush (fixed)\n");
! 808: stop = aobj->u_pages << PAGE_SHIFT;
! 809: }
! 810: by_list = (uobj->uo_npages <=
! 811: ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY);
! 812: }
! 813:
! 814: UVMHIST_LOG(maphist,
! 815: " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
! 816: (u_long)start, (u_long)stop, by_list, flags);
! 817:
! 818: /*
! 819: * Don't need to do any work here if we're not freeing
! 820: * or deactivating pages.
! 821: */
! 822: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
! 823: UVMHIST_LOG(maphist,
! 824: "<- done (no work to do)",0,0,0,0);
! 825: return (retval);
! 826: }
! 827:
! 828: /*
! 829: * now do it. note: we must update ppnext in the body of loop or we
! 830: * will get stuck. we need to use ppnext because we may free "pp"
! 831: * before doing the next loop.
! 832: */
! 833:
! 834: if (by_list) {
! 835: pp = TAILQ_FIRST(&uobj->memq);
! 836: } else {
! 837: curoff = start;
! 838: pp = uvm_pagelookup(uobj, curoff);
! 839: }
! 840:
! 841: ppnext = NULL; /* XXX: shut up gcc */
! 842: uvm_lock_pageq(); /* page queues locked */
! 843:
! 844: /* locked: both page queues and uobj */
! 845: for ( ; (by_list && pp != NULL) ||
! 846: (!by_list && curoff < stop) ; pp = ppnext) {
! 847: if (by_list) {
! 848: ppnext = TAILQ_NEXT(pp, listq);
! 849:
! 850: /* range check */
! 851: if (pp->offset < start || pp->offset >= stop)
! 852: continue;
! 853: } else {
! 854: curoff += PAGE_SIZE;
! 855: if (curoff < stop)
! 856: ppnext = uvm_pagelookup(uobj, curoff);
! 857:
! 858: /* null check */
! 859: if (pp == NULL)
! 860: continue;
! 861: }
! 862:
! 863: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
! 864: /*
! 865: * XXX In these first 3 cases, we always just
! 866: * XXX deactivate the page. We may want to
! 867: * XXX handle the different cases more specifically
! 868: * XXX in the future.
! 869: */
! 870: case PGO_CLEANIT|PGO_FREE:
! 871: case PGO_CLEANIT|PGO_DEACTIVATE:
! 872: case PGO_DEACTIVATE:
! 873: deactivate_it:
! 874: /* skip the page if it's loaned or wired */
! 875: if (pp->loan_count != 0 ||
! 876: pp->wire_count != 0)
! 877: continue;
! 878:
! 879: #ifdef UBC
! 880: /* ...and deactivate the page. */
! 881: pmap_clear_reference(pp);
! 882: #else
! 883: /* zap all mappings for the page. */
! 884: pmap_page_protect(pp, VM_PROT_NONE);
! 885:
! 886: /* ...and deactivate the page. */
! 887: #endif
! 888: uvm_pagedeactivate(pp);
! 889:
! 890: continue;
! 891:
! 892: case PGO_FREE:
! 893: /*
! 894: * If there are multiple references to
! 895: * the object, just deactivate the page.
! 896: */
! 897: if (uobj->uo_refs > 1)
! 898: goto deactivate_it;
! 899:
! 900: /* XXX skip the page if it's loaned or wired */
! 901: if (pp->loan_count != 0 ||
! 902: pp->wire_count != 0)
! 903: continue;
! 904:
! 905: /*
! 906: * mark the page as released if its busy.
! 907: */
! 908: if (pp->pg_flags & PG_BUSY) {
! 909: atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
! 910: continue;
! 911: }
! 912:
! 913: /* zap all mappings for the page. */
! 914: pmap_page_protect(pp, VM_PROT_NONE);
! 915:
! 916: uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
! 917: uvm_pagefree(pp);
! 918:
! 919: continue;
! 920:
! 921: default:
! 922: panic("uao_flush: weird flags");
! 923: }
! 924: }
! 925:
! 926: uvm_unlock_pageq();
! 927:
! 928: UVMHIST_LOG(maphist,
! 929: "<- done, rv=%ld",retval,0,0,0);
! 930: return (retval);
! 931: }
! 932:
! 933: /*
! 934: * uao_get: fetch me a page
! 935: *
! 936: * we have three cases:
! 937: * 1: page is resident -> just return the page.
! 938: * 2: page is zero-fill -> allocate a new page and zero it.
! 939: * 3: page is swapped out -> fetch the page from swap.
! 940: *
! 941: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
! 942: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
! 943: * then we will need to return VM_PAGER_UNLOCK.
! 944: *
! 945: * => prefer map unlocked (not required)
! 946: * => object must be locked! we will _unlock_ it before starting any I/O.
! 947: * => flags: PGO_ALLPAGES: get all of the pages
! 948: * PGO_LOCKED: fault data structures are locked
! 949: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
! 950: * => NOTE: caller must check for released pages!!
! 951: */
! 952: static int
! 953: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
! 954: struct uvm_object *uobj;
! 955: voff_t offset;
! 956: struct vm_page **pps;
! 957: int *npagesp;
! 958: int centeridx, advice, flags;
! 959: vm_prot_t access_type;
! 960: {
! 961: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
! 962: voff_t current_offset;
! 963: vm_page_t ptmp;
! 964: int lcv, gotpages, maxpages, swslot, rv, pageidx;
! 965: boolean_t done;
! 966: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
! 967:
! 968: UVMHIST_LOG(pdhist, "aobj=%p offset=%ld, flags=%ld",
! 969: aobj, (u_long)offset, flags,0);
! 970:
! 971: /*
! 972: * get number of pages
! 973: */
! 974: maxpages = *npagesp;
! 975:
! 976: /*
! 977: * step 1: handled the case where fault data structures are locked.
! 978: */
! 979:
! 980: if (flags & PGO_LOCKED) {
! 981: /*
! 982: * step 1a: get pages that are already resident. only do
! 983: * this if the data structures are locked (i.e. the first
! 984: * time through).
! 985: */
! 986:
! 987: done = TRUE; /* be optimistic */
! 988: gotpages = 0; /* # of pages we got so far */
! 989:
! 990: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
! 991: lcv++, current_offset += PAGE_SIZE) {
! 992: /* do we care about this page? if not, skip it */
! 993: if (pps[lcv] == PGO_DONTCARE)
! 994: continue;
! 995:
! 996: ptmp = uvm_pagelookup(uobj, current_offset);
! 997:
! 998: /*
! 999: * if page is new, attempt to allocate the page,
! 1000: * zero-fill'd.
! 1001: */
! 1002: if (ptmp == NULL && uao_find_swslot(aobj,
! 1003: current_offset >> PAGE_SHIFT) == 0) {
! 1004: ptmp = uvm_pagealloc(uobj, current_offset,
! 1005: NULL, UVM_PGA_ZERO);
! 1006: if (ptmp) {
! 1007: /* new page */
! 1008: atomic_clearbits_int(&ptmp->pg_flags,
! 1009: PG_BUSY|PG_FAKE);
! 1010: atomic_setbits_int(&ptmp->pg_flags,
! 1011: PQ_AOBJ);
! 1012: UVM_PAGE_OWN(ptmp, NULL);
! 1013: }
! 1014: }
! 1015:
! 1016: /*
! 1017: * to be useful must get a non-busy, non-released page
! 1018: */
! 1019: if (ptmp == NULL ||
! 1020: (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
! 1021: if (lcv == centeridx ||
! 1022: (flags & PGO_ALLPAGES) != 0)
! 1023: /* need to do a wait or I/O! */
! 1024: done = FALSE;
! 1025: continue;
! 1026: }
! 1027:
! 1028: /*
! 1029: * useful page: busy/lock it and plug it in our
! 1030: * result array
! 1031: */
! 1032: /* caller must un-busy this page */
! 1033: atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
! 1034: UVM_PAGE_OWN(ptmp, "uao_get1");
! 1035: pps[lcv] = ptmp;
! 1036: gotpages++;
! 1037:
! 1038: } /* "for" lcv loop */
! 1039:
! 1040: /*
! 1041: * step 1b: now we've either done everything needed or we
! 1042: * to unlock and do some waiting or I/O.
! 1043: */
! 1044:
! 1045: UVMHIST_LOG(pdhist, "<- done (done=%ld)", done, 0,0,0);
! 1046:
! 1047: *npagesp = gotpages;
! 1048: if (done)
! 1049: /* bingo! */
! 1050: return(VM_PAGER_OK);
! 1051: else
! 1052: /* EEK! Need to unlock and I/O */
! 1053: return(VM_PAGER_UNLOCK);
! 1054: }
! 1055:
! 1056: /*
! 1057: * step 2: get non-resident or busy pages.
! 1058: * object is locked. data structures are unlocked.
! 1059: */
! 1060:
! 1061: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
! 1062: lcv++, current_offset += PAGE_SIZE) {
! 1063:
! 1064: /*
! 1065: * - skip over pages we've already gotten or don't want
! 1066: * - skip over pages we don't _have_ to get
! 1067: */
! 1068:
! 1069: if (pps[lcv] != NULL ||
! 1070: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
! 1071: continue;
! 1072:
! 1073: pageidx = current_offset >> PAGE_SHIFT;
! 1074:
! 1075: /*
! 1076: * we have yet to locate the current page (pps[lcv]). we
! 1077: * first look for a page that is already at the current offset.
! 1078: * if we find a page, we check to see if it is busy or
! 1079: * released. if that is the case, then we sleep on the page
! 1080: * until it is no longer busy or released and repeat the lookup.
! 1081: * if the page we found is neither busy nor released, then we
! 1082: * busy it (so we own it) and plug it into pps[lcv]. this
! 1083: * 'break's the following while loop and indicates we are
! 1084: * ready to move on to the next page in the "lcv" loop above.
! 1085: *
! 1086: * if we exit the while loop with pps[lcv] still set to NULL,
! 1087: * then it means that we allocated a new busy/fake/clean page
! 1088: * ptmp in the object and we need to do I/O to fill in the data.
! 1089: */
! 1090:
! 1091: /* top of "pps" while loop */
! 1092: while (pps[lcv] == NULL) {
! 1093: /* look for a resident page */
! 1094: ptmp = uvm_pagelookup(uobj, current_offset);
! 1095:
! 1096: /* not resident? allocate one now (if we can) */
! 1097: if (ptmp == NULL) {
! 1098:
! 1099: ptmp = uvm_pagealloc(uobj, current_offset,
! 1100: NULL, 0);
! 1101:
! 1102: /* out of RAM? */
! 1103: if (ptmp == NULL) {
! 1104: simple_unlock(&uobj->vmobjlock);
! 1105: UVMHIST_LOG(pdhist,
! 1106: "sleeping, ptmp == NULL\n",0,0,0,0);
! 1107: uvm_wait("uao_getpage");
! 1108: simple_lock(&uobj->vmobjlock);
! 1109: /* goto top of pps while loop */
! 1110: continue;
! 1111: }
! 1112:
! 1113: /*
! 1114: * safe with PQ's unlocked: because we just
! 1115: * alloc'd the page
! 1116: */
! 1117: atomic_setbits_int(&ptmp->pg_flags, PQ_AOBJ);
! 1118:
! 1119: /*
! 1120: * got new page ready for I/O. break pps while
! 1121: * loop. pps[lcv] is still NULL.
! 1122: */
! 1123: break;
! 1124: }
! 1125:
! 1126: /* page is there, see if we need to wait on it */
! 1127: if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
! 1128: atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
! 1129: UVMHIST_LOG(pdhist,
! 1130: "sleeping, ptmp->flags 0x%lx\n",
! 1131: ptmp->pg_flags,0,0,0);
! 1132: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
! 1133: FALSE, "uao_get", 0);
! 1134: simple_lock(&uobj->vmobjlock);
! 1135: continue; /* goto top of pps while loop */
! 1136: }
! 1137:
! 1138: /*
! 1139: * if we get here then the page has become resident and
! 1140: * unbusy between steps 1 and 2. we busy it now (so we
! 1141: * own it) and set pps[lcv] (so that we exit the while
! 1142: * loop).
! 1143: */
! 1144: /* we own it, caller must un-busy */
! 1145: atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
! 1146: UVM_PAGE_OWN(ptmp, "uao_get2");
! 1147: pps[lcv] = ptmp;
! 1148: }
! 1149:
! 1150: /*
! 1151: * if we own the valid page at the correct offset, pps[lcv] will
! 1152: * point to it. nothing more to do except go to the next page.
! 1153: */
! 1154: if (pps[lcv])
! 1155: continue; /* next lcv */
! 1156:
! 1157: /*
! 1158: * we have a "fake/busy/clean" page that we just allocated.
! 1159: * do the needed "i/o", either reading from swap or zeroing.
! 1160: */
! 1161: swslot = uao_find_swslot(aobj, pageidx);
! 1162:
! 1163: /*
! 1164: * just zero the page if there's nothing in swap.
! 1165: */
! 1166: if (swslot == 0)
! 1167: {
! 1168: /*
! 1169: * page hasn't existed before, just zero it.
! 1170: */
! 1171: uvm_pagezero(ptmp);
! 1172: } else {
! 1173: UVMHIST_LOG(pdhist, "pagein from swslot %ld",
! 1174: swslot, 0,0,0);
! 1175:
! 1176: /*
! 1177: * page in the swapped-out page.
! 1178: * unlock object for i/o, relock when done.
! 1179: */
! 1180: simple_unlock(&uobj->vmobjlock);
! 1181: rv = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
! 1182: simple_lock(&uobj->vmobjlock);
! 1183:
! 1184: /*
! 1185: * I/O done. check for errors.
! 1186: */
! 1187: if (rv != VM_PAGER_OK)
! 1188: {
! 1189: UVMHIST_LOG(pdhist, "<- done (error=%ld)",
! 1190: rv,0,0,0);
! 1191: if (ptmp->pg_flags & PG_WANTED)
! 1192: wakeup(ptmp);
! 1193:
! 1194: /*
! 1195: * remove the swap slot from the aobj
! 1196: * and mark the aobj as having no real slot.
! 1197: * don't free the swap slot, thus preventing
! 1198: * it from being used again.
! 1199: */
! 1200: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
! 1201: SWSLOT_BAD);
! 1202: uvm_swap_markbad(swslot, 1);
! 1203:
! 1204: atomic_clearbits_int(&ptmp->pg_flags,
! 1205: PG_WANTED|PG_BUSY);
! 1206: UVM_PAGE_OWN(ptmp, NULL);
! 1207: uvm_lock_pageq();
! 1208: uvm_pagefree(ptmp);
! 1209: uvm_unlock_pageq();
! 1210:
! 1211: simple_unlock(&uobj->vmobjlock);
! 1212: return (rv);
! 1213: }
! 1214: }
! 1215:
! 1216: /*
! 1217: * we got the page! clear the fake flag (indicates valid
! 1218: * data now in page) and plug into our result array. note
! 1219: * that page is still busy.
! 1220: *
! 1221: * it is the callers job to:
! 1222: * => check if the page is released
! 1223: * => unbusy the page
! 1224: * => activate the page
! 1225: */
! 1226:
! 1227: /* data is valid ... */
! 1228: atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE);
! 1229: pmap_clear_modify(ptmp); /* ... and clean */
! 1230: pps[lcv] = ptmp;
! 1231:
! 1232: } /* lcv loop */
! 1233:
! 1234: /*
! 1235: * finally, unlock object and return.
! 1236: */
! 1237:
! 1238: simple_unlock(&uobj->vmobjlock);
! 1239: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
! 1240: return(VM_PAGER_OK);
! 1241: }
! 1242:
! 1243: /*
! 1244: * uao_releasepg: handle released page in an aobj
! 1245: *
! 1246: * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
! 1247: * to dispose of.
! 1248: * => caller must handle PG_WANTED case
! 1249: * => called with page's object locked, pageq's unlocked
! 1250: * => returns TRUE if page's object is still alive, FALSE if we
! 1251: * killed the page's object. if we return TRUE, then we
! 1252: * return with the object locked.
! 1253: * => if (nextpgp != NULL) => we return the next page on the queue, and return
! 1254: * with the page queues locked [for pagedaemon]
! 1255: * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
! 1256: * => we kill the aobj if it is not referenced and we are suppose to
! 1257: * kill it ("KILLME").
! 1258: */
! 1259: static boolean_t
! 1260: uao_releasepg(pg, nextpgp)
! 1261: struct vm_page *pg;
! 1262: struct vm_page **nextpgp; /* OUT */
! 1263: {
! 1264: struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject;
! 1265:
! 1266: KASSERT(pg->pg_flags & PG_RELEASED);
! 1267:
! 1268: /*
! 1269: * dispose of the page [caller handles PG_WANTED] and swap slot.
! 1270: */
! 1271: pmap_page_protect(pg, VM_PROT_NONE);
! 1272: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
! 1273: uvm_lock_pageq();
! 1274: if (nextpgp)
! 1275: *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
! 1276: uvm_pagefree(pg);
! 1277: if (!nextpgp)
! 1278: uvm_unlock_pageq(); /* keep locked for daemon */
! 1279:
! 1280: /*
! 1281: * if we're not killing the object, we're done.
! 1282: */
! 1283: if ((aobj->u_flags & UAO_FLAG_KILLME) == 0)
! 1284: return TRUE;
! 1285: KASSERT(aobj->u_obj.uo_refs == 0);
! 1286:
! 1287: /*
! 1288: * if there are still pages in the object, we're done for now.
! 1289: */
! 1290: if (aobj->u_obj.uo_npages != 0)
! 1291: return TRUE;
! 1292:
! 1293: KASSERT(TAILQ_EMPTY(&aobj->u_obj.memq));
! 1294:
! 1295: /*
! 1296: * finally, free the rest.
! 1297: */
! 1298: uao_free(aobj);
! 1299:
! 1300: return FALSE;
! 1301: }
! 1302:
! 1303:
! 1304: /*
! 1305: * uao_dropswap: release any swap resources from this aobj page.
! 1306: *
! 1307: * => aobj must be locked or have a reference count of 0.
! 1308: */
! 1309:
! 1310: void
! 1311: uao_dropswap(uobj, pageidx)
! 1312: struct uvm_object *uobj;
! 1313: int pageidx;
! 1314: {
! 1315: int slot;
! 1316:
! 1317: slot = uao_set_swslot(uobj, pageidx, 0);
! 1318: if (slot) {
! 1319: uvm_swap_free(slot, 1);
! 1320: }
! 1321: }
! 1322:
! 1323:
! 1324: /*
! 1325: * page in every page in every aobj that is paged-out to a range of swslots.
! 1326: *
! 1327: * => nothing should be locked.
! 1328: * => returns TRUE if pagein was aborted due to lack of memory.
! 1329: */
! 1330: boolean_t
! 1331: uao_swap_off(startslot, endslot)
! 1332: int startslot, endslot;
! 1333: {
! 1334: struct uvm_aobj *aobj, *nextaobj;
! 1335:
! 1336: /*
! 1337: * walk the list of all aobjs.
! 1338: */
! 1339:
! 1340: restart:
! 1341: simple_lock(&uao_list_lock);
! 1342:
! 1343: for (aobj = LIST_FIRST(&uao_list);
! 1344: aobj != NULL;
! 1345: aobj = nextaobj) {
! 1346: boolean_t rv;
! 1347:
! 1348: /*
! 1349: * try to get the object lock,
! 1350: * start all over if we fail.
! 1351: * most of the time we'll get the aobj lock,
! 1352: * so this should be a rare case.
! 1353: */
! 1354: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
! 1355: simple_unlock(&uao_list_lock);
! 1356: goto restart;
! 1357: }
! 1358:
! 1359: /*
! 1360: * add a ref to the aobj so it doesn't disappear
! 1361: * while we're working.
! 1362: */
! 1363: uao_reference_locked(&aobj->u_obj);
! 1364:
! 1365: /*
! 1366: * now it's safe to unlock the uao list.
! 1367: */
! 1368: simple_unlock(&uao_list_lock);
! 1369:
! 1370: /*
! 1371: * page in any pages in the swslot range.
! 1372: * if there's an error, abort and return the error.
! 1373: */
! 1374: rv = uao_pagein(aobj, startslot, endslot);
! 1375: if (rv) {
! 1376: uao_detach_locked(&aobj->u_obj);
! 1377: return rv;
! 1378: }
! 1379:
! 1380: /*
! 1381: * we're done with this aobj.
! 1382: * relock the list and drop our ref on the aobj.
! 1383: */
! 1384: simple_lock(&uao_list_lock);
! 1385: nextaobj = LIST_NEXT(aobj, u_list);
! 1386: uao_detach_locked(&aobj->u_obj);
! 1387: }
! 1388:
! 1389: /*
! 1390: * done with traversal, unlock the list
! 1391: */
! 1392: simple_unlock(&uao_list_lock);
! 1393: return FALSE;
! 1394: }
! 1395:
! 1396:
! 1397: /*
! 1398: * page in any pages from aobj in the given range.
! 1399: *
! 1400: * => aobj must be locked and is returned locked.
! 1401: * => returns TRUE if pagein was aborted due to lack of memory.
! 1402: */
! 1403: static boolean_t
! 1404: uao_pagein(aobj, startslot, endslot)
! 1405: struct uvm_aobj *aobj;
! 1406: int startslot, endslot;
! 1407: {
! 1408: boolean_t rv;
! 1409:
! 1410: if (UAO_USES_SWHASH(aobj)) {
! 1411: struct uao_swhash_elt *elt;
! 1412: int bucket;
! 1413:
! 1414: restart:
! 1415: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
! 1416: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
! 1417: elt != NULL;
! 1418: elt = LIST_NEXT(elt, list)) {
! 1419: int i;
! 1420:
! 1421: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
! 1422: int slot = elt->slots[i];
! 1423:
! 1424: /*
! 1425: * if the slot isn't in range, skip it.
! 1426: */
! 1427: if (slot < startslot ||
! 1428: slot >= endslot) {
! 1429: continue;
! 1430: }
! 1431:
! 1432: /*
! 1433: * process the page,
! 1434: * the start over on this object
! 1435: * since the swhash elt
! 1436: * may have been freed.
! 1437: */
! 1438: rv = uao_pagein_page(aobj,
! 1439: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
! 1440: if (rv) {
! 1441: return rv;
! 1442: }
! 1443: goto restart;
! 1444: }
! 1445: }
! 1446: }
! 1447: } else {
! 1448: int i;
! 1449:
! 1450: for (i = 0; i < aobj->u_pages; i++) {
! 1451: int slot = aobj->u_swslots[i];
! 1452:
! 1453: /*
! 1454: * if the slot isn't in range, skip it
! 1455: */
! 1456: if (slot < startslot || slot >= endslot) {
! 1457: continue;
! 1458: }
! 1459:
! 1460: /*
! 1461: * process the page.
! 1462: */
! 1463: rv = uao_pagein_page(aobj, i);
! 1464: if (rv) {
! 1465: return rv;
! 1466: }
! 1467: }
! 1468: }
! 1469:
! 1470: return FALSE;
! 1471: }
! 1472:
! 1473: /*
! 1474: * page in a page from an aobj. used for swap_off.
! 1475: * returns TRUE if pagein was aborted due to lack of memory.
! 1476: *
! 1477: * => aobj must be locked and is returned locked.
! 1478: */
! 1479: static boolean_t
! 1480: uao_pagein_page(aobj, pageidx)
! 1481: struct uvm_aobj *aobj;
! 1482: int pageidx;
! 1483: {
! 1484: struct vm_page *pg;
! 1485: int rv, slot, npages;
! 1486:
! 1487: pg = NULL;
! 1488: npages = 1;
! 1489: /* locked: aobj */
! 1490: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
! 1491: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
! 1492: /* unlocked: aobj */
! 1493:
! 1494: /*
! 1495: * relock and finish up.
! 1496: */
! 1497: simple_lock(&aobj->u_obj.vmobjlock);
! 1498:
! 1499: switch (rv) {
! 1500: case VM_PAGER_OK:
! 1501: break;
! 1502:
! 1503: case VM_PAGER_ERROR:
! 1504: case VM_PAGER_REFAULT:
! 1505: /*
! 1506: * nothing more to do on errors.
! 1507: * VM_PAGER_REFAULT can only mean that the anon was freed,
! 1508: * so again there's nothing to do.
! 1509: */
! 1510: return FALSE;
! 1511:
! 1512: }
! 1513: KASSERT((pg->pg_flags & PG_RELEASED) == 0);
! 1514:
! 1515: /*
! 1516: * ok, we've got the page now.
! 1517: * mark it as dirty, clear its swslot and un-busy it.
! 1518: */
! 1519: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
! 1520: uvm_swap_free(slot, 1);
! 1521: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_CLEAN|PG_FAKE);
! 1522: UVM_PAGE_OWN(pg, NULL);
! 1523:
! 1524: /*
! 1525: * deactivate the page (to put it on a page queue).
! 1526: */
! 1527: pmap_clear_reference(pg);
! 1528: #ifndef UBC
! 1529: pmap_page_protect(pg, VM_PROT_NONE);
! 1530: #endif
! 1531: uvm_lock_pageq();
! 1532: uvm_pagedeactivate(pg);
! 1533: uvm_unlock_pageq();
! 1534:
! 1535: return FALSE;
! 1536: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_aobj.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm