[BACK]Return to uvm_amap.c CVS log [TXT][DIR] Up to [local] / sys / uvm

Annotation of sys/uvm/uvm_amap.c, Revision 1.1

1.1     ! nbrk        1: /*     $OpenBSD: uvm_amap.c,v 1.39 2007/06/18 21:51:15 pedro Exp $     */
        !             2: /*     $NetBSD: uvm_amap.c,v 1.27 2000/11/25 06:27:59 chs Exp $        */
        !             3:
        !             4: /*
        !             5:  *
        !             6:  * Copyright (c) 1997 Charles D. Cranor and Washington University.
        !             7:  * All rights reserved.
        !             8:  *
        !             9:  * Redistribution and use in source and binary forms, with or without
        !            10:  * modification, are permitted provided that the following conditions
        !            11:  * are met:
        !            12:  * 1. Redistributions of source code must retain the above copyright
        !            13:  *    notice, this list of conditions and the following disclaimer.
        !            14:  * 2. Redistributions in binary form must reproduce the above copyright
        !            15:  *    notice, this list of conditions and the following disclaimer in the
        !            16:  *    documentation and/or other materials provided with the distribution.
        !            17:  * 3. All advertising materials mentioning features or use of this software
        !            18:  *    must display the following acknowledgement:
        !            19:  *      This product includes software developed by Charles D. Cranor and
        !            20:  *      Washington University.
        !            21:  * 4. The name of the author may not be used to endorse or promote products
        !            22:  *    derived from this software without specific prior written permission.
        !            23:  *
        !            24:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
        !            25:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
        !            26:  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
        !            27:  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
        !            28:  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
        !            29:  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
        !            30:  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
        !            31:  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
        !            32:  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
        !            33:  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        !            34:  */
        !            35:
        !            36: /*
        !            37:  * uvm_amap.c: amap operations
        !            38:  */
        !            39:
        !            40: /*
        !            41:  * this file contains functions that perform operations on amaps.  see
        !            42:  * uvm_amap.h for a brief explanation of the role of amaps in uvm.
        !            43:  */
        !            44:
        !            45: #undef UVM_AMAP_INLINE         /* enable/disable amap inlines */
        !            46:
        !            47: #include <sys/param.h>
        !            48: #include <sys/systm.h>
        !            49: #include <sys/proc.h>
        !            50: #include <sys/malloc.h>
        !            51: #include <sys/kernel.h>
        !            52: #include <sys/pool.h>
        !            53:
        !            54: #define UVM_AMAP_C             /* ensure disabled inlines are in */
        !            55: #include <uvm/uvm.h>
        !            56: #include <uvm/uvm_swap.h>
        !            57:
        !            58: /*
        !            59:  * pool for allocation of vm_map structures.  note that the pool has
        !            60:  * its own simplelock for its protection.  also note that in order to
        !            61:  * avoid an endless loop, the amap pool's allocator cannot allocate
        !            62:  * memory from an amap (it currently goes through the kernel uobj, so
        !            63:  * we are ok).
        !            64:  */
        !            65:
        !            66: struct pool uvm_amap_pool;
        !            67:
        !            68: LIST_HEAD(, vm_amap) amap_list;
        !            69:
        !            70: /*
        !            71:  * local functions
        !            72:  */
        !            73:
        !            74: static struct vm_amap *amap_alloc1(int, int, int);
        !            75: static __inline void amap_list_insert(struct vm_amap *);
        !            76: static __inline void amap_list_remove(struct vm_amap *);
        !            77:
        !            78: static __inline void
        !            79: amap_list_insert(struct vm_amap *amap)
        !            80: {
        !            81:        LIST_INSERT_HEAD(&amap_list, amap, am_list);
        !            82: }
        !            83:
        !            84: static __inline void
        !            85: amap_list_remove(struct vm_amap *amap)
        !            86: {
        !            87:        LIST_REMOVE(amap, am_list);
        !            88: }
        !            89:
        !            90: #ifdef UVM_AMAP_PPREF
        !            91: /*
        !            92:  * what is ppref?   ppref is an _optional_ amap feature which is used
        !            93:  * to keep track of reference counts on a per-page basis.  it is enabled
        !            94:  * when UVM_AMAP_PPREF is defined.
        !            95:  *
        !            96:  * when enabled, an array of ints is allocated for the pprefs.  this
        !            97:  * array is allocated only when a partial reference is added to the
        !            98:  * map (either by unmapping part of the amap, or gaining a reference
        !            99:  * to only a part of an amap).  if the malloc of the array fails
        !           100:  * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate
        !           101:  * that we tried to do ppref's but couldn't alloc the array so just
        !           102:  * give up (after all, this is an optional feature!).
        !           103:  *
        !           104:  * the array is divided into page sized "chunks."   for chunks of length 1,
        !           105:  * the chunk reference count plus one is stored in that chunk's slot.
        !           106:  * for chunks of length > 1 the first slot contains (the reference count
        !           107:  * plus one) * -1.    [the negative value indicates that the length is
        !           108:  * greater than one.]   the second slot of the chunk contains the length
        !           109:  * of the chunk.   here is an example:
        !           110:  *
        !           111:  * actual REFS:  2  2  2  2  3  1  1  0  0  0  4  4  0  1  1  1
        !           112:  *       ppref: -3  4  x  x  4 -2  2 -1  3  x -5  2  1 -2  3  x
        !           113:  *              <----------><-><----><-------><----><-><------->
        !           114:  * (x = don't care)
        !           115:  *
        !           116:  * this allows us to allow one int to contain the ref count for the whole
        !           117:  * chunk.    note that the "plus one" part is needed because a reference
        !           118:  * count of zero is neither positive or negative (need a way to tell
        !           119:  * if we've got one zero or a bunch of them).
        !           120:  *
        !           121:  * here are some in-line functions to help us.
        !           122:  */
        !           123:
        !           124: static __inline void pp_getreflen(int *, int, int *, int *);
        !           125: static __inline void pp_setreflen(int *, int, int, int);
        !           126:
        !           127: /*
        !           128:  * pp_getreflen: get the reference and length for a specific offset
        !           129:  *
        !           130:  * => ppref's amap must be locked
        !           131:  */
        !           132: static __inline void
        !           133: pp_getreflen(int *ppref, int offset, int *refp, int *lenp)
        !           134: {
        !           135:
        !           136:        if (ppref[offset] > 0) {                /* chunk size must be 1 */
        !           137:                *refp = ppref[offset] - 1;      /* don't forget to adjust */
        !           138:                *lenp = 1;
        !           139:        } else {
        !           140:                *refp = (ppref[offset] * -1) - 1;
        !           141:                *lenp = ppref[offset+1];
        !           142:        }
        !           143: }
        !           144:
        !           145: /*
        !           146:  * pp_setreflen: set the reference and length for a specific offset
        !           147:  *
        !           148:  * => ppref's amap must be locked
        !           149:  */
        !           150: static __inline void
        !           151: pp_setreflen(int *ppref, int offset, int ref, int len)
        !           152: {
        !           153:        if (len == 1) {
        !           154:                ppref[offset] = ref + 1;
        !           155:        } else {
        !           156:                ppref[offset] = (ref + 1) * -1;
        !           157:                ppref[offset+1] = len;
        !           158:        }
        !           159: }
        !           160: #endif
        !           161:
        !           162: /*
        !           163:  * amap_init: called at boot time to init global amap data structures
        !           164:  */
        !           165:
        !           166: void
        !           167: amap_init(void)
        !           168: {
        !           169:        /*
        !           170:         * Initialize the vm_amap pool.
        !           171:         */
        !           172:        pool_init(&uvm_amap_pool, sizeof(struct vm_amap), 0, 0, 0,
        !           173:            "amappl", &pool_allocator_nointr);
        !           174:        pool_sethiwat(&uvm_amap_pool, 4096);
        !           175: }
        !           176:
        !           177: /*
        !           178:  * amap_alloc1: internal function that allocates an amap, but does not
        !           179:  *     init the overlay.
        !           180:  *
        !           181:  * => lock on returned amap is init'd
        !           182:  */
        !           183: static inline struct vm_amap *
        !           184: amap_alloc1(int slots, int padslots, int waitf)
        !           185: {
        !           186:        struct vm_amap *amap;
        !           187:        int totalslots;
        !           188:
        !           189:        amap = pool_get(&uvm_amap_pool, (waitf == M_WAITOK) ? PR_WAITOK : 0);
        !           190:        if (amap == NULL)
        !           191:                return(NULL);
        !           192:
        !           193:        totalslots = malloc_roundup((slots + padslots) * sizeof(int)) /
        !           194:            sizeof(int);
        !           195:        amap->am_ref = 1;
        !           196:        amap->am_flags = 0;
        !           197: #ifdef UVM_AMAP_PPREF
        !           198:        amap->am_ppref = NULL;
        !           199: #endif
        !           200:        amap->am_maxslot = totalslots;
        !           201:        amap->am_nslot = slots;
        !           202:        amap->am_nused = 0;
        !           203:
        !           204:        amap->am_slots = malloc(totalslots * sizeof(int), M_UVMAMAP,
        !           205:            waitf);
        !           206:        if (amap->am_slots == NULL)
        !           207:                goto fail1;
        !           208:
        !           209:        amap->am_bckptr = malloc(totalslots * sizeof(int), M_UVMAMAP, waitf);
        !           210:        if (amap->am_bckptr == NULL)
        !           211:                goto fail2;
        !           212:
        !           213:        amap->am_anon = malloc(totalslots * sizeof(struct vm_anon *),
        !           214:            M_UVMAMAP, waitf);
        !           215:        if (amap->am_anon == NULL)
        !           216:                goto fail3;
        !           217:
        !           218:        return(amap);
        !           219:
        !           220: fail3:
        !           221:        free(amap->am_bckptr, M_UVMAMAP);
        !           222: fail2:
        !           223:        free(amap->am_slots, M_UVMAMAP);
        !           224: fail1:
        !           225:        pool_put(&uvm_amap_pool, amap);
        !           226:        return (NULL);
        !           227: }
        !           228:
        !           229: /*
        !           230:  * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM
        !           231:  *
        !           232:  * => caller should ensure sz is a multiple of PAGE_SIZE
        !           233:  * => reference count to new amap is set to one
        !           234:  * => new amap is returned unlocked
        !           235:  */
        !           236:
        !           237: struct vm_amap *
        !           238: amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf)
        !           239: {
        !           240:        struct vm_amap *amap;
        !           241:        int slots, padslots;
        !           242:        UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist);
        !           243:
        !           244:        AMAP_B2SLOT(slots, sz);         /* load slots */
        !           245:        AMAP_B2SLOT(padslots, padsz);
        !           246:
        !           247:        amap = amap_alloc1(slots, padslots, waitf);
        !           248:        if (amap) {
        !           249:                memset(amap->am_anon, 0,
        !           250:                    amap->am_maxslot * sizeof(struct vm_anon *));
        !           251:                amap_list_insert(amap);
        !           252:        }
        !           253:
        !           254:        UVMHIST_LOG(maphist,"<- done, amap = %p, sz=%lu", amap, sz, 0, 0);
        !           255:        return(amap);
        !           256: }
        !           257:
        !           258:
        !           259: /*
        !           260:  * amap_free: free an amap
        !           261:  *
        !           262:  * => the amap must be locked (mainly for simplelock accounting)
        !           263:  * => the amap should have a zero reference count and be empty
        !           264:  */
        !           265: void
        !           266: amap_free(struct vm_amap *amap)
        !           267: {
        !           268:        UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist);
        !           269:
        !           270:        KASSERT(amap->am_ref == 0 && amap->am_nused == 0);
        !           271:        KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0);
        !           272:
        !           273:        free(amap->am_slots, M_UVMAMAP);
        !           274:        free(amap->am_bckptr, M_UVMAMAP);
        !           275:        free(amap->am_anon, M_UVMAMAP);
        !           276: #ifdef UVM_AMAP_PPREF
        !           277:        if (amap->am_ppref && amap->am_ppref != PPREF_NONE)
        !           278:                free(amap->am_ppref, M_UVMAMAP);
        !           279: #endif
        !           280:        pool_put(&uvm_amap_pool, amap);
        !           281:
        !           282:        UVMHIST_LOG(maphist,"<- done, freed amap = %p", amap, 0, 0, 0);
        !           283: }
        !           284:
        !           285: /*
        !           286:  * amap_extend: extend the size of an amap (if needed)
        !           287:  *
        !           288:  * => called from uvm_map when we want to extend an amap to cover
        !           289:  *    a new mapping (rather than allocate a new one)
        !           290:  * => amap should be unlocked (we will lock it)
        !           291:  * => to safely extend an amap it should have a reference count of
        !           292:  *    one (thus it can't be shared)
        !           293:  * => XXXCDC: support padding at this level?
        !           294:  */
        !           295: int
        !           296: amap_extend(struct vm_map_entry *entry, vsize_t addsize)
        !           297: {
        !           298:        struct vm_amap *amap = entry->aref.ar_amap;
        !           299:        int slotoff = entry->aref.ar_pageoff;
        !           300:        int slotmapped, slotadd, slotneed, slotalloc;
        !           301: #ifdef UVM_AMAP_PPREF
        !           302:        int *newppref, *oldppref;
        !           303: #endif
        !           304:        u_int *newsl, *newbck, *oldsl, *oldbck;
        !           305:        struct vm_anon **newover, **oldover;
        !           306:        int slotadded;
        !           307:        UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist);
        !           308:
        !           309:        UVMHIST_LOG(maphist, "  (entry=%p, addsize=%lu)", entry, addsize, 0, 0);
        !           310:
        !           311:        /*
        !           312:         * first, determine how many slots we need in the amap.  don't
        !           313:         * forget that ar_pageoff could be non-zero: this means that
        !           314:         * there are some unused slots before us in the amap.
        !           315:         */
        !           316:
        !           317:        AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */
        !           318:        AMAP_B2SLOT(slotadd, addsize);                  /* slots to add */
        !           319:        slotneed = slotoff + slotmapped + slotadd;
        !           320:
        !           321:        /*
        !           322:         * case 1: we already have enough slots in the map and thus
        !           323:         * only need to bump the reference counts on the slots we are
        !           324:         * adding.
        !           325:         */
        !           326:
        !           327:        if (amap->am_nslot >= slotneed) {
        !           328: #ifdef UVM_AMAP_PPREF
        !           329:                if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
        !           330:                        amap_pp_adjref(amap, slotoff + slotmapped, slotadd, 1);
        !           331:                }
        !           332: #endif
        !           333:                UVMHIST_LOG(maphist,"<- done (case 1), amap = %p, sltneed=%ld",
        !           334:                    amap, slotneed, 0, 0);
        !           335:                return (0);
        !           336:        }
        !           337:
        !           338:        /*
        !           339:         * case 2: we pre-allocated slots for use and we just need to
        !           340:         * bump nslot up to take account for these slots.
        !           341:         */
        !           342:
        !           343:        if (amap->am_maxslot >= slotneed) {
        !           344: #ifdef UVM_AMAP_PPREF
        !           345:                if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
        !           346:                        if ((slotoff + slotmapped) < amap->am_nslot)
        !           347:                                amap_pp_adjref(amap, slotoff + slotmapped,
        !           348:                                    (amap->am_nslot - (slotoff + slotmapped)),
        !           349:                                    1);
        !           350:                        pp_setreflen(amap->am_ppref, amap->am_nslot, 1,
        !           351:                           slotneed - amap->am_nslot);
        !           352:                }
        !           353: #endif
        !           354:                amap->am_nslot = slotneed;
        !           355:
        !           356:                /*
        !           357:                 * no need to zero am_anon since that was done at
        !           358:                 * alloc time and we never shrink an allocation.
        !           359:                 */
        !           360:                UVMHIST_LOG(maphist,"<- done (case 2), amap = %p, slotneed=%ld",
        !           361:                    amap, slotneed, 0, 0);
        !           362:                return (0);
        !           363:        }
        !           364:
        !           365:        /*
        !           366:         * case 3: we need to malloc a new amap and copy all the amap
        !           367:         * data over from old amap to the new one.
        !           368:         *
        !           369:         * XXXCDC: could we take advantage of a kernel realloc()?
        !           370:         */
        !           371:
        !           372:        slotalloc = malloc_roundup(slotneed * sizeof(int)) / sizeof(int);
        !           373: #ifdef UVM_AMAP_PPREF
        !           374:        newppref = NULL;
        !           375:        if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
        !           376:                newppref = malloc(slotalloc *sizeof(int), M_UVMAMAP,
        !           377:                    M_WAITOK | M_CANFAIL);
        !           378:                if (newppref == NULL) {
        !           379:                        /* give up if malloc fails */
        !           380:                        free(amap->am_ppref, M_UVMAMAP);
        !           381:                        amap->am_ppref = PPREF_NONE;
        !           382:                }
        !           383:        }
        !           384: #endif
        !           385:        newsl = malloc(slotalloc * sizeof(int), M_UVMAMAP,
        !           386:            M_WAITOK | M_CANFAIL);
        !           387:        newbck = malloc(slotalloc * sizeof(int), M_UVMAMAP,
        !           388:            M_WAITOK | M_CANFAIL);
        !           389:        newover = malloc(slotalloc * sizeof(struct vm_anon *), M_UVMAMAP,
        !           390:            M_WAITOK | M_CANFAIL);
        !           391:        if (newsl == NULL || newbck == NULL || newover == NULL) {
        !           392:                if (newsl != NULL) {
        !           393:                        free(newsl, M_UVMAMAP);
        !           394:                }
        !           395:                if (newbck != NULL) {
        !           396:                        free(newbck, M_UVMAMAP);
        !           397:                }
        !           398:                if (newover != NULL) {
        !           399:                        free(newover, M_UVMAMAP);
        !           400:                }
        !           401:                return (ENOMEM);
        !           402:        }
        !           403:        KASSERT(amap->am_maxslot < slotneed);
        !           404:
        !           405:        /*
        !           406:         * now copy everything over to new malloc'd areas...
        !           407:         */
        !           408:
        !           409:        slotadded = slotalloc - amap->am_nslot;
        !           410:
        !           411:        /* do am_slots */
        !           412:        oldsl = amap->am_slots;
        !           413:        memcpy(newsl, oldsl, sizeof(int) * amap->am_nused);
        !           414:        amap->am_slots = newsl;
        !           415:
        !           416:        /* do am_anon */
        !           417:        oldover = amap->am_anon;
        !           418:        memcpy(newover, oldover, sizeof(struct vm_anon *) * amap->am_nslot);
        !           419:        memset(newover + amap->am_nslot, 0, sizeof(struct vm_anon *) *
        !           420:            slotadded);
        !           421:        amap->am_anon = newover;
        !           422:
        !           423:        /* do am_bckptr */
        !           424:        oldbck = amap->am_bckptr;
        !           425:        memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot);
        !           426:        memset(newbck + amap->am_nslot, 0, sizeof(int) * slotadded); /* XXX: needed? */
        !           427:        amap->am_bckptr = newbck;
        !           428:
        !           429: #ifdef UVM_AMAP_PPREF
        !           430:        /* do ppref */
        !           431:        oldppref = amap->am_ppref;
        !           432:        if (newppref) {
        !           433:                memcpy(newppref, oldppref, sizeof(int) * amap->am_nslot);
        !           434:                memset(newppref + amap->am_nslot, 0, sizeof(int) * slotadded);
        !           435:                amap->am_ppref = newppref;
        !           436:                if ((slotoff + slotmapped) < amap->am_nslot)
        !           437:                        amap_pp_adjref(amap, slotoff + slotmapped,
        !           438:                            (amap->am_nslot - (slotoff + slotmapped)), 1);
        !           439:                pp_setreflen(newppref, amap->am_nslot, 1,
        !           440:                    slotneed - amap->am_nslot);
        !           441:        }
        !           442: #endif
        !           443:
        !           444:        /* update master values */
        !           445:        amap->am_nslot = slotneed;
        !           446:        amap->am_maxslot = slotalloc;
        !           447:
        !           448:        /* and free */
        !           449:        free(oldsl, M_UVMAMAP);
        !           450:        free(oldbck, M_UVMAMAP);
        !           451:        free(oldover, M_UVMAMAP);
        !           452: #ifdef UVM_AMAP_PPREF
        !           453:        if (oldppref && oldppref != PPREF_NONE)
        !           454:                free(oldppref, M_UVMAMAP);
        !           455: #endif
        !           456:        UVMHIST_LOG(maphist,"<- done (case 3), amap = %p, slotneed=%ld",
        !           457:            amap, slotneed, 0, 0);
        !           458:        return (0);
        !           459: }
        !           460:
        !           461: /*
        !           462:  * amap_share_protect: change protection of anons in a shared amap
        !           463:  *
        !           464:  * for shared amaps, given the current data structure layout, it is
        !           465:  * not possible for us to directly locate all maps referencing the
        !           466:  * shared anon (to change the protection).  in order to protect data
        !           467:  * in shared maps we use pmap_page_protect().  [this is useful for IPC
        !           468:  * mechanisms like map entry passing that may want to write-protect
        !           469:  * all mappings of a shared amap.]  we traverse am_anon or am_slots
        !           470:  * depending on the current state of the amap.
        !           471:  *
        !           472:  * => entry's map and amap must be locked by the caller
        !           473:  */
        !           474: void
        !           475: amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot)
        !           476: {
        !           477:        struct vm_amap *amap = entry->aref.ar_amap;
        !           478:        int slots, lcv, slot, stop;
        !           479:
        !           480:        AMAP_B2SLOT(slots, (entry->end - entry->start));
        !           481:        stop = entry->aref.ar_pageoff + slots;
        !           482:
        !           483:        if (slots < amap->am_nused) {
        !           484:                /* cheaper to traverse am_anon */
        !           485:                for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) {
        !           486:                        if (amap->am_anon[lcv] == NULL)
        !           487:                                continue;
        !           488:                        if (amap->am_anon[lcv]->an_page != NULL)
        !           489:                                pmap_page_protect(amap->am_anon[lcv]->an_page,
        !           490:                                                  prot);
        !           491:                }
        !           492:                return;
        !           493:        }
        !           494:
        !           495:        /* cheaper to traverse am_slots */
        !           496:        for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
        !           497:                slot = amap->am_slots[lcv];
        !           498:                if (slot < entry->aref.ar_pageoff || slot >= stop)
        !           499:                        continue;
        !           500:                if (amap->am_anon[slot]->an_page != NULL)
        !           501:                        pmap_page_protect(amap->am_anon[slot]->an_page, prot);
        !           502:        }
        !           503:        return;
        !           504: }
        !           505:
        !           506: /*
        !           507:  * amap_wipeout: wipeout all anon's in an amap; then free the amap!
        !           508:  *
        !           509:  * => called from amap_unref when the final reference to an amap is
        !           510:  *     discarded (i.e. when reference count == 1)
        !           511:  * => the amap should be locked (by the caller)
        !           512:  */
        !           513:
        !           514: void
        !           515: amap_wipeout(struct vm_amap *amap)
        !           516: {
        !           517:        int lcv, slot;
        !           518:        struct vm_anon *anon;
        !           519:        UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
        !           520:        UVMHIST_LOG(maphist,"(amap=%p)", amap, 0,0,0);
        !           521:
        !           522:        KASSERT(amap->am_ref == 0);
        !           523:
        !           524:        if (__predict_false((amap->am_flags & AMAP_SWAPOFF) != 0)) {
        !           525:                /*
        !           526:                 * amap_swap_off will call us again.
        !           527:                 */
        !           528:                return;
        !           529:        }
        !           530:        amap_list_remove(amap);
        !           531:
        !           532:        for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
        !           533:                int refs;
        !           534:
        !           535:                slot = amap->am_slots[lcv];
        !           536:                anon = amap->am_anon[slot];
        !           537:
        !           538:                if (anon == NULL || anon->an_ref == 0)
        !           539:                        panic("amap_wipeout: corrupt amap");
        !           540:
        !           541:                simple_lock(&anon->an_lock); /* lock anon */
        !           542:
        !           543:                UVMHIST_LOG(maphist,"  processing anon %p, ref=%ld", anon,
        !           544:                    anon->an_ref, 0, 0);
        !           545:
        !           546:                refs = --anon->an_ref;
        !           547:                simple_unlock(&anon->an_lock);
        !           548:                if (refs == 0) {
        !           549:                        /*
        !           550:                         * we had the last reference to a vm_anon. free it.
        !           551:                         */
        !           552:                        uvm_anfree(anon);
        !           553:                }
        !           554:        }
        !           555:
        !           556:        /*
        !           557:         * now we free the map
        !           558:         */
        !           559:
        !           560:        amap->am_ref = 0;       /* ... was one */
        !           561:        amap->am_nused = 0;
        !           562:        amap_free(amap);        /* will unlock and free amap */
        !           563:        UVMHIST_LOG(maphist,"<- done!", 0,0,0,0);
        !           564: }
        !           565:
        !           566: /*
        !           567:  * amap_copy: ensure that a map entry's "needs_copy" flag is false
        !           568:  *     by copying the amap if necessary.
        !           569:  *
        !           570:  * => an entry with a null amap pointer will get a new (blank) one.
        !           571:  * => the map that the map entry belongs to must be locked by caller.
        !           572:  * => the amap currently attached to "entry" (if any) must be unlocked.
        !           573:  * => if canchunk is true, then we may clip the entry into a chunk
        !           574:  * => "startva" and "endva" are used only if canchunk is true.  they are
        !           575:  *     used to limit chunking (e.g. if you have a large space that you
        !           576:  *     know you are going to need to allocate amaps for, there is no point
        !           577:  *     in allowing that to be chunked)
        !           578:  */
        !           579:
        !           580: void
        !           581: amap_copy(struct vm_map *map, struct vm_map_entry *entry, int waitf,
        !           582:     boolean_t canchunk, vaddr_t startva, vaddr_t endva)
        !           583: {
        !           584:        struct vm_amap *amap, *srcamap;
        !           585:        int slots, lcv;
        !           586:        vaddr_t chunksize;
        !           587:        UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist);
        !           588:        UVMHIST_LOG(maphist, "  (map=%p, entry=%p, waitf=%ld)",
        !           589:                    map, entry, waitf, 0);
        !           590:
        !           591:        /*
        !           592:         * is there a map to copy?   if not, create one from scratch.
        !           593:         */
        !           594:
        !           595:        if (entry->aref.ar_amap == NULL) {
        !           596:
        !           597:                /*
        !           598:                 * check to see if we have a large amap that we can
        !           599:                 * chunk.  we align startva/endva to chunk-sized
        !           600:                 * boundaries and then clip to them.
        !           601:                 */
        !           602:
        !           603:                if (canchunk && atop(entry->end - entry->start) >=
        !           604:                    UVM_AMAP_LARGE) {
        !           605:                        /* convert slots to bytes */
        !           606:                        chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT;
        !           607:                        startva = (startva / chunksize) * chunksize;
        !           608:                        endva = roundup(endva, chunksize);
        !           609:                        UVMHIST_LOG(maphist, "  chunk amap ==> clip "
        !           610:                            "0x%lx->0x%lx to 0x%lx->0x%lx",
        !           611:                            entry->start, entry->end, startva, endva);
        !           612:                        UVM_MAP_CLIP_START(map, entry, startva);
        !           613:                        /* watch out for endva wrap-around! */
        !           614:                        if (endva >= startva)
        !           615:                                UVM_MAP_CLIP_END(map, entry, endva);
        !           616:                }
        !           617:
        !           618:                UVMHIST_LOG(maphist, "<- done [creating new amap 0x%lx->0x%lx]",
        !           619:                    entry->start, entry->end, 0, 0);
        !           620:                entry->aref.ar_pageoff = 0;
        !           621:                entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 0,
        !           622:                    waitf);
        !           623:                if (entry->aref.ar_amap != NULL)
        !           624:                        entry->etype &= ~UVM_ET_NEEDSCOPY;
        !           625:                return;
        !           626:        }
        !           627:
        !           628:        /*
        !           629:         * first check and see if we are the only map entry
        !           630:         * referencing the amap we currently have.  if so, then we can
        !           631:         * just take it over rather than copying it.  note that we are
        !           632:         * reading am_ref with the amap unlocked... the value can only
        !           633:         * be one if we have the only reference to the amap (via our
        !           634:         * locked map).  if we are greater than one we fall through to
        !           635:         * the next case (where we double check the value).
        !           636:         */
        !           637:
        !           638:        if (entry->aref.ar_amap->am_ref == 1) {
        !           639:                entry->etype &= ~UVM_ET_NEEDSCOPY;
        !           640:                UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]",
        !           641:                    0, 0, 0, 0);
        !           642:                return;
        !           643:        }
        !           644:
        !           645:        /*
        !           646:         * looks like we need to copy the map.
        !           647:         */
        !           648:
        !           649:        UVMHIST_LOG(maphist,"  amap=%p, ref=%ld, must copy it",
        !           650:            entry->aref.ar_amap, entry->aref.ar_amap->am_ref, 0, 0);
        !           651:        AMAP_B2SLOT(slots, entry->end - entry->start);
        !           652:        amap = amap_alloc1(slots, 0, waitf);
        !           653:        if (amap == NULL) {
        !           654:                UVMHIST_LOG(maphist, "  amap_alloc1 failed", 0,0,0,0);
        !           655:                return;
        !           656:        }
        !           657:        srcamap = entry->aref.ar_amap;
        !           658:
        !           659:        /*
        !           660:         * need to double check reference count now that we've got the
        !           661:         * src amap locked down.  the reference count could have
        !           662:         * changed while we were in malloc.  if the reference count
        !           663:         * dropped down to one we take over the old map rather than
        !           664:         * copying the amap.
        !           665:         */
        !           666:
        !           667:        if (srcamap->am_ref == 1) {             /* take it over? */
        !           668:                entry->etype &= ~UVM_ET_NEEDSCOPY;
        !           669:                amap->am_ref--;         /* drop final reference to map */
        !           670:                amap_free(amap);        /* dispose of new (unused) amap */
        !           671:                return;
        !           672:        }
        !           673:
        !           674:        /*
        !           675:         * we must copy it now.
        !           676:         */
        !           677:
        !           678:        UVMHIST_LOG(maphist, "  copying amap now",0, 0, 0, 0);
        !           679:        for (lcv = 0 ; lcv < slots; lcv++) {
        !           680:                amap->am_anon[lcv] =
        !           681:                    srcamap->am_anon[entry->aref.ar_pageoff + lcv];
        !           682:                if (amap->am_anon[lcv] == NULL)
        !           683:                        continue;
        !           684:                simple_lock(&amap->am_anon[lcv]->an_lock);
        !           685:                amap->am_anon[lcv]->an_ref++;
        !           686:                simple_unlock(&amap->am_anon[lcv]->an_lock);
        !           687:                amap->am_bckptr[lcv] = amap->am_nused;
        !           688:                amap->am_slots[amap->am_nused] = lcv;
        !           689:                amap->am_nused++;
        !           690:        }
        !           691:        memset(&amap->am_anon[lcv], 0,
        !           692:            (amap->am_maxslot - lcv) * sizeof(struct vm_anon *));
        !           693:
        !           694:        /*
        !           695:         * drop our reference to the old amap (srcamap) and unlock.
        !           696:         * we know that the reference count on srcamap is greater than
        !           697:         * one (we checked above), so there is no way we could drop
        !           698:         * the count to zero.  [and no need to worry about freeing it]
        !           699:         */
        !           700:
        !           701:        srcamap->am_ref--;
        !           702:        if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0)
        !           703:                srcamap->am_flags &= ~AMAP_SHARED;   /* clear shared flag */
        !           704: #ifdef UVM_AMAP_PPREF
        !           705:        if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) {
        !           706:                amap_pp_adjref(srcamap, entry->aref.ar_pageoff,
        !           707:                    (entry->end - entry->start) >> PAGE_SHIFT, -1);
        !           708:        }
        !           709: #endif
        !           710:
        !           711:        /*
        !           712:         * install new amap.
        !           713:         */
        !           714:
        !           715:        entry->aref.ar_pageoff = 0;
        !           716:        entry->aref.ar_amap = amap;
        !           717:        entry->etype &= ~UVM_ET_NEEDSCOPY;
        !           718:
        !           719:        amap_list_insert(amap);
        !           720:
        !           721:        /*
        !           722:         * done!
        !           723:         */
        !           724:        UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0);
        !           725: }
        !           726:
        !           727: /*
        !           728:  * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2)
        !           729:  *
        !           730:  *     called during fork(2) when the parent process has a wired map
        !           731:  *     entry.   in that case we want to avoid write-protecting pages
        !           732:  *     in the parent's map (e.g. like what you'd do for a COW page)
        !           733:  *     so we resolve the COW here.
        !           734:  *
        !           735:  * => assume parent's entry was wired, thus all pages are resident.
        !           736:  * => assume pages that are loaned out (loan_count) are already mapped
        !           737:  *     read-only in all maps, and thus no need for us to worry about them
        !           738:  * => assume both parent and child vm_map's are locked
        !           739:  * => caller passes child's map/entry in to us
        !           740:  * => if we run out of memory we will unlock the amap and sleep _with_ the
        !           741:  *     parent and child vm_map's locked(!).    we have to do this since
        !           742:  *     we are in the middle of a fork(2) and we can't let the parent
        !           743:  *     map change until we are done copying all the map entries.
        !           744:  * => XXXCDC: out of memory should cause fork to fail, but there is
        !           745:  *     currently no easy way to do this (needs fix)
        !           746:  * => page queues must be unlocked (we may lock them)
        !           747:  */
        !           748:
        !           749: void
        !           750: amap_cow_now(struct vm_map *map, struct vm_map_entry *entry)
        !           751: {
        !           752:        struct vm_amap *amap = entry->aref.ar_amap;
        !           753:        int lcv, slot;
        !           754:        struct vm_anon *anon, *nanon;
        !           755:        struct vm_page *pg, *npg;
        !           756:
        !           757:        /*
        !           758:         * note that if we unlock the amap then we must ReStart the "lcv" for
        !           759:         * loop because some other process could reorder the anon's in the
        !           760:         * am_anon[] array on us while the lock is dropped.
        !           761:         */
        !           762: ReStart:
        !           763:        for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
        !           764:
        !           765:                /*
        !           766:                 * get the page
        !           767:                 */
        !           768:
        !           769:                slot = amap->am_slots[lcv];
        !           770:                anon = amap->am_anon[slot];
        !           771:                simple_lock(&anon->an_lock);
        !           772:                pg = anon->an_page;
        !           773:
        !           774:                /*
        !           775:                 * page must be resident since parent is wired
        !           776:                 */
        !           777:
        !           778:                if (pg == NULL)
        !           779:                    panic("amap_cow_now: non-resident wired page in anon %p",
        !           780:                        anon);
        !           781:
        !           782:                /*
        !           783:                 * if the anon ref count is one and the page is not loaned,
        !           784:                 * then we are safe (the child has exclusive access to the
        !           785:                 * page).  if the page is loaned, then it must already be
        !           786:                 * mapped read-only.
        !           787:                 *
        !           788:                 * we only need to get involved when these are not true.
        !           789:                 * [note: if loan_count == 0, then the anon must own the page]
        !           790:                 */
        !           791:
        !           792:                if (anon->an_ref > 1 && pg->loan_count == 0) {
        !           793:
        !           794:                        /*
        !           795:                         * if the page is busy then we have to unlock, wait for
        !           796:                         * it and then restart.
        !           797:                         */
        !           798:                        if (pg->pg_flags & PG_BUSY) {
        !           799:                                atomic_setbits_int(&pg->pg_flags, PG_WANTED);
        !           800:                                UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, FALSE,
        !           801:                                    "cownow", 0);
        !           802:                                goto ReStart;
        !           803:                        }
        !           804:
        !           805:                        /*
        !           806:                         * ok, time to do a copy-on-write to a new anon
        !           807:                         */
        !           808:                        nanon = uvm_analloc();
        !           809:                        if (nanon) {
        !           810:                                npg = uvm_pagealloc(NULL, 0, nanon, 0);
        !           811:                        } else
        !           812:                                npg = NULL;     /* XXX: quiet gcc warning */
        !           813:
        !           814:                        if (nanon == NULL || npg == NULL) {
        !           815:                                /* out of memory */
        !           816:                                /*
        !           817:                                 * XXXCDC: we should cause fork to fail, but
        !           818:                                 * we can't ...
        !           819:                                 */
        !           820:                                if (nanon) {
        !           821:                                        simple_lock(&nanon->an_lock);
        !           822:                                        uvm_anfree(nanon);
        !           823:                                }
        !           824:                                simple_unlock(&anon->an_lock);
        !           825:                                uvm_wait("cownowpage");
        !           826:                                goto ReStart;
        !           827:                        }
        !           828:
        !           829:                        /*
        !           830:                         * got it... now we can copy the data and replace anon
        !           831:                         * with our new one...
        !           832:                         */
        !           833:                        uvm_pagecopy(pg, npg);          /* old -> new */
        !           834:                        anon->an_ref--;                 /* can't drop to zero */
        !           835:                        amap->am_anon[slot] = nanon;    /* replace */
        !           836:
        !           837:                        /*
        !           838:                         * drop PG_BUSY on new page ... since we have had it's
        !           839:                         * owner locked the whole time it can't be
        !           840:                         * PG_RELEASED | PG_WANTED.
        !           841:                         */
        !           842:                        atomic_clearbits_int(&npg->pg_flags, PG_BUSY|PG_FAKE);
        !           843:                        UVM_PAGE_OWN(npg, NULL);
        !           844:                        uvm_lock_pageq();
        !           845:                        uvm_pageactivate(npg);
        !           846:                        uvm_unlock_pageq();
        !           847:                }
        !           848:
        !           849:                simple_unlock(&anon->an_lock);
        !           850:                /*
        !           851:                 * done with this anon, next ...!
        !           852:                 */
        !           853:
        !           854:        }       /* end of 'for' loop */
        !           855: }
        !           856:
        !           857: /*
        !           858:  * amap_splitref: split a single reference into two separate references
        !           859:  *
        !           860:  * => called from uvm_map's clip routines
        !           861:  * => origref's map should be locked
        !           862:  * => origref->ar_amap should be unlocked (we will lock)
        !           863:  */
        !           864: void
        !           865: amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset)
        !           866: {
        !           867:        int leftslots;
        !           868:
        !           869:        AMAP_B2SLOT(leftslots, offset);
        !           870:        if (leftslots == 0)
        !           871:                panic("amap_splitref: split at zero offset");
        !           872:
        !           873:        /*
        !           874:         * now: amap is locked and we have a valid am_mapped array.
        !           875:         */
        !           876:
        !           877:        if (origref->ar_amap->am_nslot - origref->ar_pageoff - leftslots <= 0)
        !           878:                panic("amap_splitref: map size check failed");
        !           879:
        !           880: #ifdef UVM_AMAP_PPREF
        !           881:         /*
        !           882:         * establish ppref before we add a duplicate reference to the amap
        !           883:         */
        !           884:        if (origref->ar_amap->am_ppref == NULL)
        !           885:                amap_pp_establish(origref->ar_amap);
        !           886: #endif
        !           887:
        !           888:        splitref->ar_amap = origref->ar_amap;
        !           889:        splitref->ar_amap->am_ref++;            /* not a share reference */
        !           890:        splitref->ar_pageoff = origref->ar_pageoff + leftslots;
        !           891: }
        !           892:
        !           893: #ifdef UVM_AMAP_PPREF
        !           894:
        !           895: /*
        !           896:  * amap_pp_establish: add a ppref array to an amap, if possible
        !           897:  *
        !           898:  * => amap locked by caller
        !           899:  */
        !           900: void
        !           901: amap_pp_establish(struct vm_amap *amap)
        !           902: {
        !           903:
        !           904:        amap->am_ppref = malloc(sizeof(int) * amap->am_maxslot,
        !           905:            M_UVMAMAP, M_NOWAIT);
        !           906:
        !           907:        /*
        !           908:         * if we fail then we just won't use ppref for this amap
        !           909:         */
        !           910:        if (amap->am_ppref == NULL) {
        !           911:                amap->am_ppref = PPREF_NONE;    /* not using it */
        !           912:                return;
        !           913:        }
        !           914:
        !           915:        /*
        !           916:         * init ppref
        !           917:         */
        !           918:        memset(amap->am_ppref, 0, sizeof(int) * amap->am_maxslot);
        !           919:        pp_setreflen(amap->am_ppref, 0, amap->am_ref, amap->am_nslot);
        !           920: }
        !           921:
        !           922: /*
        !           923:  * amap_pp_adjref: adjust reference count to a part of an amap using the
        !           924:  * per-page reference count array.
        !           925:  *
        !           926:  * => map and amap locked by caller
        !           927:  * => caller must check that ppref != PPREF_NONE before calling
        !           928:  */
        !           929: void
        !           930: amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval)
        !           931: {
        !           932:        int stopslot, *ppref, lcv, prevlcv;
        !           933:        int ref, len, prevref, prevlen;
        !           934:
        !           935:        stopslot = curslot + slotlen;
        !           936:        ppref = amap->am_ppref;
        !           937:        prevlcv = 0;
        !           938:
        !           939:        /*
        !           940:         * first advance to the correct place in the ppref array,
        !           941:         * fragment if needed.
        !           942:         */
        !           943:
        !           944:        for (lcv = 0 ; lcv < curslot ; lcv += len) {
        !           945:                pp_getreflen(ppref, lcv, &ref, &len);
        !           946:                if (lcv + len > curslot) {     /* goes past start? */
        !           947:                        pp_setreflen(ppref, lcv, ref, curslot - lcv);
        !           948:                        pp_setreflen(ppref, curslot, ref, len - (curslot -lcv));
        !           949:                        len = curslot - lcv;   /* new length of entry @ lcv */
        !           950:                }
        !           951:                prevlcv = lcv;
        !           952:        }
        !           953:        if (lcv != 0)
        !           954:                pp_getreflen(ppref, prevlcv, &prevref, &prevlen);
        !           955:        else {
        !           956:                /* Ensure that the "prevref == ref" test below always
        !           957:                 * fails, since we're starting from the beginning of
        !           958:                 * the ppref array; that is, there is no previous
        !           959:                 * chunk.
        !           960:                 */
        !           961:                prevref = -1;
        !           962:                prevlen = 0;
        !           963:        }
        !           964:
        !           965:        /*
        !           966:         * now adjust reference counts in range.  merge the first
        !           967:         * changed entry with the last unchanged entry if possible.
        !           968:         */
        !           969:
        !           970:        if (lcv != curslot)
        !           971:                panic("amap_pp_adjref: overshot target");
        !           972:
        !           973:        for (/* lcv already set */; lcv < stopslot ; lcv += len) {
        !           974:                pp_getreflen(ppref, lcv, &ref, &len);
        !           975:                if (lcv + len > stopslot) {     /* goes past end? */
        !           976:                        pp_setreflen(ppref, lcv, ref, stopslot - lcv);
        !           977:                        pp_setreflen(ppref, stopslot, ref,
        !           978:                            len - (stopslot - lcv));
        !           979:                        len = stopslot - lcv;
        !           980:                }
        !           981:                ref += adjval;
        !           982:                if (ref < 0)
        !           983:                        panic("amap_pp_adjref: negative reference count");
        !           984:                if (lcv == prevlcv + prevlen && ref == prevref) {
        !           985:                        pp_setreflen(ppref, prevlcv, ref, prevlen + len);
        !           986:                } else {
        !           987:                        pp_setreflen(ppref, lcv, ref, len);
        !           988:                }
        !           989:                if (ref == 0)
        !           990:                        amap_wiperange(amap, lcv, len);
        !           991:        }
        !           992:
        !           993: }
        !           994:
        !           995: /*
        !           996:  * amap_wiperange: wipe out a range of an amap
        !           997:  * [different from amap_wipeout because the amap is kept intact]
        !           998:  *
        !           999:  * => both map and amap must be locked by caller.
        !          1000:  */
        !          1001: void
        !          1002: amap_wiperange(struct vm_amap *amap, int slotoff, int slots)
        !          1003: {
        !          1004:        int byanon, lcv, stop, curslot, ptr, slotend;
        !          1005:        struct vm_anon *anon;
        !          1006:
        !          1007:        /*
        !          1008:         * we can either traverse the amap by am_anon or by am_slots depending
        !          1009:         * on which is cheaper.    decide now.
        !          1010:         */
        !          1011:
        !          1012:        if (slots < amap->am_nused) {
        !          1013:                byanon = TRUE;
        !          1014:                lcv = slotoff;
        !          1015:                stop = slotoff + slots;
        !          1016:        } else {
        !          1017:                byanon = FALSE;
        !          1018:                lcv = 0;
        !          1019:                stop = amap->am_nused;
        !          1020:                slotend = slotoff + slots;
        !          1021:        }
        !          1022:
        !          1023:        while (lcv < stop) {
        !          1024:                int refs;
        !          1025:
        !          1026:                if (byanon) {
        !          1027:                        curslot = lcv++;        /* lcv advances here */
        !          1028:                        if (amap->am_anon[curslot] == NULL)
        !          1029:                                continue;
        !          1030:                } else {
        !          1031:                        curslot = amap->am_slots[lcv];
        !          1032:                        if (curslot < slotoff || curslot >= slotend) {
        !          1033:                                lcv++;          /* lcv advances here */
        !          1034:                                continue;
        !          1035:                        }
        !          1036:                        stop--; /* drop stop, since anon will be removed */
        !          1037:                }
        !          1038:                anon = amap->am_anon[curslot];
        !          1039:
        !          1040:                /*
        !          1041:                 * remove it from the amap
        !          1042:                 */
        !          1043:                amap->am_anon[curslot] = NULL;
        !          1044:                ptr = amap->am_bckptr[curslot];
        !          1045:                if (ptr != (amap->am_nused - 1)) {
        !          1046:                        amap->am_slots[ptr] =
        !          1047:                            amap->am_slots[amap->am_nused - 1];
        !          1048:                        amap->am_bckptr[amap->am_slots[ptr]] =
        !          1049:                            ptr;    /* back ptr. */
        !          1050:                }
        !          1051:                amap->am_nused--;
        !          1052:
        !          1053:                /*
        !          1054:                 * drop anon reference count
        !          1055:                 */
        !          1056:                simple_lock(&anon->an_lock);
        !          1057:                refs = --anon->an_ref;
        !          1058:                simple_unlock(&anon->an_lock);
        !          1059:                if (refs == 0) {
        !          1060:                        /*
        !          1061:                         * we just eliminated the last reference to an anon.
        !          1062:                         * free it.
        !          1063:                         */
        !          1064:                        uvm_anfree(anon);
        !          1065:                }
        !          1066:        }
        !          1067: }
        !          1068:
        !          1069: #endif
        !          1070:
        !          1071: /*
        !          1072:  * amap_swap_off: pagein anonymous pages in amaps and drop swap slots.
        !          1073:  *
        !          1074:  * => called with swap_syscall_lock held.
        !          1075:  * => note that we don't always traverse all anons.
        !          1076:  *    eg. amaps being wiped out, released anons.
        !          1077:  * => return TRUE if failed.
        !          1078:  */
        !          1079:
        !          1080: boolean_t
        !          1081: amap_swap_off(int startslot, int endslot)
        !          1082: {
        !          1083:        struct vm_amap *am;
        !          1084:        struct vm_amap *am_next;
        !          1085:        struct vm_amap marker_prev;
        !          1086:        struct vm_amap marker_next;
        !          1087:        boolean_t rv = FALSE;
        !          1088:
        !          1089: #if defined(DIAGNOSTIC)
        !          1090:        memset(&marker_prev, 0, sizeof(marker_prev));
        !          1091:        memset(&marker_next, 0, sizeof(marker_next));
        !          1092: #endif /* defined(DIAGNOSTIC) */
        !          1093:
        !          1094:        for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) {
        !          1095:                int i;
        !          1096:
        !          1097:                LIST_INSERT_BEFORE(am, &marker_prev, am_list);
        !          1098:                LIST_INSERT_AFTER(am, &marker_next, am_list);
        !          1099:
        !          1100:                if (am->am_nused <= 0) {
        !          1101:                        goto next;
        !          1102:                }
        !          1103:
        !          1104:                for (i = 0; i < am->am_nused; i++) {
        !          1105:                        int slot;
        !          1106:                        int swslot;
        !          1107:                        struct vm_anon *anon;
        !          1108:
        !          1109:                        slot = am->am_slots[i];
        !          1110:                        anon = am->am_anon[slot];
        !          1111:                        simple_lock(&anon->an_lock);
        !          1112:
        !          1113:                        swslot = anon->an_swslot;
        !          1114:                        if (swslot < startslot || endslot <= swslot) {
        !          1115:                                simple_unlock(&anon->an_lock);
        !          1116:                                continue;
        !          1117:                        }
        !          1118:
        !          1119:                        am->am_flags |= AMAP_SWAPOFF;
        !          1120:
        !          1121:                        rv = uvm_anon_pagein(anon);
        !          1122:
        !          1123:                        am->am_flags &= ~AMAP_SWAPOFF;
        !          1124:                        if (amap_refs(am) == 0) {
        !          1125:                                amap_wipeout(am);
        !          1126:                                am = NULL;
        !          1127:                                break;
        !          1128:                        }
        !          1129:                        if (rv) {
        !          1130:                                break;
        !          1131:                        }
        !          1132:                        i = 0;
        !          1133:                }
        !          1134:
        !          1135: next:
        !          1136:                KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next ||
        !          1137:                    LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) ==
        !          1138:                    &marker_next);
        !          1139:                am_next = LIST_NEXT(&marker_next, am_list);
        !          1140:                LIST_REMOVE(&marker_prev, am_list);
        !          1141:                LIST_REMOVE(&marker_next, am_list);
        !          1142:        }
        !          1143:
        !          1144:        return rv;
        !          1145: }

CVSweb