![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_aobj.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm|
Return to uvm_aobj.c CVS log | Up to [local] / sys / uvm |
1.1 nbrk 1: /* $OpenBSD: uvm_aobj.c,v 1.34 2007/04/13 18:57:49 art Exp $ */
2: /* $NetBSD: uvm_aobj.c,v 1.39 2001/02/18 21:19:08 chs Exp $ */
3:
4: /*
5: * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
6: * Washington University.
7: * All rights reserved.
8: *
9: * Redistribution and use in source and binary forms, with or without
10: * modification, are permitted provided that the following conditions
11: * are met:
12: * 1. Redistributions of source code must retain the above copyright
13: * notice, this list of conditions and the following disclaimer.
14: * 2. Redistributions in binary form must reproduce the above copyright
15: * notice, this list of conditions and the following disclaimer in the
16: * documentation and/or other materials provided with the distribution.
17: * 3. All advertising materials mentioning features or use of this software
18: * must display the following acknowledgement:
19: * This product includes software developed by Charles D. Cranor and
20: * Washington University.
21: * 4. The name of the author may not be used to endorse or promote products
22: * derived from this software without specific prior written permission.
23: *
24: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
25: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
26: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
27: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
28: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
29: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
33: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34: *
35: * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp
36: */
37: /*
38: * uvm_aobj.c: anonymous memory uvm_object pager
39: *
40: * author: Chuck Silvers <chuq@chuq.com>
41: * started: Jan-1998
42: *
43: * - design mostly from Chuck Cranor
44: */
45:
46: #include <sys/param.h>
47: #include <sys/systm.h>
48: #include <sys/proc.h>
49: #include <sys/malloc.h>
50: #include <sys/kernel.h>
51: #include <sys/pool.h>
52: #include <sys/kernel.h>
53:
54: #include <uvm/uvm.h>
55:
56: /*
57: * an aobj manages anonymous-memory backed uvm_objects. in addition
58: * to keeping the list of resident pages, it also keeps a list of
59: * allocated swap blocks. depending on the size of the aobj this list
60: * of allocated swap blocks is either stored in an array (small objects)
61: * or in a hash table (large objects).
62: */
63:
64: /*
65: * local structures
66: */
67:
68: /*
69: * for hash tables, we break the address space of the aobj into blocks
70: * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to
71: * be a power of two.
72: */
73:
74: #define UAO_SWHASH_CLUSTER_SHIFT 4
75: #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT)
76:
77: /* get the "tag" for this page index */
78: #define UAO_SWHASH_ELT_TAG(PAGEIDX) \
79: ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT)
80:
81: /* given an ELT and a page index, find the swap slot */
82: #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \
83: ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)])
84:
85: /* given an ELT, return its pageidx base */
86: #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \
87: ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT)
88:
89: /*
90: * the swhash hash function
91: */
92: #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \
93: (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \
94: & (AOBJ)->u_swhashmask)])
95:
96: /*
97: * the swhash threshold determines if we will use an array or a
98: * hash table to store the list of allocated swap blocks.
99: */
100:
101: #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4)
102: #define UAO_USES_SWHASH(AOBJ) \
103: ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */
104:
105: /*
106: * the number of buckets in a swhash, with an upper bound
107: */
108: #define UAO_SWHASH_MAXBUCKETS 256
109: #define UAO_SWHASH_BUCKETS(AOBJ) \
110: (min((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \
111: UAO_SWHASH_MAXBUCKETS))
112:
113:
114: /*
115: * uao_swhash_elt: when a hash table is being used, this structure defines
116: * the format of an entry in the bucket list.
117: */
118:
119: struct uao_swhash_elt {
120: LIST_ENTRY(uao_swhash_elt) list; /* the hash list */
121: voff_t tag; /* our 'tag' */
122: int count; /* our number of active slots */
123: int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */
124: };
125:
126: /*
127: * uao_swhash: the swap hash table structure
128: */
129:
130: LIST_HEAD(uao_swhash, uao_swhash_elt);
131:
132: /*
133: * uao_swhash_elt_pool: pool of uao_swhash_elt structures
134: */
135:
136: struct pool uao_swhash_elt_pool;
137:
138: /*
139: * uvm_aobj: the actual anon-backed uvm_object
140: *
141: * => the uvm_object is at the top of the structure, this allows
142: * (struct uvm_device *) == (struct uvm_object *)
143: * => only one of u_swslots and u_swhash is used in any given aobj
144: */
145:
146: struct uvm_aobj {
147: struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */
148: int u_pages; /* number of pages in entire object */
149: int u_flags; /* the flags (see uvm_aobj.h) */
150: int *u_swslots; /* array of offset->swapslot mappings */
151: /*
152: * hashtable of offset->swapslot mappings
153: * (u_swhash is an array of bucket heads)
154: */
155: struct uao_swhash *u_swhash;
156: u_long u_swhashmask; /* mask for hashtable */
157: LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */
158: };
159:
160: /*
161: * uvm_aobj_pool: pool of uvm_aobj structures
162: */
163:
164: struct pool uvm_aobj_pool;
165:
166: /*
167: * local functions
168: */
169:
170: static struct uao_swhash_elt *uao_find_swhash_elt(struct uvm_aobj *,
171: int, boolean_t);
172: static int uao_find_swslot(struct uvm_aobj *, int);
173: static boolean_t uao_flush(struct uvm_object *,
174: voff_t, voff_t, int);
175: static void uao_free(struct uvm_aobj *);
176: static int uao_get(struct uvm_object *, voff_t,
177: vm_page_t *, int *, int,
178: vm_prot_t, int, int);
179: static boolean_t uao_releasepg(struct vm_page *,
180: struct vm_page **);
181: static boolean_t uao_pagein(struct uvm_aobj *, int, int);
182: static boolean_t uao_pagein_page(struct uvm_aobj *, int);
183:
184: /*
185: * aobj_pager
186: *
187: * note that some functions (e.g. put) are handled elsewhere
188: */
189:
190: struct uvm_pagerops aobj_pager = {
191: NULL, /* init */
192: uao_reference, /* reference */
193: uao_detach, /* detach */
194: NULL, /* fault */
195: uao_flush, /* flush */
196: uao_get, /* get */
197: NULL, /* put (done by pagedaemon) */
198: NULL, /* cluster */
199: NULL, /* mk_pcluster */
200: uao_releasepg /* releasepg */
201: };
202:
203: /*
204: * uao_list: global list of active aobjs, locked by uao_list_lock
205: */
206:
207: static LIST_HEAD(aobjlist, uvm_aobj) uao_list;
208: static simple_lock_data_t uao_list_lock;
209:
210:
211: /*
212: * functions
213: */
214:
215: /*
216: * hash table/array related functions
217: */
218:
219: /*
220: * uao_find_swhash_elt: find (or create) a hash table entry for a page
221: * offset.
222: *
223: * => the object should be locked by the caller
224: */
225:
226: static struct uao_swhash_elt *
227: uao_find_swhash_elt(aobj, pageidx, create)
228: struct uvm_aobj *aobj;
229: int pageidx;
230: boolean_t create;
231: {
232: struct uao_swhash *swhash;
233: struct uao_swhash_elt *elt;
234: voff_t page_tag;
235:
236: swhash = UAO_SWHASH_HASH(aobj, pageidx); /* first hash to get bucket */
237: page_tag = UAO_SWHASH_ELT_TAG(pageidx); /* tag to search for */
238:
239: /*
240: * now search the bucket for the requested tag
241: */
242: LIST_FOREACH(elt, swhash, list) {
243: if (elt->tag == page_tag)
244: return(elt);
245: }
246:
247: /* fail now if we are not allowed to create a new entry in the bucket */
248: if (!create)
249: return NULL;
250:
251:
252: /*
253: * allocate a new entry for the bucket and init/insert it in
254: */
255: elt = pool_get(&uao_swhash_elt_pool, PR_WAITOK);
256: LIST_INSERT_HEAD(swhash, elt, list);
257: elt->tag = page_tag;
258: elt->count = 0;
259: memset(elt->slots, 0, sizeof(elt->slots));
260:
261: return(elt);
262: }
263:
264: /*
265: * uao_find_swslot: find the swap slot number for an aobj/pageidx
266: *
267: * => object must be locked by caller
268: */
269: __inline static int
270: uao_find_swslot(aobj, pageidx)
271: struct uvm_aobj *aobj;
272: int pageidx;
273: {
274:
275: /*
276: * if noswap flag is set, then we never return a slot
277: */
278:
279: if (aobj->u_flags & UAO_FLAG_NOSWAP)
280: return(0);
281:
282: /*
283: * if hashing, look in hash table.
284: */
285:
286: if (UAO_USES_SWHASH(aobj)) {
287: struct uao_swhash_elt *elt =
288: uao_find_swhash_elt(aobj, pageidx, FALSE);
289:
290: if (elt)
291: return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx));
292: else
293: return(0);
294: }
295:
296: /*
297: * otherwise, look in the array
298: */
299: return(aobj->u_swslots[pageidx]);
300: }
301:
302: /*
303: * uao_set_swslot: set the swap slot for a page in an aobj.
304: *
305: * => setting a slot to zero frees the slot
306: * => object must be locked by caller
307: */
308: int
309: uao_set_swslot(uobj, pageidx, slot)
310: struct uvm_object *uobj;
311: int pageidx, slot;
312: {
313: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
314: int oldslot;
315: UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist);
316: UVMHIST_LOG(pdhist, "aobj %p pageidx %ld slot %ld",
317: aobj, pageidx, slot, 0);
318:
319: /*
320: * if noswap flag is set, then we can't set a slot
321: */
322:
323: if (aobj->u_flags & UAO_FLAG_NOSWAP) {
324:
325: if (slot == 0)
326: return(0); /* a clear is ok */
327:
328: /* but a set is not */
329: printf("uao_set_swslot: uobj = %p\n", uobj);
330: panic("uao_set_swslot: attempt to set a slot on a NOSWAP object");
331: }
332:
333: /*
334: * are we using a hash table? if so, add it in the hash.
335: */
336:
337: if (UAO_USES_SWHASH(aobj)) {
338:
339: /*
340: * Avoid allocating an entry just to free it again if
341: * the page had not swap slot in the first place, and
342: * we are freeing.
343: */
344:
345: struct uao_swhash_elt *elt =
346: uao_find_swhash_elt(aobj, pageidx, slot ? TRUE : FALSE);
347: if (elt == NULL) {
348: KASSERT(slot == 0);
349: return (0);
350: }
351:
352: oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx);
353: UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot;
354:
355: /*
356: * now adjust the elt's reference counter and free it if we've
357: * dropped it to zero.
358: */
359:
360: /* an allocation? */
361: if (slot) {
362: if (oldslot == 0)
363: elt->count++;
364: } else { /* freeing slot ... */
365: if (oldslot) /* to be safe */
366: elt->count--;
367:
368: if (elt->count == 0) {
369: LIST_REMOVE(elt, list);
370: pool_put(&uao_swhash_elt_pool, elt);
371: }
372: }
373: } else {
374: /* we are using an array */
375: oldslot = aobj->u_swslots[pageidx];
376: aobj->u_swslots[pageidx] = slot;
377: }
378: return (oldslot);
379: }
380:
381: /*
382: * end of hash/array functions
383: */
384:
385: /*
386: * uao_free: free all resources held by an aobj, and then free the aobj
387: *
388: * => the aobj should be dead
389: */
390: static void
391: uao_free(aobj)
392: struct uvm_aobj *aobj;
393: {
394:
395: simple_unlock(&aobj->u_obj.vmobjlock);
396:
397: if (UAO_USES_SWHASH(aobj)) {
398: int i, hashbuckets = aobj->u_swhashmask + 1;
399:
400: /*
401: * free the swslots from each hash bucket,
402: * then the hash bucket, and finally the hash table itself.
403: */
404: for (i = 0; i < hashbuckets; i++) {
405: struct uao_swhash_elt *elt, *next;
406:
407: for (elt = LIST_FIRST(&aobj->u_swhash[i]);
408: elt != NULL;
409: elt = next) {
410: int j;
411:
412: for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) {
413: int slot = elt->slots[j];
414:
415: if (slot == 0) {
416: continue;
417: }
418: uvm_swap_free(slot, 1);
419:
420: /*
421: * this page is no longer
422: * only in swap.
423: */
424: simple_lock(&uvm.swap_data_lock);
425: uvmexp.swpgonly--;
426: simple_unlock(&uvm.swap_data_lock);
427: }
428:
429: next = LIST_NEXT(elt, list);
430: pool_put(&uao_swhash_elt_pool, elt);
431: }
432: }
433: free(aobj->u_swhash, M_UVMAOBJ);
434: } else {
435: int i;
436:
437: /*
438: * free the array
439: */
440:
441: for (i = 0; i < aobj->u_pages; i++) {
442: int slot = aobj->u_swslots[i];
443:
444: if (slot) {
445: uvm_swap_free(slot, 1);
446:
447: /* this page is no longer only in swap. */
448: simple_lock(&uvm.swap_data_lock);
449: uvmexp.swpgonly--;
450: simple_unlock(&uvm.swap_data_lock);
451: }
452: }
453: free(aobj->u_swslots, M_UVMAOBJ);
454: }
455:
456: /*
457: * finally free the aobj itself
458: */
459: pool_put(&uvm_aobj_pool, aobj);
460: }
461:
462: /*
463: * pager functions
464: */
465:
466: /*
467: * uao_create: create an aobj of the given size and return its uvm_object.
468: *
469: * => for normal use, flags are always zero
470: * => for the kernel object, the flags are:
471: * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once)
472: * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ")
473: */
474: struct uvm_object *
475: uao_create(size, flags)
476: vsize_t size;
477: int flags;
478: {
479: static struct uvm_aobj kernel_object_store; /* home of kernel_object */
480: static int kobj_alloced = 0; /* not allocated yet */
481: int pages = round_page(size) >> PAGE_SHIFT;
482: struct uvm_aobj *aobj;
483:
484: /*
485: * malloc a new aobj unless we are asked for the kernel object
486: */
487: if (flags & UAO_FLAG_KERNOBJ) { /* want kernel object? */
488: if (kobj_alloced)
489: panic("uao_create: kernel object already allocated");
490:
491: aobj = &kernel_object_store;
492: aobj->u_pages = pages;
493: aobj->u_flags = UAO_FLAG_NOSWAP; /* no swap to start */
494: /* we are special, we never die */
495: aobj->u_obj.uo_refs = UVM_OBJ_KERN;
496: kobj_alloced = UAO_FLAG_KERNOBJ;
497: } else if (flags & UAO_FLAG_KERNSWAP) {
498: aobj = &kernel_object_store;
499: if (kobj_alloced != UAO_FLAG_KERNOBJ)
500: panic("uao_create: asked to enable swap on kernel object");
501: kobj_alloced = UAO_FLAG_KERNSWAP;
502: } else { /* normal object */
503: aobj = pool_get(&uvm_aobj_pool, PR_WAITOK);
504: aobj->u_pages = pages;
505: aobj->u_flags = 0; /* normal object */
506: aobj->u_obj.uo_refs = 1; /* start with 1 reference */
507: }
508:
509: /*
510: * allocate hash/array if necessary
511: *
512: * note: in the KERNSWAP case no need to worry about locking since
513: * we are still booting we should be the only thread around.
514: */
515: if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) {
516: int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ?
517: M_NOWAIT : M_WAITOK;
518:
519: /* allocate hash table or array depending on object size */
520: if (UAO_USES_SWHASH(aobj)) {
521: aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj),
522: M_UVMAOBJ, mflags, &aobj->u_swhashmask);
523: if (aobj->u_swhash == NULL)
524: panic("uao_create: hashinit swhash failed");
525: } else {
526: aobj->u_swslots = malloc(pages * sizeof(int),
527: M_UVMAOBJ, mflags);
528: if (aobj->u_swslots == NULL)
529: panic("uao_create: malloc swslots failed");
530: memset(aobj->u_swslots, 0, pages * sizeof(int));
531: }
532:
533: if (flags) {
534: aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */
535: return(&aobj->u_obj);
536: /* done! */
537: }
538: }
539:
540: /*
541: * init aobj fields
542: */
543: simple_lock_init(&aobj->u_obj.vmobjlock);
544: aobj->u_obj.pgops = &aobj_pager;
545: TAILQ_INIT(&aobj->u_obj.memq);
546: aobj->u_obj.uo_npages = 0;
547:
548: /*
549: * now that aobj is ready, add it to the global list
550: */
551: simple_lock(&uao_list_lock);
552: LIST_INSERT_HEAD(&uao_list, aobj, u_list);
553: simple_unlock(&uao_list_lock);
554:
555: /*
556: * done!
557: */
558: return(&aobj->u_obj);
559: }
560:
561:
562:
563: /*
564: * uao_init: set up aobj pager subsystem
565: *
566: * => called at boot time from uvm_pager_init()
567: */
568: void
569: uao_init()
570: {
571: static int uao_initialized;
572:
573: if (uao_initialized)
574: return;
575: uao_initialized = TRUE;
576:
577: LIST_INIT(&uao_list);
578: simple_lock_init(&uao_list_lock);
579:
580: /*
581: * NOTE: Pages fror this pool must not come from a pageable
582: * kernel map!
583: */
584: pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
585: 0, 0, 0, "uaoeltpl", &pool_allocator_nointr);
586:
587: pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0,
588: "aobjpl", &pool_allocator_nointr);
589: }
590:
591: /*
592: * uao_reference: add a ref to an aobj
593: *
594: * => aobj must be unlocked
595: * => just lock it and call the locked version
596: */
597: void
598: uao_reference(uobj)
599: struct uvm_object *uobj;
600: {
601: simple_lock(&uobj->vmobjlock);
602: uao_reference_locked(uobj);
603: simple_unlock(&uobj->vmobjlock);
604: }
605:
606: /*
607: * uao_reference_locked: add a ref to an aobj that is already locked
608: *
609: * => aobj must be locked
610: * this needs to be separate from the normal routine
611: * since sometimes we need to add a reference to an aobj when
612: * it's already locked.
613: */
614: void
615: uao_reference_locked(uobj)
616: struct uvm_object *uobj;
617: {
618: UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist);
619:
620: /*
621: * kernel_object already has plenty of references, leave it alone.
622: */
623:
624: if (UVM_OBJ_IS_KERN_OBJECT(uobj))
625: return;
626:
627: uobj->uo_refs++; /* bump! */
628: UVMHIST_LOG(maphist, "<- done (uobj=%p, ref = %ld)",
629: uobj, uobj->uo_refs,0,0);
630: }
631:
632:
633: /*
634: * uao_detach: drop a reference to an aobj
635: *
636: * => aobj must be unlocked
637: * => just lock it and call the locked version
638: */
639: void
640: uao_detach(uobj)
641: struct uvm_object *uobj;
642: {
643: simple_lock(&uobj->vmobjlock);
644: uao_detach_locked(uobj);
645: }
646:
647:
648: /*
649: * uao_detach_locked: drop a reference to an aobj
650: *
651: * => aobj must be locked, and is unlocked (or freed) upon return.
652: * this needs to be separate from the normal routine
653: * since sometimes we need to detach from an aobj when
654: * it's already locked.
655: */
656: void
657: uao_detach_locked(uobj)
658: struct uvm_object *uobj;
659: {
660: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
661: struct vm_page *pg, *next;
662: boolean_t busybody;
663: UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
664:
665: /*
666: * detaching from kernel_object is a noop.
667: */
668: if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
669: simple_unlock(&uobj->vmobjlock);
670: return;
671: }
672:
673: UVMHIST_LOG(maphist," (uobj=%p) ref=%ld", uobj,uobj->uo_refs,0,0);
674: uobj->uo_refs--; /* drop ref! */
675: if (uobj->uo_refs) { /* still more refs? */
676: simple_unlock(&uobj->vmobjlock);
677: UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0);
678: return;
679: }
680:
681: /*
682: * remove the aobj from the global list.
683: */
684: simple_lock(&uao_list_lock);
685: LIST_REMOVE(aobj, u_list);
686: simple_unlock(&uao_list_lock);
687:
688: /*
689: * free all the pages that aren't PG_BUSY,
690: * mark for release any that are.
691: */
692: busybody = FALSE;
693: for (pg = TAILQ_FIRST(&uobj->memq); pg != NULL; pg = next) {
694: next = TAILQ_NEXT(pg, listq);
695: if (pg->pg_flags & PG_BUSY) {
696: atomic_setbits_int(&pg->pg_flags, PG_RELEASED);
697: busybody = TRUE;
698: continue;
699: }
700:
701: /* zap the mappings, free the swap slot, free the page */
702: pmap_page_protect(pg, VM_PROT_NONE);
703: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
704: uvm_lock_pageq();
705: uvm_pagefree(pg);
706: uvm_unlock_pageq();
707: }
708:
709: /*
710: * if we found any busy pages, we're done for now.
711: * mark the aobj for death, releasepg will finish up for us.
712: */
713: if (busybody) {
714: aobj->u_flags |= UAO_FLAG_KILLME;
715: simple_unlock(&aobj->u_obj.vmobjlock);
716: return;
717: }
718:
719: /*
720: * finally, free the rest.
721: */
722: uao_free(aobj);
723: }
724:
725: /*
726: * uao_flush: "flush" pages out of a uvm object
727: *
728: * => object should be locked by caller. we may _unlock_ the object
729: * if (and only if) we need to clean a page (PGO_CLEANIT).
730: * XXXJRT Currently, however, we don't. In the case of cleaning
731: * XXXJRT a page, we simply just deactivate it. Should probably
732: * XXXJRT handle this better, in the future (although "flushing"
733: * XXXJRT anonymous memory isn't terribly important).
734: * => if PGO_CLEANIT is not set, then we will neither unlock the object
735: * or block.
736: * => if PGO_ALLPAGE is set, then all pages in the object are valid targets
737: * for flushing.
738: * => NOTE: we rely on the fact that the object's memq is a TAILQ and
739: * that new pages are inserted on the tail end of the list. thus,
740: * we can make a complete pass through the object in one go by starting
741: * at the head and working towards the tail (new pages are put in
742: * front of us).
743: * => NOTE: we are allowed to lock the page queues, so the caller
744: * must not be holding the lock on them [e.g. pagedaemon had
745: * better not call us with the queues locked]
746: * => we return TRUE unless we encountered some sort of I/O error
747: * XXXJRT currently never happens, as we never directly initiate
748: * XXXJRT I/O
749: *
750: * comment on "cleaning" object and PG_BUSY pages:
751: * this routine is holding the lock on the object. the only time
752: * that is can run into a PG_BUSY page that it does not own is if
753: * some other process has started I/O on the page (e.g. either
754: * a pagein or a pageout). if the PG_BUSY page is being paged
755: * in, then it can not be dirty (!PG_CLEAN) because no one has
756: * had a change to modify it yet. if the PG_BUSY page is being
757: * paged out then it means that someone else has already started
758: * cleaning the page for us (how nice!). in this case, if we
759: * have syncio specified, then after we make our pass through the
760: * object we need to wait for the other PG_BUSY pages to clear
761: * off (i.e. we need to do an iosync). also note that once a
762: * page is PG_BUSY is must stary in its object until it is un-busyed.
763: * XXXJRT We never actually do this, as we are "flushing" anonymous
764: * XXXJRT memory, which doesn't have persistent backing store.
765: *
766: * note on page traversal:
767: * we can traverse the pages in an object either by going down the
768: * linked list in "uobj->memq", or we can go over the address range
769: * by page doing hash table lookups for each address. depending
770: * on how many pages are in the object it may be cheaper to do one
771: * or the other. we set "by_list" to true if we are using memq.
772: * if the cost of a hash lookup was equal to the cost of the list
773: * traversal we could compare the number of pages in the start->stop
774: * range to the total number of pages in the object. however, it
775: * seems that a hash table lookup is more expensive than the linked
776: * list traversal, so we multiply the number of pages in the
777: * start->stop range by a penalty which we define below.
778: */
779:
780: #define UAO_HASH_PENALTY 4 /* XXX: a guess */
781:
782: boolean_t
783: uao_flush(uobj, start, stop, flags)
784: struct uvm_object *uobj;
785: voff_t start, stop;
786: int flags;
787: {
788: struct uvm_aobj *aobj = (struct uvm_aobj *) uobj;
789: struct vm_page *pp, *ppnext;
790: boolean_t retval, by_list;
791: voff_t curoff;
792: UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist);
793:
794: curoff = 0; /* XXX: shut up gcc */
795:
796: retval = TRUE; /* default to success */
797:
798: if (flags & PGO_ALLPAGES) {
799: start = 0;
800: stop = aobj->u_pages << PAGE_SHIFT;
801: by_list = TRUE; /* always go by the list */
802: } else {
803: start = trunc_page(start);
804: stop = round_page(stop);
805: if (stop > (aobj->u_pages << PAGE_SHIFT)) {
806: printf("uao_flush: strange, got an out of range "
807: "flush (fixed)\n");
808: stop = aobj->u_pages << PAGE_SHIFT;
809: }
810: by_list = (uobj->uo_npages <=
811: ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY);
812: }
813:
814: UVMHIST_LOG(maphist,
815: " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
816: (u_long)start, (u_long)stop, by_list, flags);
817:
818: /*
819: * Don't need to do any work here if we're not freeing
820: * or deactivating pages.
821: */
822: if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
823: UVMHIST_LOG(maphist,
824: "<- done (no work to do)",0,0,0,0);
825: return (retval);
826: }
827:
828: /*
829: * now do it. note: we must update ppnext in the body of loop or we
830: * will get stuck. we need to use ppnext because we may free "pp"
831: * before doing the next loop.
832: */
833:
834: if (by_list) {
835: pp = TAILQ_FIRST(&uobj->memq);
836: } else {
837: curoff = start;
838: pp = uvm_pagelookup(uobj, curoff);
839: }
840:
841: ppnext = NULL; /* XXX: shut up gcc */
842: uvm_lock_pageq(); /* page queues locked */
843:
844: /* locked: both page queues and uobj */
845: for ( ; (by_list && pp != NULL) ||
846: (!by_list && curoff < stop) ; pp = ppnext) {
847: if (by_list) {
848: ppnext = TAILQ_NEXT(pp, listq);
849:
850: /* range check */
851: if (pp->offset < start || pp->offset >= stop)
852: continue;
853: } else {
854: curoff += PAGE_SIZE;
855: if (curoff < stop)
856: ppnext = uvm_pagelookup(uobj, curoff);
857:
858: /* null check */
859: if (pp == NULL)
860: continue;
861: }
862:
863: switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
864: /*
865: * XXX In these first 3 cases, we always just
866: * XXX deactivate the page. We may want to
867: * XXX handle the different cases more specifically
868: * XXX in the future.
869: */
870: case PGO_CLEANIT|PGO_FREE:
871: case PGO_CLEANIT|PGO_DEACTIVATE:
872: case PGO_DEACTIVATE:
873: deactivate_it:
874: /* skip the page if it's loaned or wired */
875: if (pp->loan_count != 0 ||
876: pp->wire_count != 0)
877: continue;
878:
879: #ifdef UBC
880: /* ...and deactivate the page. */
881: pmap_clear_reference(pp);
882: #else
883: /* zap all mappings for the page. */
884: pmap_page_protect(pp, VM_PROT_NONE);
885:
886: /* ...and deactivate the page. */
887: #endif
888: uvm_pagedeactivate(pp);
889:
890: continue;
891:
892: case PGO_FREE:
893: /*
894: * If there are multiple references to
895: * the object, just deactivate the page.
896: */
897: if (uobj->uo_refs > 1)
898: goto deactivate_it;
899:
900: /* XXX skip the page if it's loaned or wired */
901: if (pp->loan_count != 0 ||
902: pp->wire_count != 0)
903: continue;
904:
905: /*
906: * mark the page as released if its busy.
907: */
908: if (pp->pg_flags & PG_BUSY) {
909: atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
910: continue;
911: }
912:
913: /* zap all mappings for the page. */
914: pmap_page_protect(pp, VM_PROT_NONE);
915:
916: uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
917: uvm_pagefree(pp);
918:
919: continue;
920:
921: default:
922: panic("uao_flush: weird flags");
923: }
924: }
925:
926: uvm_unlock_pageq();
927:
928: UVMHIST_LOG(maphist,
929: "<- done, rv=%ld",retval,0,0,0);
930: return (retval);
931: }
932:
933: /*
934: * uao_get: fetch me a page
935: *
936: * we have three cases:
937: * 1: page is resident -> just return the page.
938: * 2: page is zero-fill -> allocate a new page and zero it.
939: * 3: page is swapped out -> fetch the page from swap.
940: *
941: * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot.
942: * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES),
943: * then we will need to return VM_PAGER_UNLOCK.
944: *
945: * => prefer map unlocked (not required)
946: * => object must be locked! we will _unlock_ it before starting any I/O.
947: * => flags: PGO_ALLPAGES: get all of the pages
948: * PGO_LOCKED: fault data structures are locked
949: * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx]
950: * => NOTE: caller must check for released pages!!
951: */
952: static int
953: uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags)
954: struct uvm_object *uobj;
955: voff_t offset;
956: struct vm_page **pps;
957: int *npagesp;
958: int centeridx, advice, flags;
959: vm_prot_t access_type;
960: {
961: struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
962: voff_t current_offset;
963: vm_page_t ptmp;
964: int lcv, gotpages, maxpages, swslot, rv, pageidx;
965: boolean_t done;
966: UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist);
967:
968: UVMHIST_LOG(pdhist, "aobj=%p offset=%ld, flags=%ld",
969: aobj, (u_long)offset, flags,0);
970:
971: /*
972: * get number of pages
973: */
974: maxpages = *npagesp;
975:
976: /*
977: * step 1: handled the case where fault data structures are locked.
978: */
979:
980: if (flags & PGO_LOCKED) {
981: /*
982: * step 1a: get pages that are already resident. only do
983: * this if the data structures are locked (i.e. the first
984: * time through).
985: */
986:
987: done = TRUE; /* be optimistic */
988: gotpages = 0; /* # of pages we got so far */
989:
990: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
991: lcv++, current_offset += PAGE_SIZE) {
992: /* do we care about this page? if not, skip it */
993: if (pps[lcv] == PGO_DONTCARE)
994: continue;
995:
996: ptmp = uvm_pagelookup(uobj, current_offset);
997:
998: /*
999: * if page is new, attempt to allocate the page,
1000: * zero-fill'd.
1001: */
1002: if (ptmp == NULL && uao_find_swslot(aobj,
1003: current_offset >> PAGE_SHIFT) == 0) {
1004: ptmp = uvm_pagealloc(uobj, current_offset,
1005: NULL, UVM_PGA_ZERO);
1006: if (ptmp) {
1007: /* new page */
1008: atomic_clearbits_int(&ptmp->pg_flags,
1009: PG_BUSY|PG_FAKE);
1010: atomic_setbits_int(&ptmp->pg_flags,
1011: PQ_AOBJ);
1012: UVM_PAGE_OWN(ptmp, NULL);
1013: }
1014: }
1015:
1016: /*
1017: * to be useful must get a non-busy, non-released page
1018: */
1019: if (ptmp == NULL ||
1020: (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
1021: if (lcv == centeridx ||
1022: (flags & PGO_ALLPAGES) != 0)
1023: /* need to do a wait or I/O! */
1024: done = FALSE;
1025: continue;
1026: }
1027:
1028: /*
1029: * useful page: busy/lock it and plug it in our
1030: * result array
1031: */
1032: /* caller must un-busy this page */
1033: atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
1034: UVM_PAGE_OWN(ptmp, "uao_get1");
1035: pps[lcv] = ptmp;
1036: gotpages++;
1037:
1038: } /* "for" lcv loop */
1039:
1040: /*
1041: * step 1b: now we've either done everything needed or we
1042: * to unlock and do some waiting or I/O.
1043: */
1044:
1045: UVMHIST_LOG(pdhist, "<- done (done=%ld)", done, 0,0,0);
1046:
1047: *npagesp = gotpages;
1048: if (done)
1049: /* bingo! */
1050: return(VM_PAGER_OK);
1051: else
1052: /* EEK! Need to unlock and I/O */
1053: return(VM_PAGER_UNLOCK);
1054: }
1055:
1056: /*
1057: * step 2: get non-resident or busy pages.
1058: * object is locked. data structures are unlocked.
1059: */
1060:
1061: for (lcv = 0, current_offset = offset ; lcv < maxpages ;
1062: lcv++, current_offset += PAGE_SIZE) {
1063:
1064: /*
1065: * - skip over pages we've already gotten or don't want
1066: * - skip over pages we don't _have_ to get
1067: */
1068:
1069: if (pps[lcv] != NULL ||
1070: (lcv != centeridx && (flags & PGO_ALLPAGES) == 0))
1071: continue;
1072:
1073: pageidx = current_offset >> PAGE_SHIFT;
1074:
1075: /*
1076: * we have yet to locate the current page (pps[lcv]). we
1077: * first look for a page that is already at the current offset.
1078: * if we find a page, we check to see if it is busy or
1079: * released. if that is the case, then we sleep on the page
1080: * until it is no longer busy or released and repeat the lookup.
1081: * if the page we found is neither busy nor released, then we
1082: * busy it (so we own it) and plug it into pps[lcv]. this
1083: * 'break's the following while loop and indicates we are
1084: * ready to move on to the next page in the "lcv" loop above.
1085: *
1086: * if we exit the while loop with pps[lcv] still set to NULL,
1087: * then it means that we allocated a new busy/fake/clean page
1088: * ptmp in the object and we need to do I/O to fill in the data.
1089: */
1090:
1091: /* top of "pps" while loop */
1092: while (pps[lcv] == NULL) {
1093: /* look for a resident page */
1094: ptmp = uvm_pagelookup(uobj, current_offset);
1095:
1096: /* not resident? allocate one now (if we can) */
1097: if (ptmp == NULL) {
1098:
1099: ptmp = uvm_pagealloc(uobj, current_offset,
1100: NULL, 0);
1101:
1102: /* out of RAM? */
1103: if (ptmp == NULL) {
1104: simple_unlock(&uobj->vmobjlock);
1105: UVMHIST_LOG(pdhist,
1106: "sleeping, ptmp == NULL\n",0,0,0,0);
1107: uvm_wait("uao_getpage");
1108: simple_lock(&uobj->vmobjlock);
1109: /* goto top of pps while loop */
1110: continue;
1111: }
1112:
1113: /*
1114: * safe with PQ's unlocked: because we just
1115: * alloc'd the page
1116: */
1117: atomic_setbits_int(&ptmp->pg_flags, PQ_AOBJ);
1118:
1119: /*
1120: * got new page ready for I/O. break pps while
1121: * loop. pps[lcv] is still NULL.
1122: */
1123: break;
1124: }
1125:
1126: /* page is there, see if we need to wait on it */
1127: if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
1128: atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
1129: UVMHIST_LOG(pdhist,
1130: "sleeping, ptmp->flags 0x%lx\n",
1131: ptmp->pg_flags,0,0,0);
1132: UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock,
1133: FALSE, "uao_get", 0);
1134: simple_lock(&uobj->vmobjlock);
1135: continue; /* goto top of pps while loop */
1136: }
1137:
1138: /*
1139: * if we get here then the page has become resident and
1140: * unbusy between steps 1 and 2. we busy it now (so we
1141: * own it) and set pps[lcv] (so that we exit the while
1142: * loop).
1143: */
1144: /* we own it, caller must un-busy */
1145: atomic_setbits_int(&ptmp->pg_flags, PG_BUSY);
1146: UVM_PAGE_OWN(ptmp, "uao_get2");
1147: pps[lcv] = ptmp;
1148: }
1149:
1150: /*
1151: * if we own the valid page at the correct offset, pps[lcv] will
1152: * point to it. nothing more to do except go to the next page.
1153: */
1154: if (pps[lcv])
1155: continue; /* next lcv */
1156:
1157: /*
1158: * we have a "fake/busy/clean" page that we just allocated.
1159: * do the needed "i/o", either reading from swap or zeroing.
1160: */
1161: swslot = uao_find_swslot(aobj, pageidx);
1162:
1163: /*
1164: * just zero the page if there's nothing in swap.
1165: */
1166: if (swslot == 0)
1167: {
1168: /*
1169: * page hasn't existed before, just zero it.
1170: */
1171: uvm_pagezero(ptmp);
1172: } else {
1173: UVMHIST_LOG(pdhist, "pagein from swslot %ld",
1174: swslot, 0,0,0);
1175:
1176: /*
1177: * page in the swapped-out page.
1178: * unlock object for i/o, relock when done.
1179: */
1180: simple_unlock(&uobj->vmobjlock);
1181: rv = uvm_swap_get(ptmp, swslot, PGO_SYNCIO);
1182: simple_lock(&uobj->vmobjlock);
1183:
1184: /*
1185: * I/O done. check for errors.
1186: */
1187: if (rv != VM_PAGER_OK)
1188: {
1189: UVMHIST_LOG(pdhist, "<- done (error=%ld)",
1190: rv,0,0,0);
1191: if (ptmp->pg_flags & PG_WANTED)
1192: wakeup(ptmp);
1193:
1194: /*
1195: * remove the swap slot from the aobj
1196: * and mark the aobj as having no real slot.
1197: * don't free the swap slot, thus preventing
1198: * it from being used again.
1199: */
1200: swslot = uao_set_swslot(&aobj->u_obj, pageidx,
1201: SWSLOT_BAD);
1202: uvm_swap_markbad(swslot, 1);
1203:
1204: atomic_clearbits_int(&ptmp->pg_flags,
1205: PG_WANTED|PG_BUSY);
1206: UVM_PAGE_OWN(ptmp, NULL);
1207: uvm_lock_pageq();
1208: uvm_pagefree(ptmp);
1209: uvm_unlock_pageq();
1210:
1211: simple_unlock(&uobj->vmobjlock);
1212: return (rv);
1213: }
1214: }
1215:
1216: /*
1217: * we got the page! clear the fake flag (indicates valid
1218: * data now in page) and plug into our result array. note
1219: * that page is still busy.
1220: *
1221: * it is the callers job to:
1222: * => check if the page is released
1223: * => unbusy the page
1224: * => activate the page
1225: */
1226:
1227: /* data is valid ... */
1228: atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE);
1229: pmap_clear_modify(ptmp); /* ... and clean */
1230: pps[lcv] = ptmp;
1231:
1232: } /* lcv loop */
1233:
1234: /*
1235: * finally, unlock object and return.
1236: */
1237:
1238: simple_unlock(&uobj->vmobjlock);
1239: UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0);
1240: return(VM_PAGER_OK);
1241: }
1242:
1243: /*
1244: * uao_releasepg: handle released page in an aobj
1245: *
1246: * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
1247: * to dispose of.
1248: * => caller must handle PG_WANTED case
1249: * => called with page's object locked, pageq's unlocked
1250: * => returns TRUE if page's object is still alive, FALSE if we
1251: * killed the page's object. if we return TRUE, then we
1252: * return with the object locked.
1253: * => if (nextpgp != NULL) => we return the next page on the queue, and return
1254: * with the page queues locked [for pagedaemon]
1255: * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
1256: * => we kill the aobj if it is not referenced and we are suppose to
1257: * kill it ("KILLME").
1258: */
1259: static boolean_t
1260: uao_releasepg(pg, nextpgp)
1261: struct vm_page *pg;
1262: struct vm_page **nextpgp; /* OUT */
1263: {
1264: struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject;
1265:
1266: KASSERT(pg->pg_flags & PG_RELEASED);
1267:
1268: /*
1269: * dispose of the page [caller handles PG_WANTED] and swap slot.
1270: */
1271: pmap_page_protect(pg, VM_PROT_NONE);
1272: uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
1273: uvm_lock_pageq();
1274: if (nextpgp)
1275: *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
1276: uvm_pagefree(pg);
1277: if (!nextpgp)
1278: uvm_unlock_pageq(); /* keep locked for daemon */
1279:
1280: /*
1281: * if we're not killing the object, we're done.
1282: */
1283: if ((aobj->u_flags & UAO_FLAG_KILLME) == 0)
1284: return TRUE;
1285: KASSERT(aobj->u_obj.uo_refs == 0);
1286:
1287: /*
1288: * if there are still pages in the object, we're done for now.
1289: */
1290: if (aobj->u_obj.uo_npages != 0)
1291: return TRUE;
1292:
1293: KASSERT(TAILQ_EMPTY(&aobj->u_obj.memq));
1294:
1295: /*
1296: * finally, free the rest.
1297: */
1298: uao_free(aobj);
1299:
1300: return FALSE;
1301: }
1302:
1303:
1304: /*
1305: * uao_dropswap: release any swap resources from this aobj page.
1306: *
1307: * => aobj must be locked or have a reference count of 0.
1308: */
1309:
1310: void
1311: uao_dropswap(uobj, pageidx)
1312: struct uvm_object *uobj;
1313: int pageidx;
1314: {
1315: int slot;
1316:
1317: slot = uao_set_swslot(uobj, pageidx, 0);
1318: if (slot) {
1319: uvm_swap_free(slot, 1);
1320: }
1321: }
1322:
1323:
1324: /*
1325: * page in every page in every aobj that is paged-out to a range of swslots.
1326: *
1327: * => nothing should be locked.
1328: * => returns TRUE if pagein was aborted due to lack of memory.
1329: */
1330: boolean_t
1331: uao_swap_off(startslot, endslot)
1332: int startslot, endslot;
1333: {
1334: struct uvm_aobj *aobj, *nextaobj;
1335:
1336: /*
1337: * walk the list of all aobjs.
1338: */
1339:
1340: restart:
1341: simple_lock(&uao_list_lock);
1342:
1343: for (aobj = LIST_FIRST(&uao_list);
1344: aobj != NULL;
1345: aobj = nextaobj) {
1346: boolean_t rv;
1347:
1348: /*
1349: * try to get the object lock,
1350: * start all over if we fail.
1351: * most of the time we'll get the aobj lock,
1352: * so this should be a rare case.
1353: */
1354: if (!simple_lock_try(&aobj->u_obj.vmobjlock)) {
1355: simple_unlock(&uao_list_lock);
1356: goto restart;
1357: }
1358:
1359: /*
1360: * add a ref to the aobj so it doesn't disappear
1361: * while we're working.
1362: */
1363: uao_reference_locked(&aobj->u_obj);
1364:
1365: /*
1366: * now it's safe to unlock the uao list.
1367: */
1368: simple_unlock(&uao_list_lock);
1369:
1370: /*
1371: * page in any pages in the swslot range.
1372: * if there's an error, abort and return the error.
1373: */
1374: rv = uao_pagein(aobj, startslot, endslot);
1375: if (rv) {
1376: uao_detach_locked(&aobj->u_obj);
1377: return rv;
1378: }
1379:
1380: /*
1381: * we're done with this aobj.
1382: * relock the list and drop our ref on the aobj.
1383: */
1384: simple_lock(&uao_list_lock);
1385: nextaobj = LIST_NEXT(aobj, u_list);
1386: uao_detach_locked(&aobj->u_obj);
1387: }
1388:
1389: /*
1390: * done with traversal, unlock the list
1391: */
1392: simple_unlock(&uao_list_lock);
1393: return FALSE;
1394: }
1395:
1396:
1397: /*
1398: * page in any pages from aobj in the given range.
1399: *
1400: * => aobj must be locked and is returned locked.
1401: * => returns TRUE if pagein was aborted due to lack of memory.
1402: */
1403: static boolean_t
1404: uao_pagein(aobj, startslot, endslot)
1405: struct uvm_aobj *aobj;
1406: int startslot, endslot;
1407: {
1408: boolean_t rv;
1409:
1410: if (UAO_USES_SWHASH(aobj)) {
1411: struct uao_swhash_elt *elt;
1412: int bucket;
1413:
1414: restart:
1415: for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) {
1416: for (elt = LIST_FIRST(&aobj->u_swhash[bucket]);
1417: elt != NULL;
1418: elt = LIST_NEXT(elt, list)) {
1419: int i;
1420:
1421: for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) {
1422: int slot = elt->slots[i];
1423:
1424: /*
1425: * if the slot isn't in range, skip it.
1426: */
1427: if (slot < startslot ||
1428: slot >= endslot) {
1429: continue;
1430: }
1431:
1432: /*
1433: * process the page,
1434: * the start over on this object
1435: * since the swhash elt
1436: * may have been freed.
1437: */
1438: rv = uao_pagein_page(aobj,
1439: UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i);
1440: if (rv) {
1441: return rv;
1442: }
1443: goto restart;
1444: }
1445: }
1446: }
1447: } else {
1448: int i;
1449:
1450: for (i = 0; i < aobj->u_pages; i++) {
1451: int slot = aobj->u_swslots[i];
1452:
1453: /*
1454: * if the slot isn't in range, skip it
1455: */
1456: if (slot < startslot || slot >= endslot) {
1457: continue;
1458: }
1459:
1460: /*
1461: * process the page.
1462: */
1463: rv = uao_pagein_page(aobj, i);
1464: if (rv) {
1465: return rv;
1466: }
1467: }
1468: }
1469:
1470: return FALSE;
1471: }
1472:
1473: /*
1474: * page in a page from an aobj. used for swap_off.
1475: * returns TRUE if pagein was aborted due to lack of memory.
1476: *
1477: * => aobj must be locked and is returned locked.
1478: */
1479: static boolean_t
1480: uao_pagein_page(aobj, pageidx)
1481: struct uvm_aobj *aobj;
1482: int pageidx;
1483: {
1484: struct vm_page *pg;
1485: int rv, slot, npages;
1486:
1487: pg = NULL;
1488: npages = 1;
1489: /* locked: aobj */
1490: rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT,
1491: &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0);
1492: /* unlocked: aobj */
1493:
1494: /*
1495: * relock and finish up.
1496: */
1497: simple_lock(&aobj->u_obj.vmobjlock);
1498:
1499: switch (rv) {
1500: case VM_PAGER_OK:
1501: break;
1502:
1503: case VM_PAGER_ERROR:
1504: case VM_PAGER_REFAULT:
1505: /*
1506: * nothing more to do on errors.
1507: * VM_PAGER_REFAULT can only mean that the anon was freed,
1508: * so again there's nothing to do.
1509: */
1510: return FALSE;
1511:
1512: }
1513: KASSERT((pg->pg_flags & PG_RELEASED) == 0);
1514:
1515: /*
1516: * ok, we've got the page now.
1517: * mark it as dirty, clear its swslot and un-busy it.
1518: */
1519: slot = uao_set_swslot(&aobj->u_obj, pageidx, 0);
1520: uvm_swap_free(slot, 1);
1521: atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_CLEAN|PG_FAKE);
1522: UVM_PAGE_OWN(pg, NULL);
1523:
1524: /*
1525: * deactivate the page (to put it on a page queue).
1526: */
1527: pmap_clear_reference(pg);
1528: #ifndef UBC
1529: pmap_page_protect(pg, VM_PROT_NONE);
1530: #endif
1531: uvm_lock_pageq();
1532: uvm_pagedeactivate(pg);
1533: uvm_unlock_pageq();
1534:
1535: return FALSE;
1536: }