Annotation of sys/uvm/uvm_swap.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: uvm_swap.c,v 1.72 2007/06/18 21:51:15 pedro Exp $ */
! 2: /* $NetBSD: uvm_swap.c,v 1.40 2000/11/17 11:39:39 mrg Exp $ */
! 3:
! 4: /*
! 5: * Copyright (c) 1995, 1996, 1997 Matthew R. Green
! 6: * All rights reserved.
! 7: *
! 8: * Redistribution and use in source and binary forms, with or without
! 9: * modification, are permitted provided that the following conditions
! 10: * are met:
! 11: * 1. Redistributions of source code must retain the above copyright
! 12: * notice, this list of conditions and the following disclaimer.
! 13: * 2. Redistributions in binary form must reproduce the above copyright
! 14: * notice, this list of conditions and the following disclaimer in the
! 15: * documentation and/or other materials provided with the distribution.
! 16: * 3. The name of the author may not be used to endorse or promote products
! 17: * derived from this software without specific prior written permission.
! 18: *
! 19: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
! 20: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
! 21: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
! 22: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
! 23: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
! 24: * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
! 25: * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
! 26: * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
! 27: * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
! 28: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
! 29: * SUCH DAMAGE.
! 30: *
! 31: * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
! 32: * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
! 33: */
! 34:
! 35: #include <sys/param.h>
! 36: #include <sys/systm.h>
! 37: #include <sys/buf.h>
! 38: #include <sys/conf.h>
! 39: #include <sys/proc.h>
! 40: #include <sys/namei.h>
! 41: #include <sys/disklabel.h>
! 42: #include <sys/errno.h>
! 43: #include <sys/kernel.h>
! 44: #include <sys/malloc.h>
! 45: #include <sys/vnode.h>
! 46: #include <sys/file.h>
! 47: #include <sys/extent.h>
! 48: #include <sys/mount.h>
! 49: #include <sys/pool.h>
! 50: #include <sys/syscallargs.h>
! 51: #include <sys/swap.h>
! 52:
! 53: #include <uvm/uvm.h>
! 54: #ifdef UVM_SWAP_ENCRYPT
! 55: #include <sys/syslog.h>
! 56: #endif
! 57:
! 58: #include <miscfs/specfs/specdev.h>
! 59:
! 60: /*
! 61: * uvm_swap.c: manage configuration and i/o to swap space.
! 62: */
! 63:
! 64: /*
! 65: * swap space is managed in the following way:
! 66: *
! 67: * each swap partition or file is described by a "swapdev" structure.
! 68: * each "swapdev" structure contains a "swapent" structure which contains
! 69: * information that is passed up to the user (via system calls).
! 70: *
! 71: * each swap partition is assigned a "priority" (int) which controls
! 72: * swap partition usage.
! 73: *
! 74: * the system maintains a global data structure describing all swap
! 75: * partitions/files. there is a sorted LIST of "swappri" structures
! 76: * which describe "swapdev"'s at that priority. this LIST is headed
! 77: * by the "swap_priority" global var. each "swappri" contains a
! 78: * CIRCLEQ of "swapdev" structures at that priority.
! 79: *
! 80: * locking:
! 81: * - swap_syscall_lock (sleep lock): this lock serializes the swapctl
! 82: * system call and prevents the swap priority list from changing
! 83: * while we are in the middle of a system call (e.g. SWAP_STATS).
! 84: * - uvm.swap_data_lock (simple_lock): this lock protects all swap data
! 85: * structures including the priority list, the swapdev structures,
! 86: * and the swapmap extent.
! 87: *
! 88: * each swap device has the following info:
! 89: * - swap device in use (could be disabled, preventing future use)
! 90: * - swap enabled (allows new allocations on swap)
! 91: * - map info in /dev/drum
! 92: * - vnode pointer
! 93: * for swap files only:
! 94: * - block size
! 95: * - max byte count in buffer
! 96: * - buffer
! 97: * - credentials to use when doing i/o to file
! 98: *
! 99: * userland controls and configures swap with the swapctl(2) system call.
! 100: * the sys_swapctl performs the following operations:
! 101: * [1] SWAP_NSWAP: returns the number of swap devices currently configured
! 102: * [2] SWAP_STATS: given a pointer to an array of swapent structures
! 103: * (passed in via "arg") of a size passed in via "misc" ... we load
! 104: * the current swap config into the array.
! 105: * [3] SWAP_ON: given a pathname in arg (could be device or file) and a
! 106: * priority in "misc", start swapping on it.
! 107: * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
! 108: * [5] SWAP_CTL: changes the priority of a swap device (new priority in
! 109: * "misc")
! 110: */
! 111:
! 112: /*
! 113: * swapdev: describes a single swap partition/file
! 114: *
! 115: * note the following should be true:
! 116: * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks]
! 117: * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel]
! 118: */
! 119: struct swapdev {
! 120: struct swapent swd_se;
! 121: #define swd_dev swd_se.se_dev /* device id */
! 122: #define swd_flags swd_se.se_flags /* flags:inuse/enable/fake */
! 123: #define swd_priority swd_se.se_priority /* our priority */
! 124: #define swd_inuse swd_se.se_inuse /* our priority */
! 125: #define swd_nblks swd_se.se_nblks /* our priority */
! 126: char *swd_path; /* saved pathname of device */
! 127: int swd_pathlen; /* length of pathname */
! 128: int swd_npages; /* #pages we can use */
! 129: int swd_npginuse; /* #pages in use */
! 130: int swd_npgbad; /* #pages bad */
! 131: int swd_drumoffset; /* page0 offset in drum */
! 132: int swd_drumsize; /* #pages in drum */
! 133: struct extent *swd_ex; /* extent for this swapdev */
! 134: char swd_exname[12]; /* name of extent above */
! 135: struct vnode *swd_vp; /* backing vnode */
! 136: CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */
! 137:
! 138: int swd_bsize; /* blocksize (bytes) */
! 139: int swd_maxactive; /* max active i/o reqs */
! 140: struct buf swd_tab; /* buffer list */
! 141: struct ucred *swd_cred; /* cred for file access */
! 142: #ifdef UVM_SWAP_ENCRYPT
! 143: #define SWD_KEY_SHIFT 7 /* One key per 0.5 MByte */
! 144: #define SWD_KEY(x,y) &((x)->swd_keys[((y) - (x)->swd_drumoffset) >> SWD_KEY_SHIFT])
! 145:
! 146: #define SWD_DCRYPT_SHIFT 5
! 147: #define SWD_DCRYPT_BITS 32
! 148: #define SWD_DCRYPT_MASK (SWD_DCRYPT_BITS - 1)
! 149: #define SWD_DCRYPT_OFF(x) ((x) >> SWD_DCRYPT_SHIFT)
! 150: #define SWD_DCRYPT_BIT(x) ((x) & SWD_DCRYPT_MASK)
! 151: #define SWD_DCRYPT_SIZE(x) (SWD_DCRYPT_OFF((x) + SWD_DCRYPT_MASK) * sizeof(u_int32_t))
! 152: u_int32_t *swd_decrypt; /* bitmap for decryption */
! 153: struct swap_key *swd_keys; /* keys for different parts */
! 154: int swd_nkeys; /* active keys */
! 155: #endif
! 156: };
! 157:
! 158: /*
! 159: * swap device priority entry; the list is kept sorted on `spi_priority'.
! 160: */
! 161: struct swappri {
! 162: int spi_priority; /* priority */
! 163: CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev;
! 164: /* circleq of swapdevs at this priority */
! 165: LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */
! 166: };
! 167:
! 168: /*
! 169: * The following two structures are used to keep track of data transfers
! 170: * on swap devices associated with regular files.
! 171: * NOTE: this code is more or less a copy of vnd.c; we use the same
! 172: * structure names here to ease porting..
! 173: */
! 174: struct vndxfer {
! 175: struct buf *vx_bp; /* Pointer to parent buffer */
! 176: struct swapdev *vx_sdp;
! 177: int vx_error;
! 178: int vx_pending; /* # of pending aux buffers */
! 179: int vx_flags;
! 180: #define VX_BUSY 1
! 181: #define VX_DEAD 2
! 182: };
! 183:
! 184: struct vndbuf {
! 185: struct buf vb_buf;
! 186: struct vndxfer *vb_xfer;
! 187: };
! 188:
! 189:
! 190: /*
! 191: * We keep a of pool vndbuf's and vndxfer structures.
! 192: */
! 193: struct pool vndxfer_pool;
! 194: struct pool vndbuf_pool;
! 195:
! 196: #define getvndxfer(vnx) do { \
! 197: int s = splbio(); \
! 198: vnx = pool_get(&vndxfer_pool, PR_WAITOK); \
! 199: splx(s); \
! 200: } while (0)
! 201:
! 202: #define putvndxfer(vnx) { \
! 203: pool_put(&vndxfer_pool, (void *)(vnx)); \
! 204: }
! 205:
! 206: #define getvndbuf(vbp) do { \
! 207: int s = splbio(); \
! 208: vbp = pool_get(&vndbuf_pool, PR_WAITOK); \
! 209: splx(s); \
! 210: } while (0)
! 211:
! 212: #define putvndbuf(vbp) { \
! 213: pool_put(&vndbuf_pool, (void *)(vbp)); \
! 214: }
! 215:
! 216: /* /dev/drum */
! 217: bdev_decl(sw);
! 218: cdev_decl(sw);
! 219:
! 220: /*
! 221: * local variables
! 222: */
! 223: static struct extent *swapmap; /* controls the mapping of /dev/drum */
! 224:
! 225: /* list of all active swap devices [by priority] */
! 226: LIST_HEAD(swap_priority, swappri);
! 227: static struct swap_priority swap_priority;
! 228:
! 229: /* locks */
! 230: struct rwlock swap_syscall_lock = RWLOCK_INITIALIZER("swplk");
! 231:
! 232: /*
! 233: * prototypes
! 234: */
! 235: static void swapdrum_add(struct swapdev *, int);
! 236: static struct swapdev *swapdrum_getsdp(int);
! 237:
! 238: static struct swapdev *swaplist_find(struct vnode *, int);
! 239: static void swaplist_insert(struct swapdev *,
! 240: struct swappri *, int);
! 241: static void swaplist_trim(void);
! 242:
! 243: static int swap_on(struct proc *, struct swapdev *);
! 244: static int swap_off(struct proc *, struct swapdev *);
! 245:
! 246: static void sw_reg_strategy(struct swapdev *, struct buf *, int);
! 247: static void sw_reg_iodone(struct buf *);
! 248: static void sw_reg_start(struct swapdev *);
! 249:
! 250: static int uvm_swap_io(struct vm_page **, int, int, int);
! 251:
! 252: static void swapmount(void);
! 253:
! 254: #ifdef UVM_SWAP_ENCRYPT
! 255: /* for swap encrypt */
! 256: boolean_t uvm_swap_allocpages(struct vm_page **, int);
! 257: void uvm_swap_markdecrypt(struct swapdev *, int, int, int);
! 258: boolean_t uvm_swap_needdecrypt(struct swapdev *, int);
! 259: void uvm_swap_initcrypt(struct swapdev *, int);
! 260: #endif
! 261:
! 262: /*
! 263: * uvm_swap_init: init the swap system data structures and locks
! 264: *
! 265: * => called at boot time from init_main.c after the filesystems
! 266: * are brought up (which happens after uvm_init())
! 267: */
! 268: void
! 269: uvm_swap_init()
! 270: {
! 271: UVMHIST_FUNC("uvm_swap_init");
! 272:
! 273: UVMHIST_CALLED(pdhist);
! 274: /*
! 275: * first, init the swap list, its counter, and its lock.
! 276: * then get a handle on the vnode for /dev/drum by using
! 277: * the its dev_t number ("swapdev", from MD conf.c).
! 278: */
! 279:
! 280: LIST_INIT(&swap_priority);
! 281: uvmexp.nswapdev = 0;
! 282: simple_lock_init(&uvm.swap_data_lock);
! 283:
! 284: if (!swapdev_vp && bdevvp(swapdev, &swapdev_vp))
! 285: panic("uvm_swap_init: can't get vnode for swap device");
! 286:
! 287: /*
! 288: * create swap block resource map to map /dev/drum. the range
! 289: * from 1 to INT_MAX allows 2 gigablocks of swap space. note
! 290: * that block 0 is reserved (used to indicate an allocation
! 291: * failure, or no allocation).
! 292: */
! 293: swapmap = extent_create("swapmap", 1, INT_MAX,
! 294: M_VMSWAP, 0, 0, EX_NOWAIT);
! 295: if (swapmap == 0)
! 296: panic("uvm_swap_init: extent_create failed");
! 297:
! 298: /*
! 299: * allocate pools for structures used for swapping to files.
! 300: */
! 301:
! 302:
! 303: pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx",
! 304: NULL);
! 305:
! 306: pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd",
! 307: NULL);
! 308:
! 309: /*
! 310: * Setup the initial swap partition
! 311: */
! 312: swapmount();
! 313:
! 314: /*
! 315: * done!
! 316: */
! 317: UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0);
! 318: }
! 319:
! 320: #ifdef UVM_SWAP_ENCRYPT
! 321: void
! 322: uvm_swap_initcrypt_all(void)
! 323: {
! 324: struct swapdev *sdp;
! 325: struct swappri *spp;
! 326:
! 327: simple_lock(&uvm.swap_data_lock);
! 328:
! 329: LIST_FOREACH(spp, &swap_priority, spi_swappri) {
! 330: CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next)
! 331: if (sdp->swd_decrypt == NULL)
! 332: uvm_swap_initcrypt(sdp, sdp->swd_npages);
! 333: }
! 334: simple_unlock(&uvm.swap_data_lock);
! 335: }
! 336:
! 337: void
! 338: uvm_swap_initcrypt(struct swapdev *sdp, int npages)
! 339: {
! 340: /*
! 341: * keep information if a page needs to be decrypted when we get it
! 342: * from the swap device.
! 343: * We cannot chance a malloc later, if we are doing ASYNC puts,
! 344: * we may not call malloc with M_WAITOK. This consumes only
! 345: * 8KB memory for a 256MB swap partition.
! 346: */
! 347: sdp->swd_decrypt = malloc(SWD_DCRYPT_SIZE(npages), M_VMSWAP, M_WAITOK);
! 348: memset(sdp->swd_decrypt, 0, SWD_DCRYPT_SIZE(npages));
! 349: sdp->swd_keys = malloc((npages >> SWD_KEY_SHIFT) * sizeof(struct swap_key),
! 350: M_VMSWAP, M_WAITOK);
! 351: memset(sdp->swd_keys, 0, (npages >> SWD_KEY_SHIFT) * sizeof(struct swap_key));
! 352: sdp->swd_nkeys = 0;
! 353: }
! 354:
! 355: boolean_t
! 356: uvm_swap_allocpages(struct vm_page **pps, int npages)
! 357: {
! 358: int i, s;
! 359: int minus, reserve;
! 360: boolean_t fail;
! 361:
! 362: /* Estimate if we will succeed */
! 363: s = uvm_lock_fpageq();
! 364:
! 365: minus = uvmexp.free - npages;
! 366: reserve = uvmexp.reserve_kernel;
! 367: fail = uvmexp.free - npages < uvmexp.reserve_kernel;
! 368:
! 369: uvm_unlock_fpageq(s);
! 370:
! 371: if (fail)
! 372: return FALSE;
! 373:
! 374: /* Get new pages */
! 375: for (i = 0; i < npages; i++) {
! 376: pps[i] = uvm_pagealloc(NULL, 0, NULL, 0);
! 377: if (pps[i] == NULL)
! 378: break;
! 379: }
! 380:
! 381: /* On failure free and return */
! 382: if (i < npages) {
! 383: uvm_swap_freepages(pps, i);
! 384: return FALSE;
! 385: }
! 386:
! 387: return TRUE;
! 388: }
! 389:
! 390: void
! 391: uvm_swap_freepages(struct vm_page **pps, int npages)
! 392: {
! 393: int i;
! 394:
! 395: uvm_lock_pageq();
! 396: for (i = 0; i < npages; i++)
! 397: uvm_pagefree(pps[i]);
! 398: uvm_unlock_pageq();
! 399: }
! 400:
! 401: /*
! 402: * Mark pages on the swap device for later decryption
! 403: */
! 404:
! 405: void
! 406: uvm_swap_markdecrypt(struct swapdev *sdp, int startslot, int npages,
! 407: int decrypt)
! 408: {
! 409: int pagestart, i;
! 410: int off, bit;
! 411:
! 412: if (!sdp)
! 413: return;
! 414:
! 415: pagestart = startslot - sdp->swd_drumoffset;
! 416: for (i = 0; i < npages; i++, pagestart++) {
! 417: off = SWD_DCRYPT_OFF(pagestart);
! 418: bit = SWD_DCRYPT_BIT(pagestart);
! 419: if (decrypt)
! 420: /* pages read need decryption */
! 421: sdp->swd_decrypt[off] |= 1 << bit;
! 422: else
! 423: /* pages read do not need decryption */
! 424: sdp->swd_decrypt[off] &= ~(1 << bit);
! 425: }
! 426: }
! 427:
! 428: /*
! 429: * Check if the page that we got from disk needs to be decrypted
! 430: */
! 431:
! 432: boolean_t
! 433: uvm_swap_needdecrypt(struct swapdev *sdp, int off)
! 434: {
! 435: if (!sdp)
! 436: return FALSE;
! 437:
! 438: off -= sdp->swd_drumoffset;
! 439: return sdp->swd_decrypt[SWD_DCRYPT_OFF(off)] & (1 << SWD_DCRYPT_BIT(off)) ?
! 440: TRUE : FALSE;
! 441: }
! 442: #endif /* UVM_SWAP_ENCRYPT */
! 443: /*
! 444: * swaplist functions: functions that operate on the list of swap
! 445: * devices on the system.
! 446: */
! 447:
! 448: /*
! 449: * swaplist_insert: insert swap device "sdp" into the global list
! 450: *
! 451: * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
! 452: * => caller must provide a newly malloc'd swappri structure (we will
! 453: * FREE it if we don't need it... this it to prevent malloc blocking
! 454: * here while adding swap)
! 455: */
! 456: static void
! 457: swaplist_insert(sdp, newspp, priority)
! 458: struct swapdev *sdp;
! 459: struct swappri *newspp;
! 460: int priority;
! 461: {
! 462: struct swappri *spp, *pspp;
! 463: UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist);
! 464:
! 465: /*
! 466: * find entry at or after which to insert the new device.
! 467: */
! 468: for (pspp = NULL, spp = LIST_FIRST(&swap_priority); spp != NULL;
! 469: spp = LIST_NEXT(spp, spi_swappri)) {
! 470: if (priority <= spp->spi_priority)
! 471: break;
! 472: pspp = spp;
! 473: }
! 474:
! 475: /*
! 476: * new priority?
! 477: */
! 478: if (spp == NULL || spp->spi_priority != priority) {
! 479: spp = newspp; /* use newspp! */
! 480: UVMHIST_LOG(pdhist, "created new swappri = %ld",
! 481: priority, 0, 0, 0);
! 482:
! 483: spp->spi_priority = priority;
! 484: CIRCLEQ_INIT(&spp->spi_swapdev);
! 485:
! 486: if (pspp)
! 487: LIST_INSERT_AFTER(pspp, spp, spi_swappri);
! 488: else
! 489: LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri);
! 490: } else {
! 491: /* we don't need a new priority structure, free it */
! 492: FREE(newspp, M_VMSWAP);
! 493: }
! 494:
! 495: /*
! 496: * priority found (or created). now insert on the priority's
! 497: * circleq list and bump the total number of swapdevs.
! 498: */
! 499: sdp->swd_priority = priority;
! 500: CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
! 501: uvmexp.nswapdev++;
! 502: }
! 503:
! 504: /*
! 505: * swaplist_find: find and optionally remove a swap device from the
! 506: * global list.
! 507: *
! 508: * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
! 509: * => we return the swapdev we found (and removed)
! 510: */
! 511: static struct swapdev *
! 512: swaplist_find(vp, remove)
! 513: struct vnode *vp;
! 514: boolean_t remove;
! 515: {
! 516: struct swapdev *sdp;
! 517: struct swappri *spp;
! 518:
! 519: /*
! 520: * search the lists for the requested vp
! 521: */
! 522: for (spp = LIST_FIRST(&swap_priority); spp != NULL;
! 523: spp = LIST_NEXT(spp, spi_swappri)) {
! 524: for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
! 525: sdp != (void *)&spp->spi_swapdev;
! 526: sdp = CIRCLEQ_NEXT(sdp, swd_next))
! 527: if (sdp->swd_vp == vp) {
! 528: if (remove) {
! 529: CIRCLEQ_REMOVE(&spp->spi_swapdev,
! 530: sdp, swd_next);
! 531: uvmexp.nswapdev--;
! 532: }
! 533: return(sdp);
! 534: }
! 535: }
! 536: return (NULL);
! 537: }
! 538:
! 539:
! 540: /*
! 541: * swaplist_trim: scan priority list for empty priority entries and kill
! 542: * them.
! 543: *
! 544: * => caller must hold both swap_syscall_lock and uvm.swap_data_lock
! 545: */
! 546: static void
! 547: swaplist_trim()
! 548: {
! 549: struct swappri *spp, *nextspp;
! 550:
! 551: for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) {
! 552: nextspp = LIST_NEXT(spp, spi_swappri);
! 553: if (CIRCLEQ_FIRST(&spp->spi_swapdev) !=
! 554: (void *)&spp->spi_swapdev)
! 555: continue;
! 556: LIST_REMOVE(spp, spi_swappri);
! 557: free(spp, M_VMSWAP);
! 558: }
! 559: }
! 560:
! 561: /*
! 562: * swapdrum_add: add a "swapdev"'s blocks into /dev/drum's area.
! 563: *
! 564: * => caller must hold swap_syscall_lock
! 565: * => uvm.swap_data_lock should be unlocked (we may sleep)
! 566: */
! 567: static void
! 568: swapdrum_add(sdp, npages)
! 569: struct swapdev *sdp;
! 570: int npages;
! 571: {
! 572: u_long result;
! 573:
! 574: if (extent_alloc(swapmap, npages, EX_NOALIGN, 0, EX_NOBOUNDARY,
! 575: EX_WAITOK, &result))
! 576: panic("swapdrum_add");
! 577:
! 578: sdp->swd_drumoffset = result;
! 579: sdp->swd_drumsize = npages;
! 580: }
! 581:
! 582: /*
! 583: * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
! 584: * to the "swapdev" that maps that section of the drum.
! 585: *
! 586: * => each swapdev takes one big contig chunk of the drum
! 587: * => caller must hold uvm.swap_data_lock
! 588: */
! 589: static struct swapdev *
! 590: swapdrum_getsdp(pgno)
! 591: int pgno;
! 592: {
! 593: struct swapdev *sdp;
! 594: struct swappri *spp;
! 595:
! 596: for (spp = LIST_FIRST(&swap_priority); spp != NULL;
! 597: spp = LIST_NEXT(spp, spi_swappri))
! 598: for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
! 599: sdp != (void *)&spp->spi_swapdev;
! 600: sdp = CIRCLEQ_NEXT(sdp, swd_next))
! 601: if (pgno >= sdp->swd_drumoffset &&
! 602: pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
! 603: return sdp;
! 604: }
! 605: return NULL;
! 606: }
! 607:
! 608:
! 609: /*
! 610: * sys_swapctl: main entry point for swapctl(2) system call
! 611: * [with two helper functions: swap_on and swap_off]
! 612: */
! 613: int
! 614: sys_swapctl(p, v, retval)
! 615: struct proc *p;
! 616: void *v;
! 617: register_t *retval;
! 618: {
! 619: struct sys_swapctl_args /* {
! 620: syscallarg(int) cmd;
! 621: syscallarg(void *) arg;
! 622: syscallarg(int) misc;
! 623: } */ *uap = (struct sys_swapctl_args *)v;
! 624: struct vnode *vp;
! 625: struct nameidata nd;
! 626: struct swappri *spp;
! 627: struct swapdev *sdp;
! 628: struct swapent *sep;
! 629: char userpath[MAXPATHLEN];
! 630: size_t len;
! 631: int count, error, misc;
! 632: int priority;
! 633: UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist);
! 634:
! 635: misc = SCARG(uap, misc);
! 636:
! 637: /*
! 638: * ensure serialized syscall access by grabbing the swap_syscall_lock
! 639: */
! 640: rw_enter_write(&swap_syscall_lock);
! 641:
! 642: /*
! 643: * we handle the non-priv NSWAP and STATS request first.
! 644: *
! 645: * SWAP_NSWAP: return number of config'd swap devices
! 646: * [can also be obtained with uvmexp sysctl]
! 647: */
! 648: if (SCARG(uap, cmd) == SWAP_NSWAP) {
! 649: UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%ld", uvmexp.nswapdev,
! 650: 0, 0, 0);
! 651: *retval = uvmexp.nswapdev;
! 652: error = 0;
! 653: goto out;
! 654: }
! 655:
! 656: /*
! 657: * SWAP_STATS: get stats on current # of configured swap devs
! 658: *
! 659: * note that the swap_priority list can't change as long
! 660: * as we are holding the swap_syscall_lock. we don't want
! 661: * to grab the uvm.swap_data_lock because we may fault&sleep during
! 662: * copyout() and we don't want to be holding that lock then!
! 663: */
! 664: if (SCARG(uap, cmd) == SWAP_STATS
! 665: #if defined(COMPAT_13)
! 666: || SCARG(uap, cmd) == SWAP_OSTATS
! 667: #endif
! 668: ) {
! 669: sep = (struct swapent *)SCARG(uap, arg);
! 670: count = 0;
! 671:
! 672: for (spp = LIST_FIRST(&swap_priority); spp != NULL;
! 673: spp = LIST_NEXT(spp, spi_swappri)) {
! 674: for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
! 675: sdp != (void *)&spp->spi_swapdev && misc-- > 0;
! 676: sdp = CIRCLEQ_NEXT(sdp, swd_next)) {
! 677: sdp->swd_inuse =
! 678: btodb((u_int64_t)sdp->swd_npginuse <<
! 679: PAGE_SHIFT);
! 680: error = copyout(&sdp->swd_se, sep,
! 681: sizeof(struct swapent));
! 682:
! 683: /* now copy out the path if necessary */
! 684: #if defined(COMPAT_13)
! 685: if (error == 0 && SCARG(uap, cmd) == SWAP_STATS)
! 686: #else
! 687: if (error == 0)
! 688: #endif
! 689: error = copyout(sdp->swd_path,
! 690: &sep->se_path, sdp->swd_pathlen);
! 691:
! 692: if (error)
! 693: goto out;
! 694: count++;
! 695: #if defined(COMPAT_13)
! 696: if (SCARG(uap, cmd) == SWAP_OSTATS)
! 697: ((struct oswapent *)sep)++;
! 698: else
! 699: #endif
! 700: sep++;
! 701: }
! 702: }
! 703:
! 704: UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0);
! 705:
! 706: *retval = count;
! 707: error = 0;
! 708: goto out;
! 709: }
! 710:
! 711: /*
! 712: * all other requests require superuser privs. verify.
! 713: */
! 714: if ((error = suser(p, 0)))
! 715: goto out;
! 716:
! 717: /*
! 718: * at this point we expect a path name in arg. we will
! 719: * use namei() to gain a vnode reference (vref), and lock
! 720: * the vnode (VOP_LOCK).
! 721: *
! 722: * XXX: a NULL arg means use the root vnode pointer (e.g. for
! 723: * miniroot)
! 724: */
! 725: if (SCARG(uap, arg) == NULL) {
! 726: vp = rootvp; /* miniroot */
! 727: if (vget(vp, LK_EXCLUSIVE, p)) {
! 728: error = EBUSY;
! 729: goto out;
! 730: }
! 731: if (SCARG(uap, cmd) == SWAP_ON &&
! 732: copystr("miniroot", userpath, sizeof userpath, &len))
! 733: panic("swapctl: miniroot copy failed");
! 734: } else {
! 735: int space;
! 736: char *where;
! 737:
! 738: if (SCARG(uap, cmd) == SWAP_ON) {
! 739: if ((error = copyinstr(SCARG(uap, arg), userpath,
! 740: sizeof userpath, &len)))
! 741: goto out;
! 742: space = UIO_SYSSPACE;
! 743: where = userpath;
! 744: } else {
! 745: space = UIO_USERSPACE;
! 746: where = (char *)SCARG(uap, arg);
! 747: }
! 748: NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p);
! 749: if ((error = namei(&nd)))
! 750: goto out;
! 751: vp = nd.ni_vp;
! 752: }
! 753: /* note: "vp" is referenced and locked */
! 754:
! 755: error = 0; /* assume no error */
! 756: switch(SCARG(uap, cmd)) {
! 757:
! 758: case SWAP_DUMPDEV:
! 759: if (vp->v_type != VBLK) {
! 760: error = ENOTBLK;
! 761: break;
! 762: }
! 763: dumpdev = vp->v_rdev;
! 764: break;
! 765:
! 766: case SWAP_CTL:
! 767: /*
! 768: * get new priority, remove old entry (if any) and then
! 769: * reinsert it in the correct place. finally, prune out
! 770: * any empty priority structures.
! 771: */
! 772: priority = SCARG(uap, misc);
! 773: spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
! 774: simple_lock(&uvm.swap_data_lock);
! 775: if ((sdp = swaplist_find(vp, 1)) == NULL) {
! 776: error = ENOENT;
! 777: } else {
! 778: swaplist_insert(sdp, spp, priority);
! 779: swaplist_trim();
! 780: }
! 781: simple_unlock(&uvm.swap_data_lock);
! 782: if (error)
! 783: free(spp, M_VMSWAP);
! 784: break;
! 785:
! 786: case SWAP_ON:
! 787:
! 788: /*
! 789: * check for duplicates. if none found, then insert a
! 790: * dummy entry on the list to prevent someone else from
! 791: * trying to enable this device while we are working on
! 792: * it.
! 793: */
! 794:
! 795: priority = SCARG(uap, misc);
! 796: simple_lock(&uvm.swap_data_lock);
! 797: if ((sdp = swaplist_find(vp, 0)) != NULL) {
! 798: error = EBUSY;
! 799: simple_unlock(&uvm.swap_data_lock);
! 800: break;
! 801: }
! 802: sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK);
! 803: spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK);
! 804: memset(sdp, 0, sizeof(*sdp));
! 805: sdp->swd_flags = SWF_FAKE; /* placeholder only */
! 806: sdp->swd_vp = vp;
! 807: sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV;
! 808:
! 809: /*
! 810: * XXX Is NFS elaboration necessary?
! 811: */
! 812: if (vp->v_type == VREG) {
! 813: sdp->swd_cred = crdup(p->p_ucred);
! 814: }
! 815:
! 816: swaplist_insert(sdp, spp, priority);
! 817: simple_unlock(&uvm.swap_data_lock);
! 818:
! 819: sdp->swd_pathlen = len;
! 820: sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK);
! 821: if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0)
! 822: panic("swapctl: copystr");
! 823:
! 824: /*
! 825: * we've now got a FAKE placeholder in the swap list.
! 826: * now attempt to enable swap on it. if we fail, undo
! 827: * what we've done and kill the fake entry we just inserted.
! 828: * if swap_on is a success, it will clear the SWF_FAKE flag
! 829: */
! 830:
! 831: if ((error = swap_on(p, sdp)) != 0) {
! 832: simple_lock(&uvm.swap_data_lock);
! 833: (void) swaplist_find(vp, 1); /* kill fake entry */
! 834: swaplist_trim();
! 835: simple_unlock(&uvm.swap_data_lock);
! 836: if (vp->v_type == VREG) {
! 837: crfree(sdp->swd_cred);
! 838: }
! 839: free(sdp->swd_path, M_VMSWAP);
! 840: free(sdp, M_VMSWAP);
! 841: break;
! 842: }
! 843: break;
! 844:
! 845: case SWAP_OFF:
! 846: simple_lock(&uvm.swap_data_lock);
! 847: if ((sdp = swaplist_find(vp, 0)) == NULL) {
! 848: simple_unlock(&uvm.swap_data_lock);
! 849: error = ENXIO;
! 850: break;
! 851: }
! 852:
! 853: /*
! 854: * If a device isn't in use or enabled, we
! 855: * can't stop swapping from it (again).
! 856: */
! 857: if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) {
! 858: simple_unlock(&uvm.swap_data_lock);
! 859: error = EBUSY;
! 860: break;
! 861: }
! 862:
! 863: /*
! 864: * do the real work.
! 865: */
! 866: error = swap_off(p, sdp);
! 867: break;
! 868:
! 869: default:
! 870: error = EINVAL;
! 871: }
! 872:
! 873: /*
! 874: * done! release the ref gained by namei() and unlock.
! 875: */
! 876: vput(vp);
! 877:
! 878: out:
! 879: rw_exit_write(&swap_syscall_lock);
! 880:
! 881: UVMHIST_LOG(pdhist, "<- done! error=%ld", error, 0, 0, 0);
! 882: return (error);
! 883: }
! 884:
! 885: /*
! 886: * swap_on: attempt to enable a swapdev for swapping. note that the
! 887: * swapdev is already on the global list, but disabled (marked
! 888: * SWF_FAKE).
! 889: *
! 890: * => we avoid the start of the disk (to protect disk labels)
! 891: * => we also avoid the miniroot, if we are swapping to root.
! 892: * => caller should leave uvm.swap_data_lock unlocked, we may lock it
! 893: * if needed.
! 894: */
! 895: static int
! 896: swap_on(p, sdp)
! 897: struct proc *p;
! 898: struct swapdev *sdp;
! 899: {
! 900: static int count = 0; /* static */
! 901: struct vnode *vp;
! 902: int error, npages, nblocks, size;
! 903: long addr;
! 904: struct vattr va;
! 905: #if defined(NFSCLIENT)
! 906: extern int (**nfsv2_vnodeop_p)(void *);
! 907: #endif /* defined(NFSCLIENT) */
! 908: dev_t dev;
! 909: UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist);
! 910:
! 911: /*
! 912: * we want to enable swapping on sdp. the swd_vp contains
! 913: * the vnode we want (locked and ref'd), and the swd_dev
! 914: * contains the dev_t of the file, if it a block device.
! 915: */
! 916:
! 917: vp = sdp->swd_vp;
! 918: dev = sdp->swd_dev;
! 919:
! 920: /*
! 921: * open the swap file (mostly useful for block device files to
! 922: * let device driver know what is up).
! 923: *
! 924: * we skip the open/close for root on swap because the root
! 925: * has already been opened when root was mounted (mountroot).
! 926: */
! 927: if (vp != rootvp) {
! 928: if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p)))
! 929: return (error);
! 930: }
! 931:
! 932: /* XXX this only works for block devices */
! 933: UVMHIST_LOG(pdhist, " dev=%ld, major(dev)=%ld", dev, major(dev), 0,0);
! 934:
! 935: /*
! 936: * we now need to determine the size of the swap area. for
! 937: * block specials we can call the d_psize function.
! 938: * for normal files, we must stat [get attrs].
! 939: *
! 940: * we put the result in nblks.
! 941: * for normal files, we also want the filesystem block size
! 942: * (which we get with statfs).
! 943: */
! 944: switch (vp->v_type) {
! 945: case VBLK:
! 946: if (bdevsw[major(dev)].d_psize == 0 ||
! 947: (nblocks = (*bdevsw[major(dev)].d_psize)(dev)) == -1) {
! 948: error = ENXIO;
! 949: goto bad;
! 950: }
! 951: break;
! 952:
! 953: case VREG:
! 954: if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p)))
! 955: goto bad;
! 956: nblocks = (int)btodb(va.va_size);
! 957: if ((error =
! 958: VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0)
! 959: goto bad;
! 960:
! 961: sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize;
! 962: /*
! 963: * limit the max # of outstanding I/O requests we issue
! 964: * at any one time. take it easy on NFS servers.
! 965: */
! 966: #if defined(NFSCLIENT)
! 967: if (vp->v_op == nfsv2_vnodeop_p)
! 968: sdp->swd_maxactive = 2; /* XXX */
! 969: else
! 970: #endif /* defined(NFSCLIENT) */
! 971: sdp->swd_maxactive = 8; /* XXX */
! 972: break;
! 973:
! 974: default:
! 975: error = ENXIO;
! 976: goto bad;
! 977: }
! 978:
! 979: /*
! 980: * save nblocks in a safe place and convert to pages.
! 981: */
! 982:
! 983: sdp->swd_nblks = nblocks;
! 984: npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT;
! 985:
! 986: /*
! 987: * for block special files, we want to make sure that leave
! 988: * the disklabel and bootblocks alone, so we arrange to skip
! 989: * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
! 990: * note that because of this the "size" can be less than the
! 991: * actual number of blocks on the device.
! 992: */
! 993: if (vp->v_type == VBLK) {
! 994: /* we use pages 1 to (size - 1) [inclusive] */
! 995: size = npages - 1;
! 996: addr = 1;
! 997: } else {
! 998: /* we use pages 0 to (size - 1) [inclusive] */
! 999: size = npages;
! 1000: addr = 0;
! 1001: }
! 1002:
! 1003: /*
! 1004: * make sure we have enough blocks for a reasonable sized swap
! 1005: * area. we want at least one page.
! 1006: */
! 1007:
! 1008: if (size < 1) {
! 1009: UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0);
! 1010: error = EINVAL;
! 1011: goto bad;
! 1012: }
! 1013:
! 1014: UVMHIST_LOG(pdhist, " dev=%lx: size=%ld addr=0x%lx\n",
! 1015: dev, size, addr, 0);
! 1016:
! 1017: /*
! 1018: * now we need to allocate an extent to manage this swap device
! 1019: */
! 1020: snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x",
! 1021: count++);
! 1022:
! 1023: /* note that extent_create's 3rd arg is inclusive, thus "- 1" */
! 1024: sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP,
! 1025: 0, 0, EX_WAITOK);
! 1026: /* allocate the `saved' region from the extent so it won't be used */
! 1027: if (addr) {
! 1028: if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK))
! 1029: panic("disklabel region");
! 1030: }
! 1031:
! 1032: /*
! 1033: * if the vnode we are swapping to is the root vnode
! 1034: * (i.e. we are swapping to the miniroot) then we want
! 1035: * to make sure we don't overwrite it. do a statfs to
! 1036: * find its size and skip over it.
! 1037: */
! 1038: if (vp == rootvp) {
! 1039: struct mount *mp;
! 1040: struct statfs *sp;
! 1041: int rootblocks, rootpages;
! 1042:
! 1043: mp = rootvnode->v_mount;
! 1044: sp = &mp->mnt_stat;
! 1045: rootblocks = sp->f_blocks * btodb(sp->f_bsize);
! 1046: rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT;
! 1047: if (rootpages > size)
! 1048: panic("swap_on: miniroot larger than swap?");
! 1049:
! 1050: if (extent_alloc_region(sdp->swd_ex, addr,
! 1051: rootpages, EX_WAITOK))
! 1052: panic("swap_on: unable to preserve miniroot");
! 1053:
! 1054: size -= rootpages;
! 1055: printf("Preserved %d pages of miniroot ", rootpages);
! 1056: printf("leaving %d pages of swap\n", size);
! 1057: }
! 1058:
! 1059: /*
! 1060: * add a ref to vp to reflect usage as a swap device.
! 1061: */
! 1062: vref(vp);
! 1063:
! 1064: #ifdef UVM_SWAP_ENCRYPT
! 1065: if (uvm_doswapencrypt)
! 1066: uvm_swap_initcrypt(sdp, npages);
! 1067: #endif
! 1068: /*
! 1069: * now add the new swapdev to the drum and enable.
! 1070: */
! 1071: simple_lock(&uvm.swap_data_lock);
! 1072: swapdrum_add(sdp, npages);
! 1073: sdp->swd_npages = size;
! 1074: sdp->swd_flags &= ~SWF_FAKE; /* going live */
! 1075: sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE);
! 1076: uvmexp.swpages += size;
! 1077: simple_unlock(&uvm.swap_data_lock);
! 1078: return (0);
! 1079:
! 1080: bad:
! 1081: /*
! 1082: * failure: close device if necessary and return error.
! 1083: */
! 1084: if (vp != rootvp)
! 1085: (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p);
! 1086: return (error);
! 1087: }
! 1088:
! 1089: /*
! 1090: * swap_off: stop swapping on swapdev
! 1091: *
! 1092: * => swap data should be locked, we will unlock.
! 1093: */
! 1094: static int
! 1095: swap_off(p, sdp)
! 1096: struct proc *p;
! 1097: struct swapdev *sdp;
! 1098: {
! 1099: int error;
! 1100: UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist);
! 1101: UVMHIST_LOG(pdhist, " dev=%lx", sdp->swd_dev,0,0,0);
! 1102:
! 1103: /* disable the swap area being removed */
! 1104: sdp->swd_flags &= ~SWF_ENABLE;
! 1105: simple_unlock(&uvm.swap_data_lock);
! 1106:
! 1107: /*
! 1108: * the idea is to find all the pages that are paged out to this
! 1109: * device, and page them all in. in uvm, swap-backed pageable
! 1110: * memory can take two forms: aobjs and anons. call the
! 1111: * swapoff hook for each subsystem to bring in pages.
! 1112: */
! 1113:
! 1114: if (uao_swap_off(sdp->swd_drumoffset,
! 1115: sdp->swd_drumoffset + sdp->swd_drumsize) ||
! 1116: amap_swap_off(sdp->swd_drumoffset,
! 1117: sdp->swd_drumoffset + sdp->swd_drumsize)) {
! 1118:
! 1119: error = ENOMEM;
! 1120: } else if (sdp->swd_npginuse > sdp->swd_npgbad) {
! 1121: error = EBUSY;
! 1122: }
! 1123:
! 1124: if (error) {
! 1125: simple_lock(&uvm.swap_data_lock);
! 1126: sdp->swd_flags |= SWF_ENABLE;
! 1127: simple_unlock(&uvm.swap_data_lock);
! 1128: return (error);
! 1129: }
! 1130:
! 1131: /*
! 1132: * done with the vnode and saved creds.
! 1133: * drop our ref on the vnode before calling VOP_CLOSE()
! 1134: * so that spec_close() can tell if this is the last close.
! 1135: */
! 1136: if (sdp->swd_vp->v_type == VREG) {
! 1137: crfree(sdp->swd_cred);
! 1138: }
! 1139: vrele(sdp->swd_vp);
! 1140: if (sdp->swd_vp != rootvp) {
! 1141: (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p);
! 1142: }
! 1143:
! 1144: simple_lock(&uvm.swap_data_lock);
! 1145: uvmexp.swpages -= sdp->swd_npages;
! 1146:
! 1147: if (swaplist_find(sdp->swd_vp, 1) == NULL)
! 1148: panic("swap_off: swapdev not in list");
! 1149: swaplist_trim();
! 1150:
! 1151: /*
! 1152: * free all resources!
! 1153: */
! 1154: extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize,
! 1155: EX_WAITOK);
! 1156: extent_destroy(sdp->swd_ex);
! 1157: free(sdp, M_VMSWAP);
! 1158: simple_unlock(&uvm.swap_data_lock);
! 1159: return (0);
! 1160: }
! 1161:
! 1162: /*
! 1163: * /dev/drum interface and i/o functions
! 1164: */
! 1165:
! 1166: /*
! 1167: * swread: the read function for the drum (just a call to physio)
! 1168: */
! 1169: /*ARGSUSED*/
! 1170: int
! 1171: swread(dev, uio, ioflag)
! 1172: dev_t dev;
! 1173: struct uio *uio;
! 1174: int ioflag;
! 1175: {
! 1176: UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist);
! 1177:
! 1178: UVMHIST_LOG(pdhist, " dev=%lx offset=%lx",
! 1179: dev, (u_long)uio->uio_offset, 0, 0);
! 1180: return (physio(swstrategy, NULL, dev, B_READ, minphys, uio));
! 1181: }
! 1182:
! 1183: /*
! 1184: * swwrite: the write function for the drum (just a call to physio)
! 1185: */
! 1186: /*ARGSUSED*/
! 1187: int
! 1188: swwrite(dev, uio, ioflag)
! 1189: dev_t dev;
! 1190: struct uio *uio;
! 1191: int ioflag;
! 1192: {
! 1193: UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist);
! 1194:
! 1195: UVMHIST_LOG(pdhist, " dev=%lx offset=%lx",
! 1196: dev, (u_long)uio->uio_offset, 0, 0);
! 1197: return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio));
! 1198: }
! 1199:
! 1200: /*
! 1201: * swstrategy: perform I/O on the drum
! 1202: *
! 1203: * => we must map the i/o request from the drum to the correct swapdev.
! 1204: */
! 1205: void
! 1206: swstrategy(bp)
! 1207: struct buf *bp;
! 1208: {
! 1209: struct swapdev *sdp;
! 1210: int s, pageno, bn;
! 1211: UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist);
! 1212:
! 1213: /*
! 1214: * convert block number to swapdev. note that swapdev can't
! 1215: * be yanked out from under us because we are holding resources
! 1216: * in it (i.e. the blocks we are doing I/O on).
! 1217: */
! 1218: pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT;
! 1219: simple_lock(&uvm.swap_data_lock);
! 1220: sdp = swapdrum_getsdp(pageno);
! 1221: simple_unlock(&uvm.swap_data_lock);
! 1222: if (sdp == NULL) {
! 1223: bp->b_error = EINVAL;
! 1224: bp->b_flags |= B_ERROR;
! 1225: s = splbio();
! 1226: biodone(bp);
! 1227: splx(s);
! 1228: UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0);
! 1229: return;
! 1230: }
! 1231:
! 1232: /*
! 1233: * convert drum page number to block number on this swapdev.
! 1234: */
! 1235:
! 1236: pageno -= sdp->swd_drumoffset; /* page # on swapdev */
! 1237: bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */
! 1238:
! 1239: UVMHIST_LOG(pdhist, " %s: mapoff=%lx bn=0x%lx bcount=%ld",
! 1240: ((bp->b_flags & B_READ) == 0) ? "write" : "read",
! 1241: sdp->swd_drumoffset, bn, bp->b_bcount);
! 1242:
! 1243: /*
! 1244: * for block devices we finish up here.
! 1245: * for regular files we have to do more work which we delegate
! 1246: * to sw_reg_strategy().
! 1247: */
! 1248:
! 1249: switch (sdp->swd_vp->v_type) {
! 1250: default:
! 1251: panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type);
! 1252:
! 1253: case VBLK:
! 1254:
! 1255: /*
! 1256: * must convert "bp" from an I/O on /dev/drum to an I/O
! 1257: * on the swapdev (sdp).
! 1258: */
! 1259: s = splbio();
! 1260: buf_replacevnode(bp, sdp->swd_vp);
! 1261:
! 1262: bp->b_blkno = bn;
! 1263: splx(s);
! 1264: VOP_STRATEGY(bp);
! 1265: return;
! 1266:
! 1267: case VREG:
! 1268: /*
! 1269: * delegate to sw_reg_strategy function.
! 1270: */
! 1271: sw_reg_strategy(sdp, bp, bn);
! 1272: return;
! 1273: }
! 1274: /* NOTREACHED */
! 1275: }
! 1276:
! 1277: /*
! 1278: * sw_reg_strategy: handle swap i/o to regular files
! 1279: */
! 1280: static void
! 1281: sw_reg_strategy(sdp, bp, bn)
! 1282: struct swapdev *sdp;
! 1283: struct buf *bp;
! 1284: int bn;
! 1285: {
! 1286: struct vnode *vp;
! 1287: struct vndxfer *vnx;
! 1288: daddr64_t nbn;
! 1289: caddr_t addr;
! 1290: off_t byteoff;
! 1291: int s, off, nra, error, sz, resid;
! 1292: UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist);
! 1293:
! 1294: /*
! 1295: * allocate a vndxfer head for this transfer and point it to
! 1296: * our buffer.
! 1297: */
! 1298: getvndxfer(vnx);
! 1299: vnx->vx_flags = VX_BUSY;
! 1300: vnx->vx_error = 0;
! 1301: vnx->vx_pending = 0;
! 1302: vnx->vx_bp = bp;
! 1303: vnx->vx_sdp = sdp;
! 1304:
! 1305: /*
! 1306: * setup for main loop where we read filesystem blocks into
! 1307: * our buffer.
! 1308: */
! 1309: error = 0;
! 1310: bp->b_resid = bp->b_bcount; /* nothing transferred yet! */
! 1311: addr = bp->b_data; /* current position in buffer */
! 1312: byteoff = dbtob((u_int64_t)bn);
! 1313:
! 1314: for (resid = bp->b_resid; resid; resid -= sz) {
! 1315: struct vndbuf *nbp;
! 1316:
! 1317: /*
! 1318: * translate byteoffset into block number. return values:
! 1319: * vp = vnode of underlying device
! 1320: * nbn = new block number (on underlying vnode dev)
! 1321: * nra = num blocks we can read-ahead (excludes requested
! 1322: * block)
! 1323: */
! 1324: nra = 0;
! 1325: error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
! 1326: &vp, &nbn, &nra);
! 1327:
! 1328: if (error == 0 && nbn == (daddr64_t)-1) {
! 1329: /*
! 1330: * this used to just set error, but that doesn't
! 1331: * do the right thing. Instead, it causes random
! 1332: * memory errors. The panic() should remain until
! 1333: * this condition doesn't destabilize the system.
! 1334: */
! 1335: #if 1
! 1336: panic("sw_reg_strategy: swap to sparse file");
! 1337: #else
! 1338: error = EIO; /* failure */
! 1339: #endif
! 1340: }
! 1341:
! 1342: /*
! 1343: * punt if there was an error or a hole in the file.
! 1344: * we must wait for any i/o ops we have already started
! 1345: * to finish before returning.
! 1346: *
! 1347: * XXX we could deal with holes here but it would be
! 1348: * a hassle (in the write case).
! 1349: */
! 1350: if (error) {
! 1351: s = splbio();
! 1352: vnx->vx_error = error; /* pass error up */
! 1353: goto out;
! 1354: }
! 1355:
! 1356: /*
! 1357: * compute the size ("sz") of this transfer (in bytes).
! 1358: */
! 1359: off = byteoff % sdp->swd_bsize;
! 1360: sz = (1 + nra) * sdp->swd_bsize - off;
! 1361: if (sz > resid)
! 1362: sz = resid;
! 1363:
! 1364: UVMHIST_LOG(pdhist, "sw_reg_strategy: "
! 1365: "vp %p/%p offset 0x%lx/0x%llx",
! 1366: sdp->swd_vp, vp, (u_long)byteoff, nbn);
! 1367:
! 1368: /*
! 1369: * now get a buf structure. note that the vb_buf is
! 1370: * at the front of the nbp structure so that you can
! 1371: * cast pointers between the two structure easily.
! 1372: */
! 1373: getvndbuf(nbp);
! 1374: nbp->vb_buf.b_flags = bp->b_flags | B_CALL;
! 1375: nbp->vb_buf.b_bcount = sz;
! 1376: nbp->vb_buf.b_bufsize = sz;
! 1377: nbp->vb_buf.b_error = 0;
! 1378: nbp->vb_buf.b_data = addr;
! 1379: nbp->vb_buf.b_blkno = nbn + btodb(off);
! 1380: nbp->vb_buf.b_proc = bp->b_proc;
! 1381: nbp->vb_buf.b_iodone = sw_reg_iodone;
! 1382: nbp->vb_buf.b_vp = NULLVP;
! 1383: nbp->vb_buf.b_vnbufs.le_next = NOLIST;
! 1384: LIST_INIT(&nbp->vb_buf.b_dep);
! 1385:
! 1386: /*
! 1387: * set b_dirtyoff/end and b_validoff/end. this is
! 1388: * required by the NFS client code (otherwise it will
! 1389: * just discard our I/O request).
! 1390: */
! 1391: if (bp->b_dirtyend == 0) {
! 1392: nbp->vb_buf.b_dirtyoff = 0;
! 1393: nbp->vb_buf.b_dirtyend = sz;
! 1394: } else {
! 1395: nbp->vb_buf.b_dirtyoff =
! 1396: max(0, bp->b_dirtyoff - (bp->b_bcount-resid));
! 1397: nbp->vb_buf.b_dirtyend =
! 1398: min(sz,
! 1399: max(0, bp->b_dirtyend - (bp->b_bcount-resid)));
! 1400: }
! 1401: if (bp->b_validend == 0) {
! 1402: nbp->vb_buf.b_validoff = 0;
! 1403: nbp->vb_buf.b_validend = sz;
! 1404: } else {
! 1405: nbp->vb_buf.b_validoff =
! 1406: max(0, bp->b_validoff - (bp->b_bcount-resid));
! 1407: nbp->vb_buf.b_validend =
! 1408: min(sz,
! 1409: max(0, bp->b_validend - (bp->b_bcount-resid)));
! 1410: }
! 1411:
! 1412: nbp->vb_xfer = vnx; /* patch it back in to vnx */
! 1413:
! 1414: /*
! 1415: * Just sort by block number
! 1416: */
! 1417: nbp->vb_buf.b_cylinder = nbp->vb_buf.b_blkno;
! 1418: s = splbio();
! 1419: if (vnx->vx_error != 0) {
! 1420: putvndbuf(nbp);
! 1421: goto out;
! 1422: }
! 1423: vnx->vx_pending++;
! 1424:
! 1425: /* assoc new buffer with underlying vnode */
! 1426: bgetvp(vp, &nbp->vb_buf);
! 1427:
! 1428: /* sort it in and start I/O if we are not over our limit */
! 1429: disksort(&sdp->swd_tab, &nbp->vb_buf);
! 1430: sw_reg_start(sdp);
! 1431: splx(s);
! 1432:
! 1433: /*
! 1434: * advance to the next I/O
! 1435: */
! 1436: byteoff += sz;
! 1437: addr += sz;
! 1438: }
! 1439:
! 1440: s = splbio();
! 1441:
! 1442: out: /* Arrive here at splbio */
! 1443: vnx->vx_flags &= ~VX_BUSY;
! 1444: if (vnx->vx_pending == 0) {
! 1445: if (vnx->vx_error != 0) {
! 1446: bp->b_error = vnx->vx_error;
! 1447: bp->b_flags |= B_ERROR;
! 1448: }
! 1449: putvndxfer(vnx);
! 1450: biodone(bp);
! 1451: }
! 1452: splx(s);
! 1453: }
! 1454:
! 1455: /*
! 1456: * sw_reg_start: start an I/O request on the requested swapdev
! 1457: *
! 1458: * => reqs are sorted by disksort (above)
! 1459: */
! 1460: static void
! 1461: sw_reg_start(sdp)
! 1462: struct swapdev *sdp;
! 1463: {
! 1464: struct buf *bp;
! 1465: UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist);
! 1466:
! 1467: /* recursion control */
! 1468: if ((sdp->swd_flags & SWF_BUSY) != 0)
! 1469: return;
! 1470:
! 1471: sdp->swd_flags |= SWF_BUSY;
! 1472:
! 1473: while (sdp->swd_tab.b_active < sdp->swd_maxactive) {
! 1474: bp = sdp->swd_tab.b_actf;
! 1475: if (bp == NULL)
! 1476: break;
! 1477: sdp->swd_tab.b_actf = bp->b_actf;
! 1478: sdp->swd_tab.b_active++;
! 1479:
! 1480: UVMHIST_LOG(pdhist,
! 1481: "sw_reg_start: bp %p vp %p blkno 0x%lx cnt 0x%lx",
! 1482: bp, bp->b_vp, bp->b_blkno, bp->b_bcount);
! 1483: if ((bp->b_flags & B_READ) == 0)
! 1484: bp->b_vp->v_numoutput++;
! 1485:
! 1486: VOP_STRATEGY(bp);
! 1487: }
! 1488: sdp->swd_flags &= ~SWF_BUSY;
! 1489: }
! 1490:
! 1491: /*
! 1492: * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
! 1493: *
! 1494: * => note that we can recover the vndbuf struct by casting the buf ptr
! 1495: */
! 1496: static void
! 1497: sw_reg_iodone(bp)
! 1498: struct buf *bp;
! 1499: {
! 1500: struct vndbuf *vbp = (struct vndbuf *) bp;
! 1501: struct vndxfer *vnx = vbp->vb_xfer;
! 1502: struct buf *pbp = vnx->vx_bp; /* parent buffer */
! 1503: struct swapdev *sdp = vnx->vx_sdp;
! 1504: int resid;
! 1505: UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist);
! 1506:
! 1507: UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=0x%lx addr=%p",
! 1508: vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data);
! 1509: UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx",
! 1510: vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0);
! 1511:
! 1512: splassert(IPL_BIO);
! 1513:
! 1514: resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
! 1515: pbp->b_resid -= resid;
! 1516: vnx->vx_pending--;
! 1517:
! 1518: if (vbp->vb_buf.b_error) {
! 1519: UVMHIST_LOG(pdhist, " got error=%ld !",
! 1520: vbp->vb_buf.b_error, 0, 0, 0);
! 1521:
! 1522: /* pass error upward */
! 1523: vnx->vx_error = vbp->vb_buf.b_error;
! 1524: }
! 1525:
! 1526: /*
! 1527: * disassociate this buffer from the vnode (if any).
! 1528: */
! 1529: if (vbp->vb_buf.b_vp != NULL) {
! 1530: brelvp(&vbp->vb_buf);
! 1531: }
! 1532:
! 1533: /*
! 1534: * kill vbp structure
! 1535: */
! 1536: putvndbuf(vbp);
! 1537:
! 1538: /*
! 1539: * wrap up this transaction if it has run to completion or, in
! 1540: * case of an error, when all auxiliary buffers have returned.
! 1541: */
! 1542: if (vnx->vx_error != 0) {
! 1543: /* pass error upward */
! 1544: pbp->b_flags |= B_ERROR;
! 1545: pbp->b_error = vnx->vx_error;
! 1546: if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) {
! 1547: putvndxfer(vnx);
! 1548: biodone(pbp);
! 1549: }
! 1550: } else if (pbp->b_resid == 0) {
! 1551: KASSERT(vnx->vx_pending == 0);
! 1552: if ((vnx->vx_flags & VX_BUSY) == 0) {
! 1553: UVMHIST_LOG(pdhist, " iodone error=%ld !",
! 1554: pbp, vnx->vx_error, 0, 0);
! 1555: putvndxfer(vnx);
! 1556: biodone(pbp);
! 1557: }
! 1558: }
! 1559:
! 1560: /*
! 1561: * done! start next swapdev I/O if one is pending
! 1562: */
! 1563: sdp->swd_tab.b_active--;
! 1564: sw_reg_start(sdp);
! 1565: }
! 1566:
! 1567:
! 1568: /*
! 1569: * uvm_swap_alloc: allocate space on swap
! 1570: *
! 1571: * => allocation is done "round robin" down the priority list, as we
! 1572: * allocate in a priority we "rotate" the circle queue.
! 1573: * => space can be freed with uvm_swap_free
! 1574: * => we return the page slot number in /dev/drum (0 == invalid slot)
! 1575: * => we lock uvm.swap_data_lock
! 1576: * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
! 1577: */
! 1578: int
! 1579: uvm_swap_alloc(nslots, lessok)
! 1580: int *nslots; /* IN/OUT */
! 1581: boolean_t lessok;
! 1582: {
! 1583: struct swapdev *sdp;
! 1584: struct swappri *spp;
! 1585: u_long result;
! 1586: UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist);
! 1587:
! 1588: /*
! 1589: * no swap devices configured yet? definite failure.
! 1590: */
! 1591: if (uvmexp.nswapdev < 1)
! 1592: return 0;
! 1593:
! 1594: /*
! 1595: * lock data lock, convert slots into blocks, and enter loop
! 1596: */
! 1597: simple_lock(&uvm.swap_data_lock);
! 1598:
! 1599: ReTry: /* XXXMRG */
! 1600: for (spp = LIST_FIRST(&swap_priority); spp != NULL;
! 1601: spp = LIST_NEXT(spp, spi_swappri)) {
! 1602: for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev);
! 1603: sdp != (void *)&spp->spi_swapdev;
! 1604: sdp = CIRCLEQ_NEXT(sdp,swd_next)) {
! 1605: /* if it's not enabled, then we can't swap from it */
! 1606: if ((sdp->swd_flags & SWF_ENABLE) == 0)
! 1607: continue;
! 1608: if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
! 1609: continue;
! 1610: if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 0,
! 1611: EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT,
! 1612: &result) != 0) {
! 1613: continue;
! 1614: }
! 1615:
! 1616: /*
! 1617: * successful allocation! now rotate the circleq.
! 1618: */
! 1619: CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
! 1620: CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
! 1621: sdp->swd_npginuse += *nslots;
! 1622: uvmexp.swpginuse += *nslots;
! 1623: simple_unlock(&uvm.swap_data_lock);
! 1624: /* done! return drum slot number */
! 1625: UVMHIST_LOG(pdhist,
! 1626: "success! returning %ld slots starting at %ld",
! 1627: *nslots, result + sdp->swd_drumoffset, 0, 0);
! 1628: return(result + sdp->swd_drumoffset);
! 1629: }
! 1630: }
! 1631:
! 1632: /* XXXMRG: BEGIN HACK */
! 1633: if (*nslots > 1 && lessok) {
! 1634: *nslots = 1;
! 1635: goto ReTry; /* XXXMRG: ugh! extent should support this for us */
! 1636: }
! 1637: /* XXXMRG: END HACK */
! 1638:
! 1639: simple_unlock(&uvm.swap_data_lock);
! 1640: return 0; /* failed */
! 1641: }
! 1642:
! 1643: /*
! 1644: * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
! 1645: *
! 1646: * => we lock uvm.swap_data_lock
! 1647: */
! 1648: void
! 1649: uvm_swap_markbad(startslot, nslots)
! 1650: int startslot;
! 1651: int nslots;
! 1652: {
! 1653: struct swapdev *sdp;
! 1654: UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist);
! 1655:
! 1656: simple_lock(&uvm.swap_data_lock);
! 1657: sdp = swapdrum_getsdp(startslot);
! 1658: if (sdp != NULL) {
! 1659: /*
! 1660: * we just keep track of how many pages have been marked bad
! 1661: * in this device, to make everything add up in swap_off().
! 1662: * we assume here that the range of slots will all be within
! 1663: * one swap device.
! 1664: */
! 1665: sdp->swd_npgbad += nslots;
! 1666: UVMHIST_LOG(pdhist, "now %ld bad", sdp->swd_npgbad, 0,0,0);
! 1667: }
! 1668: simple_unlock(&uvm.swap_data_lock);
! 1669: }
! 1670:
! 1671: /*
! 1672: * uvm_swap_free: free swap slots
! 1673: *
! 1674: * => this can be all or part of an allocation made by uvm_swap_alloc
! 1675: * => we lock uvm.swap_data_lock
! 1676: */
! 1677: void
! 1678: uvm_swap_free(startslot, nslots)
! 1679: int startslot;
! 1680: int nslots;
! 1681: {
! 1682: struct swapdev *sdp;
! 1683: UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist);
! 1684:
! 1685: UVMHIST_LOG(pdhist, "freeing %ld slots starting at %ld", nslots,
! 1686: startslot, 0, 0);
! 1687:
! 1688: /*
! 1689: * ignore attempts to free the "bad" slot.
! 1690: */
! 1691:
! 1692: if (startslot == SWSLOT_BAD) {
! 1693: return;
! 1694: }
! 1695:
! 1696: /*
! 1697: * convert drum slot offset back to sdp, free the blocks
! 1698: * in the extent, and return. must hold pri lock to do
! 1699: * lookup and access the extent.
! 1700: */
! 1701:
! 1702: simple_lock(&uvm.swap_data_lock);
! 1703: sdp = swapdrum_getsdp(startslot);
! 1704: KASSERT(uvmexp.nswapdev >= 1);
! 1705: KASSERT(sdp != NULL);
! 1706: KASSERT(sdp->swd_npginuse >= nslots);
! 1707: if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots,
! 1708: EX_MALLOCOK|EX_NOWAIT) != 0) {
! 1709: printf("warning: resource shortage: %d pages of swap lost\n",
! 1710: nslots);
! 1711: }
! 1712:
! 1713: sdp->swd_npginuse -= nslots;
! 1714: uvmexp.swpginuse -= nslots;
! 1715: #ifdef UVM_SWAP_ENCRYPT
! 1716: {
! 1717: int i;
! 1718: if (swap_encrypt_initialized) {
! 1719: /* Dereference keys */
! 1720: for (i = 0; i < nslots; i++)
! 1721: if (uvm_swap_needdecrypt(sdp, startslot + i))
! 1722: SWAP_KEY_PUT(sdp, SWD_KEY(sdp, startslot + i));
! 1723:
! 1724: /* Mark range as not decrypt */
! 1725: uvm_swap_markdecrypt(sdp, startslot, nslots, 0);
! 1726: }
! 1727: }
! 1728: #endif /* UVM_SWAP_ENCRYPT */
! 1729: simple_unlock(&uvm.swap_data_lock);
! 1730: }
! 1731:
! 1732: /*
! 1733: * uvm_swap_put: put any number of pages into a contig place on swap
! 1734: *
! 1735: * => can be sync or async
! 1736: * => XXXMRG: consider making it an inline or macro
! 1737: */
! 1738: int
! 1739: uvm_swap_put(swslot, ppsp, npages, flags)
! 1740: int swslot;
! 1741: struct vm_page **ppsp;
! 1742: int npages;
! 1743: int flags;
! 1744: {
! 1745: int result;
! 1746:
! 1747: result = uvm_swap_io(ppsp, swslot, npages, B_WRITE |
! 1748: ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
! 1749:
! 1750: return (result);
! 1751: }
! 1752:
! 1753: /*
! 1754: * uvm_swap_get: get a single page from swap
! 1755: *
! 1756: * => usually a sync op (from fault)
! 1757: * => XXXMRG: consider making it an inline or macro
! 1758: */
! 1759: int
! 1760: uvm_swap_get(page, swslot, flags)
! 1761: struct vm_page *page;
! 1762: int swslot, flags;
! 1763: {
! 1764: int result;
! 1765:
! 1766: uvmexp.nswget++;
! 1767: KASSERT(flags & PGO_SYNCIO);
! 1768: if (swslot == SWSLOT_BAD) {
! 1769: return VM_PAGER_ERROR;
! 1770: }
! 1771:
! 1772: /*
! 1773: * this page is (about to be) no longer only in swap.
! 1774: */
! 1775: simple_lock(&uvm.swap_data_lock);
! 1776: uvmexp.swpgonly--;
! 1777: simple_unlock(&uvm.swap_data_lock);
! 1778:
! 1779: result = uvm_swap_io(&page, swslot, 1, B_READ |
! 1780: ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
! 1781:
! 1782: if (result != VM_PAGER_OK && result != VM_PAGER_PEND) {
! 1783: /*
! 1784: * oops, the read failed so it really is still only in swap.
! 1785: */
! 1786: simple_lock(&uvm.swap_data_lock);
! 1787: uvmexp.swpgonly++;
! 1788: simple_unlock(&uvm.swap_data_lock);
! 1789: }
! 1790:
! 1791: return (result);
! 1792: }
! 1793:
! 1794: /*
! 1795: * uvm_swap_io: do an i/o operation to swap
! 1796: */
! 1797:
! 1798: static int
! 1799: uvm_swap_io(pps, startslot, npages, flags)
! 1800: struct vm_page **pps;
! 1801: int startslot, npages, flags;
! 1802: {
! 1803: daddr64_t startblk;
! 1804: struct buf *bp;
! 1805: vaddr_t kva;
! 1806: int result, s, mapinflags, pflag;
! 1807: boolean_t write, async;
! 1808: #ifdef UVM_SWAP_ENCRYPT
! 1809: vaddr_t dstkva;
! 1810: struct vm_page *tpps[MAXBSIZE >> PAGE_SHIFT];
! 1811: struct swapdev *sdp;
! 1812: int encrypt = 0;
! 1813: #endif
! 1814: UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist);
! 1815:
! 1816: UVMHIST_LOG(pdhist, "<- called, startslot=%ld, npages=%ld, flags=%ld",
! 1817: startslot, npages, flags, 0);
! 1818:
! 1819: write = (flags & B_READ) == 0;
! 1820: async = (flags & B_ASYNC) != 0;
! 1821:
! 1822: /*
! 1823: * convert starting drum slot to block number
! 1824: */
! 1825: startblk = btodb((u_int64_t)startslot << PAGE_SHIFT);
! 1826:
! 1827: /*
! 1828: * first, map the pages into the kernel (XXX: currently required
! 1829: * by buffer system).
! 1830: */
! 1831: mapinflags = !write ? UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE;
! 1832: if (!async)
! 1833: mapinflags |= UVMPAGER_MAPIN_WAITOK;
! 1834: kva = uvm_pagermapin(pps, npages, mapinflags);
! 1835: if (kva == 0)
! 1836: return (VM_PAGER_AGAIN);
! 1837:
! 1838: #ifdef UVM_SWAP_ENCRYPT
! 1839: if (write) {
! 1840: /*
! 1841: * Check if we need to do swap encryption on old pages.
! 1842: * Later we need a different scheme, that swap encrypts
! 1843: * all pages of a process that had at least one page swap
! 1844: * encrypted. Then we might not need to copy all pages
! 1845: * in the cluster, and avoid the memory overheard in
! 1846: * swapping.
! 1847: */
! 1848: if (uvm_doswapencrypt)
! 1849: encrypt = 1;
! 1850: }
! 1851:
! 1852: if (swap_encrypt_initialized || encrypt) {
! 1853: /*
! 1854: * we need to know the swap device that we are swapping to/from
! 1855: * to see if the pages need to be marked for decryption or
! 1856: * actually need to be decrypted.
! 1857: * XXX - does this information stay the same over the whole
! 1858: * execution of this function?
! 1859: */
! 1860: simple_lock(&uvm.swap_data_lock);
! 1861: sdp = swapdrum_getsdp(startslot);
! 1862: simple_unlock(&uvm.swap_data_lock);
! 1863: }
! 1864:
! 1865: /*
! 1866: * encrypt to swap
! 1867: */
! 1868: if (write && encrypt) {
! 1869: int i, opages;
! 1870: caddr_t src, dst;
! 1871: struct swap_key *key;
! 1872: u_int64_t block;
! 1873: int swmapflags;
! 1874:
! 1875: /* We always need write access. */
! 1876: swmapflags = UVMPAGER_MAPIN_READ;
! 1877: if (!async)
! 1878: swmapflags |= UVMPAGER_MAPIN_WAITOK;
! 1879:
! 1880: if (!uvm_swap_allocpages(tpps, npages)) {
! 1881: uvm_pagermapout(kva, npages);
! 1882: return (VM_PAGER_AGAIN);
! 1883: }
! 1884:
! 1885: dstkva = uvm_pagermapin(tpps, npages, swmapflags);
! 1886: if (dstkva == 0) {
! 1887: uvm_pagermapout(kva, npages);
! 1888: uvm_swap_freepages(tpps, npages);
! 1889: return (VM_PAGER_AGAIN);
! 1890: }
! 1891:
! 1892: src = (caddr_t) kva;
! 1893: dst = (caddr_t) dstkva;
! 1894: block = startblk;
! 1895: for (i = 0; i < npages; i++) {
! 1896: key = SWD_KEY(sdp, startslot + i);
! 1897: SWAP_KEY_GET(sdp, key); /* add reference */
! 1898:
! 1899: /* mark for async writes */
! 1900: atomic_setbits_int(&tpps[i]->pg_flags, PQ_ENCRYPT);
! 1901: swap_encrypt(key, src, dst, block, 1 << PAGE_SHIFT);
! 1902: src += 1 << PAGE_SHIFT;
! 1903: dst += 1 << PAGE_SHIFT;
! 1904: block += btodb(1 << PAGE_SHIFT);
! 1905: }
! 1906:
! 1907: uvm_pagermapout(kva, npages);
! 1908:
! 1909: /* dispose of pages we dont use anymore */
! 1910: opages = npages;
! 1911: uvm_pager_dropcluster(NULL, NULL, pps, &opages,
! 1912: PGO_PDFREECLUST);
! 1913:
! 1914: kva = dstkva;
! 1915: }
! 1916: #endif /* UVM_SWAP_ENCRYPT */
! 1917:
! 1918: /*
! 1919: * now allocate a buf for the i/o.
! 1920: * [make sure we don't put the pagedaemon to sleep...]
! 1921: */
! 1922: s = splbio();
! 1923: pflag = (async || curproc == uvm.pagedaemon_proc) ? 0 : PR_WAITOK;
! 1924: bp = pool_get(&bufpool, pflag);
! 1925: splx(s);
! 1926:
! 1927: /*
! 1928: * if we failed to get a swapbuf, return "try again"
! 1929: */
! 1930: if (bp == NULL) {
! 1931: #ifdef UVM_SWAP_ENCRYPT
! 1932: if (write && encrypt) {
! 1933: int i;
! 1934:
! 1935: /* swap encrypt needs cleanup */
! 1936: for (i = 0; i < npages; i++)
! 1937: SWAP_KEY_PUT(sdp, SWD_KEY(sdp, startslot + i));
! 1938:
! 1939: uvm_pagermapout(kva, npages);
! 1940: uvm_swap_freepages(tpps, npages);
! 1941: }
! 1942: #endif
! 1943: return (VM_PAGER_AGAIN);
! 1944: }
! 1945:
! 1946: #ifdef UVM_SWAP_ENCRYPT
! 1947: /*
! 1948: * prevent ASYNC reads.
! 1949: * uvm_swap_io is only called from uvm_swap_get, uvm_swap_get
! 1950: * assumes that all gets are SYNCIO. Just make sure here.
! 1951: * XXXARTUBC - might not be true anymore.
! 1952: */
! 1953: if (!write) {
! 1954: flags &= ~B_ASYNC;
! 1955: async = 0;
! 1956: }
! 1957: #endif
! 1958: /*
! 1959: * fill in the bp. we currently route our i/o through
! 1960: * /dev/drum's vnode [swapdev_vp].
! 1961: */
! 1962: bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC));
! 1963: bp->b_proc = &proc0; /* XXX */
! 1964: bp->b_vnbufs.le_next = NOLIST;
! 1965: bp->b_data = (caddr_t)kva;
! 1966: bp->b_blkno = startblk;
! 1967: LIST_INIT(&bp->b_dep);
! 1968: s = splbio();
! 1969: bp->b_vp = NULL;
! 1970: buf_replacevnode(bp, swapdev_vp);
! 1971: splx(s);
! 1972: bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT;
! 1973:
! 1974: /*
! 1975: * for pageouts we must set "dirtyoff" [NFS client code needs it].
! 1976: * and we bump v_numoutput (counter of number of active outputs).
! 1977: */
! 1978: if (write) {
! 1979: bp->b_dirtyoff = 0;
! 1980: bp->b_dirtyend = npages << PAGE_SHIFT;
! 1981: #ifdef UVM_SWAP_ENCRYPT
! 1982: /* mark the pages in the drum for decryption */
! 1983: if (swap_encrypt_initialized)
! 1984: uvm_swap_markdecrypt(sdp, startslot, npages, encrypt);
! 1985: #endif
! 1986: s = splbio();
! 1987: swapdev_vp->v_numoutput++;
! 1988: splx(s);
! 1989: }
! 1990:
! 1991: /*
! 1992: * for async ops we must set up the iodone handler.
! 1993: */
! 1994: if (async) {
! 1995: bp->b_flags |= B_CALL | (curproc == uvm.pagedaemon_proc ?
! 1996: B_PDAEMON : 0);
! 1997: bp->b_iodone = uvm_aio_biodone;
! 1998: UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0);
! 1999: }
! 2000: UVMHIST_LOG(pdhist,
! 2001: "about to start io: data = %p blkno = 0x%lx, bcount = %ld",
! 2002: bp->b_data, bp->b_blkno, bp->b_bcount, 0);
! 2003:
! 2004: /*
! 2005: * now we start the I/O, and if async, return.
! 2006: */
! 2007: VOP_STRATEGY(bp);
! 2008: if (async)
! 2009: return (VM_PAGER_PEND);
! 2010:
! 2011: /*
! 2012: * must be sync i/o. wait for it to finish
! 2013: */
! 2014: (void) biowait(bp);
! 2015: result = (bp->b_flags & B_ERROR) ? VM_PAGER_ERROR : VM_PAGER_OK;
! 2016:
! 2017: #ifdef UVM_SWAP_ENCRYPT
! 2018: /*
! 2019: * decrypt swap
! 2020: */
! 2021: if (swap_encrypt_initialized &&
! 2022: (bp->b_flags & B_READ) && !(bp->b_flags & B_ERROR)) {
! 2023: int i;
! 2024: caddr_t data = bp->b_data;
! 2025: u_int64_t block = startblk;
! 2026: struct swap_key *key = NULL;
! 2027:
! 2028: for (i = 0; i < npages; i++) {
! 2029: /* Check if we need to decrypt */
! 2030: if (uvm_swap_needdecrypt(sdp, startslot + i)) {
! 2031: key = SWD_KEY(sdp, startslot + i);
! 2032: swap_decrypt(key, data, data, block,
! 2033: 1 << PAGE_SHIFT);
! 2034: }
! 2035: data += 1 << PAGE_SHIFT;
! 2036: block += btodb(1 << PAGE_SHIFT);
! 2037: }
! 2038: }
! 2039: #endif
! 2040: /*
! 2041: * kill the pager mapping
! 2042: */
! 2043: uvm_pagermapout(kva, npages);
! 2044:
! 2045: #ifdef UVM_SWAP_ENCRYPT
! 2046: /*
! 2047: * Not anymore needed, free after encryption
! 2048: */
! 2049: if ((bp->b_flags & B_READ) == 0 && encrypt)
! 2050: uvm_swap_freepages(tpps, npages);
! 2051: #endif
! 2052: /*
! 2053: * now dispose of the buf
! 2054: */
! 2055: s = splbio();
! 2056: if (bp->b_vp)
! 2057: brelvp(bp);
! 2058:
! 2059: if (write && bp->b_vp)
! 2060: vwakeup(bp->b_vp);
! 2061: pool_put(&bufpool, bp);
! 2062: splx(s);
! 2063:
! 2064: /*
! 2065: * finally return.
! 2066: */
! 2067: UVMHIST_LOG(pdhist, "<- done (sync) result=%ld", result, 0, 0, 0);
! 2068: return (result);
! 2069: }
! 2070:
! 2071: static void
! 2072: swapmount()
! 2073: {
! 2074: struct swapdev *sdp;
! 2075: struct swappri *spp;
! 2076: struct vnode *vp;
! 2077: dev_t swap_dev = swdevt[0].sw_dev;
! 2078:
! 2079: /*
! 2080: * No locking here since we happen to know that we will just be called
! 2081: * once before any other process has forked.
! 2082: */
! 2083:
! 2084: if (swap_dev == NODEV) {
! 2085: printf("swapmount: no device\n");
! 2086: return;
! 2087: }
! 2088:
! 2089: if (bdevvp(swap_dev, &vp)) {
! 2090: printf("swapmount: no device 2\n");
! 2091: return;
! 2092: }
! 2093:
! 2094: sdp = malloc(sizeof(*sdp), M_VMSWAP, M_WAITOK);
! 2095: spp = malloc(sizeof(*spp), M_VMSWAP, M_WAITOK);
! 2096: memset(sdp, 0, sizeof(*sdp));
! 2097:
! 2098: sdp->swd_flags = SWF_FAKE;
! 2099: sdp->swd_dev = swap_dev;
! 2100: sdp->swd_vp = vp;
! 2101: swaplist_insert(sdp, spp, 0);
! 2102: sdp->swd_pathlen = strlen("swap_device") + 1;
! 2103: sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK);
! 2104: if (copystr("swap_device", sdp->swd_path, sdp->swd_pathlen, 0))
! 2105: panic("swapmount: copystr");
! 2106:
! 2107: if (swap_on(curproc, sdp)) {
! 2108: swaplist_find(vp, 1);
! 2109: swaplist_trim();
! 2110: vput(sdp->swd_vp);
! 2111: free(sdp->swd_path, M_VMSWAP);
! 2112: free(sdp, M_VMSWAP);
! 2113: return;
! 2114: }
! 2115:
! 2116: VOP_UNLOCK(vp, 0, curproc);
! 2117: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to uvm_swap.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / uvm