Annotation of sys/arch/sparc64/dev/iommu.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: iommu.c,v 1.47 2007/05/29 09:53:59 sobrado Exp $ */
! 2: /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */
! 3:
! 4: /*
! 5: * Copyright (c) 2003 Henric Jungheim
! 6: * Copyright (c) 2001, 2002 Eduardo Horvath
! 7: * Copyright (c) 1999, 2000 Matthew R. Green
! 8: * All rights reserved.
! 9: *
! 10: * Redistribution and use in source and binary forms, with or without
! 11: * modification, are permitted provided that the following conditions
! 12: * are met:
! 13: * 1. Redistributions of source code must retain the above copyright
! 14: * notice, this list of conditions and the following disclaimer.
! 15: * 2. Redistributions in binary form must reproduce the above copyright
! 16: * notice, this list of conditions and the following disclaimer in the
! 17: * documentation and/or other materials provided with the distribution.
! 18: * 3. The name of the author may not be used to endorse or promote products
! 19: * derived from this software without specific prior written permission.
! 20: *
! 21: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
! 22: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
! 23: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
! 24: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
! 25: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
! 26: * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
! 27: * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
! 28: * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
! 29: * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
! 30: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
! 31: * SUCH DAMAGE.
! 32: */
! 33:
! 34: /*
! 35: * UltraSPARC IOMMU support; used by both the sbus and pci code.
! 36: */
! 37: #include <sys/param.h>
! 38: #include <sys/extent.h>
! 39: #include <sys/malloc.h>
! 40: #include <sys/systm.h>
! 41: #include <sys/device.h>
! 42: #include <sys/mbuf.h>
! 43:
! 44: #include <uvm/uvm_extern.h>
! 45:
! 46: #include <machine/bus.h>
! 47: #include <sparc64/sparc64/cache.h>
! 48: #include <sparc64/dev/iommureg.h>
! 49: #include <sparc64/dev/iommuvar.h>
! 50:
! 51: #include <machine/autoconf.h>
! 52: #include <machine/cpu.h>
! 53:
! 54: #ifdef DDB
! 55: #include <machine/db_machdep.h>
! 56: #include <ddb/db_sym.h>
! 57: #include <ddb/db_extern.h>
! 58: #endif
! 59:
! 60: #ifdef DEBUG
! 61: #define IDB_BUSDMA 0x1
! 62: #define IDB_IOMMU 0x2
! 63: #define IDB_INFO 0x4
! 64: #define IDB_SYNC 0x8
! 65: #define IDB_XXX 0x10
! 66: #define IDB_PRINT_MAP 0x20
! 67: #define IDB_BREAK 0x40
! 68: int iommudebug = IDB_INFO;
! 69: #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0)
! 70: #else
! 71: #define DPRINTF(l, s)
! 72: #endif
! 73:
! 74: void iommu_enter(struct iommu_state *, struct strbuf_ctl *, vaddr_t, paddr_t,
! 75: int);
! 76: void iommu_remove(struct iommu_state *, struct strbuf_ctl *, vaddr_t);
! 77: int iommu_dvmamap_sync_range(struct strbuf_ctl*, vaddr_t, bus_size_t);
! 78: int iommu_strbuf_flush_done(struct iommu_map_state *);
! 79: int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *,
! 80: bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t);
! 81: int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *,
! 82: bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t);
! 83: int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *,
! 84: bus_dmamap_t);
! 85: void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *,
! 86: bus_dmamap_t);
! 87: int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t,
! 88: bus_size_t, int, bus_size_t);
! 89: int64_t iommu_tsb_entry(struct iommu_state *, vaddr_t);
! 90: void strbuf_reset(struct strbuf_ctl *);
! 91: int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t);
! 92: vaddr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t);
! 93: int iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *,
! 94: vaddr_t, int);
! 95: int iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *);
! 96: struct iommu_map_state *iommu_iomap_create(int);
! 97: void iommu_iomap_destroy(struct iommu_map_state *);
! 98: void iommu_iomap_clear_pages(struct iommu_map_state *);
! 99: void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
! 100: bus_addr_t, bus_size_t, int);
! 101:
! 102: /*
! 103: * Initiate an STC entry flush.
! 104: */
! 105: static inline void
! 106: iommu_strbuf_flush(struct strbuf_ctl *sb, vaddr_t va)
! 107: {
! 108: #ifdef DEBUG
! 109: if (sb->sb_flush == NULL) {
! 110: printf("iommu_strbuf_flush: attempting to flush w/o STC\n");
! 111: return;
! 112: }
! 113: #endif
! 114:
! 115: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
! 116: STRBUFREG(strbuf_pgflush), va);
! 117: }
! 118:
! 119: /*
! 120: * initialise the UltraSPARC IOMMU (SBus or PCI):
! 121: * - allocate and setup the iotsb.
! 122: * - enable the IOMMU
! 123: * - initialise the streaming buffers (if they exist)
! 124: * - create a private DVMA map.
! 125: */
! 126: void
! 127: iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
! 128: {
! 129: psize_t size;
! 130: vaddr_t va;
! 131: paddr_t pa;
! 132: struct vm_page *m;
! 133: struct pglist mlist;
! 134:
! 135: /*
! 136: * Setup the iommu.
! 137: *
! 138: * The sun4u iommu is part of the SBus or PCI controller so we will
! 139: * deal with it here..
! 140: *
! 141: * For sysio and psycho/psycho+ the IOMMU address space always ends at
! 142: * 0xffffe000, but the starting address depends on the size of the
! 143: * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
! 144: * entry is 8 bytes. The start of the map can be calculated by
! 145: * (0xffffe000 << (8 + is->is_tsbsize)).
! 146: *
! 147: * But sabre and hummingbird use a different scheme that seems to
! 148: * be hard-wired, so we read the start and size from the PROM and
! 149: * just use those values.
! 150: */
! 151: is->is_cr = IOMMUCR_EN;
! 152: is->is_tsbsize = tsbsize;
! 153: if (iovabase == (u_int32_t)-1) {
! 154: is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
! 155: is->is_dvmaend = IOTSB_VEND;
! 156: } else {
! 157: is->is_dvmabase = iovabase;
! 158: is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
! 159: }
! 160:
! 161: /*
! 162: * Allocate memory for I/O pagetables. They need to be physically
! 163: * contiguous.
! 164: */
! 165:
! 166: size = PAGE_SIZE << is->is_tsbsize;
! 167: TAILQ_INIT(&mlist);
! 168: if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
! 169: (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, 0) != 0)
! 170: panic("iommu_init: no memory");
! 171:
! 172: va = uvm_km_valloc(kernel_map, size);
! 173: if (va == 0)
! 174: panic("iommu_init: no memory");
! 175: is->is_tsb = (int64_t *)va;
! 176:
! 177: m = TAILQ_FIRST(&mlist);
! 178: is->is_ptsb = VM_PAGE_TO_PHYS(m);
! 179:
! 180: /* Map the pages */
! 181: for (; m != NULL; m = TAILQ_NEXT(m,pageq)) {
! 182: pa = VM_PAGE_TO_PHYS(m);
! 183: pmap_enter(pmap_kernel(), va, pa | PMAP_NVC,
! 184: VM_PROT_READ|VM_PROT_WRITE,
! 185: VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
! 186: va += PAGE_SIZE;
! 187: }
! 188: pmap_update(pmap_kernel());
! 189: memset(is->is_tsb, 0, size);
! 190:
! 191: #ifdef DEBUG
! 192: if (iommudebug & IDB_INFO) {
! 193: /* Probe the iommu */
! 194: /* The address or contents of the regs...? */
! 195: printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
! 196: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
! 197: IOMMUREG(iommu_cr),
! 198: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
! 199: IOMMUREG(iommu_tsb),
! 200: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
! 201: IOMMUREG(iommu_flush));
! 202: printf("iommu cr=%llx tsb=%llx\n",
! 203: IOMMUREG_READ(is, iommu_cr),
! 204: IOMMUREG_READ(is, iommu_tsb));
! 205: printf("TSB base %p phys %llx\n",
! 206: (void *)is->is_tsb, (unsigned long long)is->is_ptsb);
! 207: delay(1000000); /* 1 s */
! 208: }
! 209: #endif
! 210:
! 211: /*
! 212: * Now all the hardware's working we need to allocate a dvma map.
! 213: */
! 214: printf("dvma map %x-%x, ", is->is_dvmabase, is->is_dvmaend);
! 215: printf("iotdb %llx-%llx",
! 216: (unsigned long long)is->is_ptsb,
! 217: (unsigned long long)(is->is_ptsb + size));
! 218: is->is_dvmamap = extent_create(name,
! 219: is->is_dvmabase, (u_long)is->is_dvmaend + 1,
! 220: M_DEVBUF, 0, 0, EX_NOWAIT);
! 221:
! 222: /*
! 223: * Set the TSB size. The relevant bits were moved to the TSB
! 224: * base register in the PCIe host bridges.
! 225: */
! 226: if (strncmp(name, "pyro", 4) == 0)
! 227: is->is_ptsb |= is->is_tsbsize;
! 228: else
! 229: is->is_cr |= (is->is_tsbsize << 16);
! 230:
! 231: /*
! 232: * Now actually start up the IOMMU.
! 233: */
! 234: iommu_reset(is);
! 235: printf("\n");
! 236: }
! 237:
! 238: /*
! 239: * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have
! 240: * detected that already and disabled them. If not, we will notice that
! 241: * they aren't there when the STRBUF_EN bit does not remain.
! 242: */
! 243: void
! 244: iommu_reset(struct iommu_state *is)
! 245: {
! 246: int i;
! 247:
! 248: IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
! 249:
! 250: /* Enable IOMMU */
! 251: IOMMUREG_WRITE(is, iommu_cr, is->is_cr);
! 252:
! 253: for (i = 0; i < 2; ++i) {
! 254: struct strbuf_ctl *sb = is->is_sb[i];
! 255:
! 256: if (sb == NULL)
! 257: continue;
! 258:
! 259: sb->sb_iommu = is;
! 260: strbuf_reset(sb);
! 261:
! 262: if (sb->sb_flush)
! 263: printf(", STC%d enabled", i);
! 264: }
! 265: }
! 266:
! 267: /*
! 268: * Initialize one STC.
! 269: */
! 270: void
! 271: strbuf_reset(struct strbuf_ctl *sb)
! 272: {
! 273: if(sb->sb_flush == NULL)
! 274: return;
! 275:
! 276: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
! 277: STRBUFREG(strbuf_ctl), STRBUF_EN);
! 278:
! 279: membar(Lookaside);
! 280:
! 281: /* No streaming buffers? Disable them */
! 282: if (bus_space_read_8(sb->sb_bustag, sb->sb_sb,
! 283: STRBUFREG(strbuf_ctl)) == 0) {
! 284: sb->sb_flush = NULL;
! 285: } else {
! 286: /*
! 287: * locate the pa of the flush buffer
! 288: */
! 289: if (pmap_extract(pmap_kernel(),
! 290: (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE)
! 291: sb->sb_flush = NULL;
! 292: }
! 293: }
! 294:
! 295: /*
! 296: * Add an entry to the IOMMU table.
! 297: *
! 298: * The entry is marked streaming if an STC was detected and
! 299: * the BUS_DMA_STREAMING flag is set.
! 300: */
! 301: void
! 302: iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va,
! 303: paddr_t pa, int flags)
! 304: {
! 305: int64_t tte;
! 306: volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)];
! 307:
! 308: #ifdef DIAGNOSTIC
! 309: if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
! 310: panic("iommu_enter: va %#lx not in DVMA space", va);
! 311:
! 312: tte = *tte_ptr;
! 313:
! 314: if (tte & IOTTE_V) {
! 315: printf("Overwriting valid tte entry (dva %lx pa %lx "
! 316: "&tte %p tte %llx)\n", va, pa, tte_ptr, tte);
! 317: extent_print(is->is_dvmamap);
! 318: panic("IOMMU overwrite");
! 319: }
! 320: #endif
! 321:
! 322: tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
! 323: !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING));
! 324:
! 325: DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
! 326: (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));
! 327:
! 328: *tte_ptr = tte;
! 329:
! 330: /*
! 331: * Why bother to flush this va? It should only be relevant for
! 332: * V ==> V or V ==> non-V transitions. The former is illegal and
! 333: * the latter is never done here. It is true that this provides
! 334: * some protection against a misbehaving master using an address
! 335: * after it should. The IOMMU documentations specifically warns
! 336: * that the consequences of a simultaneous IOMMU flush and DVMA
! 337: * access to the same address are undefined. (By that argument,
! 338: * the STC should probably be flushed as well.) Note that if
! 339: * a bus master keeps using a memory region after it has been
! 340: * unmapped, the specific behavior of the IOMMU is likely to
! 341: * be the least of our worries.
! 342: */
! 343: IOMMUREG_WRITE(is, iommu_flush, va);
! 344:
! 345: DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
! 346: va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
! 347: (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
! 348: (u_long)tte));
! 349: }
! 350:
! 351: /*
! 352: * Remove an entry from the IOMMU table.
! 353: *
! 354: * The entry is flushed from the STC if an STC is detected and the TSB
! 355: * entry has the IOTTE_STREAM flags set. It should be impossible for
! 356: * the TSB entry to have this flag set without the BUS_DMA_STREAMING
! 357: * flag, but better to be safe. (The IOMMU will be ignored as long
! 358: * as an STC entry exists.)
! 359: */
! 360: void
! 361: iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va)
! 362: {
! 363: int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
! 364: int64_t tte;
! 365:
! 366: #ifdef DIAGNOSTIC
! 367: if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
! 368: panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
! 369: if (va != trunc_page(va)) {
! 370: printf("iommu_remove: unaligned va: %lx\n", va);
! 371: va = trunc_page(va);
! 372: }
! 373: #endif
! 374: tte = *tte_ptr;
! 375:
! 376: DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n",
! 377: va, tte, tte_ptr));
! 378:
! 379: #ifdef DIAGNOSTIC
! 380: if ((tte & IOTTE_V) == 0) {
! 381: printf("Removing invalid tte entry (dva %lx &tte %p "
! 382: "tte %llx)\n", va, tte_ptr, tte);
! 383: extent_print(is->is_dvmamap);
! 384: panic("IOMMU remove overwrite");
! 385: }
! 386: #endif
! 387:
! 388: *tte_ptr = tte & ~IOTTE_V;
! 389:
! 390: /*
! 391: * IO operations are strongly ordered WRT each other. It is
! 392: * unclear how they relate to normal memory accesses.
! 393: */
! 394: membar(StoreStore);
! 395:
! 396: IOMMUREG_WRITE(is, iommu_flush, va);
! 397:
! 398: if (sb && (tte & IOTTE_STREAM))
! 399: iommu_strbuf_flush(sb, va);
! 400:
! 401: /* Should we sync the iommu and stc here? */
! 402: }
! 403:
! 404: /*
! 405: * Find the physical address of a DVMA address (debug routine).
! 406: */
! 407: paddr_t
! 408: iommu_extract(struct iommu_state *is, vaddr_t dva)
! 409: {
! 410: int64_t tte = 0;
! 411:
! 412: if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
! 413: tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
! 414:
! 415: return (tte & IOTTE_PAMASK);
! 416: }
! 417:
! 418: /*
! 419: * Lookup a TSB entry for a given DVMA (debug routine).
! 420: */
! 421: int64_t
! 422: iommu_lookup_tte(struct iommu_state *is, vaddr_t dva)
! 423: {
! 424: int64_t tte = 0;
! 425:
! 426: if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
! 427: tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
! 428:
! 429: return (tte);
! 430: }
! 431:
! 432: /*
! 433: * Lookup a TSB entry at a given physical address (debug routine).
! 434: */
! 435: int64_t
! 436: iommu_fetch_tte(struct iommu_state *is, paddr_t pa)
! 437: {
! 438: int64_t tte = 0;
! 439:
! 440: if (pa >= is->is_ptsb && pa < is->is_ptsb +
! 441: (PAGE_SIZE << is->is_tsbsize))
! 442: tte = ldxa(pa, ASI_PHYS_CACHED);
! 443:
! 444: return (tte);
! 445: }
! 446:
! 447: /*
! 448: * Fetch a TSB entry with some sanity checking.
! 449: */
! 450: int64_t
! 451: iommu_tsb_entry(struct iommu_state *is, vaddr_t dva)
! 452: {
! 453: int64_t tte;
! 454:
! 455: if (dva < is->is_dvmabase || dva > is->is_dvmaend)
! 456: panic("invalid dva: %llx", (long long)dva);
! 457:
! 458: tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)];
! 459:
! 460: if ((tte & IOTTE_V) == 0)
! 461: panic("iommu_tsb_entry: invalid entry %lx", dva);
! 462:
! 463: return (tte);
! 464: }
! 465:
! 466: /*
! 467: * Initiate and then block until an STC flush synchronization has completed.
! 468: */
! 469: int
! 470: iommu_strbuf_flush_done(struct iommu_map_state *ims)
! 471: {
! 472: struct strbuf_ctl *sb = ims->ims_sb;
! 473: struct strbuf_flush *sf = &ims->ims_flush;
! 474: struct timeval cur, flushtimeout;
! 475: struct timeval to = { 0, 500000 };
! 476: u_int64_t flush;
! 477: int timeout_started = 0;
! 478:
! 479: #ifdef DIAGNOSTIC
! 480: if (sb == NULL) {
! 481: panic("iommu_strbuf_flush_done: invalid flush buffer");
! 482: }
! 483: #endif
! 484:
! 485: /*
! 486: * Streaming buffer flushes:
! 487: *
! 488: * 1 Tell strbuf to flush by storing va to strbuf_pgflush.
! 489: * 2 Store 0 in flag
! 490: * 3 Store pointer to flag in flushsync
! 491: * 4 wait till flushsync becomes 0x1
! 492: *
! 493: * If it takes more than .5 sec, something went very, very wrong.
! 494: */
! 495:
! 496: /*
! 497: * If we're reading from ASI_PHYS_CACHED, then we'll write to
! 498: * it too. No need to tempt fate or learn about Si bugs or such.
! 499: * FreeBSD just uses normal "volatile" reads/writes...
! 500: */
! 501:
! 502: stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0);
! 503:
! 504: /*
! 505: * Insure any previous strbuf operations are complete and that
! 506: * memory is initialized before the IOMMU uses it.
! 507: * Is this Needed? How are IO and memory operations ordered?
! 508: */
! 509: membar(StoreStore);
! 510:
! 511: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
! 512: STRBUFREG(strbuf_flushsync), sf->sbf_flushpa);
! 513:
! 514: DPRINTF(IDB_IOMMU,
! 515: ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n",
! 516: ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa));
! 517:
! 518: membar(StoreLoad | Lookaside);
! 519:
! 520: for(;;) {
! 521: int i;
! 522:
! 523: /*
! 524: * Try to shave a few instruction cycles off the average
! 525: * latency by only checking the elapsed time every few
! 526: * fetches.
! 527: */
! 528: for (i = 0; i < 1000; ++i) {
! 529: membar(LoadLoad);
! 530: /* Bypass non-coherent D$ */
! 531: /* non-coherent...? Huh? */
! 532: flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED);
! 533:
! 534: if (flush) {
! 535: DPRINTF(IDB_IOMMU,
! 536: ("iommu_strbuf_flush_done: flushed\n"));
! 537: return (0);
! 538: }
! 539: }
! 540:
! 541: microtime(&cur);
! 542:
! 543: if (timeout_started) {
! 544: if (timercmp(&cur, &flushtimeout, >))
! 545: panic("STC timeout at %lx (%lld)",
! 546: sf->sbf_flushpa, flush);
! 547: } else {
! 548: timeradd(&cur, &to, &flushtimeout);
! 549:
! 550: timeout_started = 1;
! 551:
! 552: DPRINTF(IDB_IOMMU,
! 553: ("iommu_strbuf_flush_done: flush = %llx pa = %lx "
! 554: "now=%lx:%lx until = %lx:%lx\n",
! 555: ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED),
! 556: sf->sbf_flushpa, cur.tv_sec, cur.tv_usec,
! 557: flushtimeout.tv_sec, flushtimeout.tv_usec));
! 558: }
! 559: }
! 560: }
! 561:
! 562: /*
! 563: * IOMMU DVMA operations, common to SBus and PCI.
! 564: */
! 565:
! 566: #define BUS_DMA_FIND_PARENT(t, fn) \
! 567: if (t->_parent == NULL) \
! 568: panic("null bus_dma parent (" #fn ")"); \
! 569: for (t = t->_parent; t->fn == NULL; t = t->_parent) \
! 570: if (t->_parent == NULL) \
! 571: panic("no bus_dma " #fn " located");
! 572:
! 573: int
! 574: iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb,
! 575: bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary,
! 576: int flags, bus_dmamap_t *dmamap)
! 577: {
! 578: int ret;
! 579: bus_dmamap_t map;
! 580: struct iommu_map_state *ims;
! 581:
! 582: BUS_DMA_FIND_PARENT(t, _dmamap_create);
! 583: ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary,
! 584: flags, &map);
! 585:
! 586: if (ret)
! 587: return (ret);
! 588:
! 589: ims = iommu_iomap_create(atop(round_page(size)));
! 590:
! 591: if (ims == NULL) {
! 592: bus_dmamap_destroy(t0, map);
! 593: return (ENOMEM);
! 594: }
! 595:
! 596: ims->ims_sb = sb;
! 597: map->_dm_cookie = ims;
! 598:
! 599: #ifdef DIAGNOSTIC
! 600: if (ims->ims_sb == NULL)
! 601: panic("iommu_dvmamap_create: null sb");
! 602: if (ims->ims_sb->sb_iommu == NULL)
! 603: panic("iommu_dvmamap_create: null iommu");
! 604: #endif
! 605: *dmamap = map;
! 606:
! 607: return (0);
! 608: }
! 609:
! 610: void
! 611: iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
! 612: {
! 613: /*
! 614: * The specification (man page) requires a loaded
! 615: * map to be unloaded before it is destroyed.
! 616: */
! 617: if (map->dm_nsegs)
! 618: bus_dmamap_unload(t0, map);
! 619:
! 620: if (map->_dm_cookie)
! 621: iommu_iomap_destroy(map->_dm_cookie);
! 622: map->_dm_cookie = NULL;
! 623:
! 624: BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
! 625: (*t->_dmamap_destroy)(t, t0, map);
! 626: }
! 627:
! 628: /*
! 629: * Load a contiguous kva buffer into a dmamap. The physical pages are
! 630: * not assumed to be contiguous. Two passes are made through the buffer
! 631: * and both call pmap_extract() for the same va->pa translations. It
! 632: * is possible to run out of pa->dvma mappings; the code should be smart
! 633: * enough to resize the iomap (when the "flags" permit allocation). It
! 634: * is trivial to compute the number of entries required (round the length
! 635: * up to the page size and then divide by the page size)...
! 636: */
! 637: int
! 638: iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
! 639: void *buf, bus_size_t buflen, struct proc *p, int flags)
! 640: {
! 641: int s;
! 642: int err = 0;
! 643: bus_size_t sgsize;
! 644: u_long dvmaddr, sgstart, sgend;
! 645: bus_size_t align, boundary;
! 646: struct iommu_state *is;
! 647: struct iommu_map_state *ims = map->_dm_cookie;
! 648: pmap_t pmap;
! 649:
! 650: #ifdef DIAGNOSTIC
! 651: if (ims == NULL)
! 652: panic("iommu_dvmamap_load: null map state");
! 653: #endif
! 654: #ifdef DEBUG
! 655: if (ims->ims_sb == NULL)
! 656: panic("iommu_dvmamap_load: null sb");
! 657: if (ims->ims_sb->sb_iommu == NULL)
! 658: panic("iommu_dvmamap_load: null iommu");
! 659: #endif /* DEBUG */
! 660: is = ims->ims_sb->sb_iommu;
! 661:
! 662: if (map->dm_nsegs) {
! 663: /*
! 664: * Is it still in use? _bus_dmamap_load should have taken care
! 665: * of this.
! 666: */
! 667: #ifdef DIAGNOSTIC
! 668: panic("iommu_dvmamap_load: map still in use");
! 669: #endif
! 670: bus_dmamap_unload(t0, map);
! 671: }
! 672:
! 673: /*
! 674: * Make sure that on error condition we return "no valid mappings".
! 675: */
! 676: map->dm_nsegs = 0;
! 677:
! 678: if (buflen < 1 || buflen > map->_dm_size) {
! 679: DPRINTF(IDB_BUSDMA,
! 680: ("iommu_dvmamap_load(): error %d > %d -- "
! 681: "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
! 682: return (EINVAL);
! 683: }
! 684:
! 685: /*
! 686: * A boundary presented to bus_dmamem_alloc() takes precedence
! 687: * over boundary in the map.
! 688: */
! 689: if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
! 690: boundary = map->_dm_boundary;
! 691: align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE);
! 692:
! 693: pmap = p ? p->p_vmspace->vm_map.pmap : pmap = pmap_kernel();
! 694:
! 695: /* Count up the total number of pages we need */
! 696: iommu_iomap_clear_pages(ims);
! 697: { /* Scope */
! 698: bus_addr_t a, aend;
! 699: bus_addr_t addr = (vaddr_t)buf;
! 700: int seg_len = buflen;
! 701:
! 702: aend = round_page(addr + seg_len);
! 703: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
! 704: paddr_t pa;
! 705:
! 706: if (pmap_extract(pmap, a, &pa) == FALSE) {
! 707: printf("iomap pmap error addr 0x%llx\n", a);
! 708: iommu_iomap_clear_pages(ims);
! 709: return (EFBIG);
! 710: }
! 711:
! 712: err = iommu_iomap_insert_page(ims, pa);
! 713: if (err) {
! 714: printf("iomap insert error: %d for "
! 715: "va 0x%llx pa 0x%lx "
! 716: "(buf %p len %lld/%llx)\n",
! 717: err, a, pa, buf, buflen, buflen);
! 718: iommu_dvmamap_print_map(t, is, map);
! 719: iommu_iomap_clear_pages(ims);
! 720: return (EFBIG);
! 721: }
! 722: }
! 723: }
! 724: sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;
! 725:
! 726: if (flags & BUS_DMA_24BIT) {
! 727: sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
! 728: sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
! 729: } else {
! 730: sgstart = is->is_dvmamap->ex_start;
! 731: sgend = is->is_dvmamap->ex_end;
! 732: }
! 733:
! 734: /*
! 735: * If our segment size is larger than the boundary we need to
! 736: * split the transfer up into little pieces ourselves.
! 737: */
! 738: s = splhigh();
! 739: err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend,
! 740: sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
! 741: EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
! 742: splx(s);
! 743:
! 744: #ifdef DEBUG
! 745: if (err || (dvmaddr == (bus_addr_t)-1)) {
! 746: printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
! 747: (int)sgsize, flags);
! 748: #ifdef DDB
! 749: if (iommudebug & IDB_BREAK)
! 750: Debugger();
! 751: #endif
! 752: }
! 753: #endif
! 754: if (err != 0)
! 755: return (err);
! 756:
! 757: if (dvmaddr == (bus_addr_t)-1)
! 758: return (ENOMEM);
! 759:
! 760: /* Set the active DVMA map */
! 761: map->_dm_dvmastart = dvmaddr;
! 762: map->_dm_dvmasize = sgsize;
! 763:
! 764: map->dm_mapsize = buflen;
! 765:
! 766: #ifdef DEBUG
! 767: iommu_dvmamap_validate_map(t, is, map);
! 768: #endif
! 769:
! 770: if (iommu_iomap_load_map(is, ims, dvmaddr, flags))
! 771: return (EFBIG);
! 772:
! 773: { /* Scope */
! 774: bus_addr_t a, aend;
! 775: bus_addr_t addr = (vaddr_t)buf;
! 776: int seg_len = buflen;
! 777:
! 778: aend = round_page(addr + seg_len);
! 779: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
! 780: bus_addr_t pgstart;
! 781: bus_addr_t pgend;
! 782: paddr_t pa;
! 783: int pglen;
! 784:
! 785: /* Yuck... Redoing the same pmap_extract... */
! 786: if (pmap_extract(pmap, a, &pa) == FALSE) {
! 787: printf("iomap pmap error addr 0x%llx\n", a);
! 788: iommu_iomap_clear_pages(ims);
! 789: return (EFBIG);
! 790: }
! 791:
! 792: pgstart = pa | (MAX(a, addr) & PAGE_MASK);
! 793: pgend = pa | (MIN(a + PAGE_SIZE - 1,
! 794: addr + seg_len - 1) & PAGE_MASK);
! 795: pglen = pgend - pgstart + 1;
! 796:
! 797: if (pglen < 1)
! 798: continue;
! 799:
! 800: err = iommu_dvmamap_append_range(t, map, pgstart,
! 801: pglen, flags, boundary);
! 802: if (err == EFBIG)
! 803: return (err);
! 804: if (err) {
! 805: printf("iomap load seg page: %d for "
! 806: "va 0x%llx pa %lx (%llx - %llx) "
! 807: "for %d/0x%x\n",
! 808: err, a, pa, pgstart, pgend, pglen, pglen);
! 809: return (err);
! 810: }
! 811: }
! 812: }
! 813:
! 814: #ifdef DIAGNOSTIC
! 815: iommu_dvmamap_validate_map(t, is, map);
! 816: #endif
! 817:
! 818: #ifdef DEBUG
! 819: if (err)
! 820: printf("**** iommu_dvmamap_load failed with error %d\n",
! 821: err);
! 822:
! 823: if (err || (iommudebug & IDB_PRINT_MAP)) {
! 824: iommu_dvmamap_print_map(t, is, map);
! 825: #ifdef DDB
! 826: if (iommudebug & IDB_BREAK)
! 827: Debugger();
! 828: #endif
! 829: }
! 830: #endif
! 831:
! 832: return (err);
! 833: }
! 834:
! 835: /*
! 836: * Load a dvmamap from an array of segs or an mlist (if the first
! 837: * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs()
! 838: * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the
! 839: * mapping. This is ugly. A better solution would probably be to have
! 840: * function pointers for implementing the traversal. That way, there
! 841: * could be one core load routine for each of the three required algorithms
! 842: * (buffer, seg, and mlist). That would also mean that the traversal
! 843: * algorithm would then only need one implementation for each algorithm
! 844: * instead of two (one for populating the iomap and one for populating
! 845: * the dvma map).
! 846: */
! 847: int
! 848: iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
! 849: bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
! 850: {
! 851: int i, s;
! 852: int left;
! 853: int err = 0;
! 854: bus_size_t sgsize;
! 855: bus_size_t boundary, align;
! 856: u_long dvmaddr, sgstart, sgend;
! 857: struct iommu_state *is;
! 858: struct iommu_map_state *ims = map->_dm_cookie;
! 859:
! 860: #ifdef DIAGNOSTIC
! 861: if (ims == NULL)
! 862: panic("iommu_dvmamap_load_raw: null map state");
! 863: #endif
! 864: #ifdef DEBUG
! 865: if (ims->ims_sb == NULL)
! 866: panic("iommu_dvmamap_load_raw: null sb");
! 867: if (ims->ims_sb->sb_iommu == NULL)
! 868: panic("iommu_dvmamap_load_raw: null iommu");
! 869: #endif /* DEBUG */
! 870: is = ims->ims_sb->sb_iommu;
! 871:
! 872: if (map->dm_nsegs) {
! 873: /* Already in use?? */
! 874: #ifdef DIAGNOSTIC
! 875: panic("iommu_dvmamap_load_raw: map still in use");
! 876: #endif
! 877: bus_dmamap_unload(t0, map);
! 878: }
! 879:
! 880: /*
! 881: * A boundary presented to bus_dmamem_alloc() takes precedence
! 882: * over boundary in the map.
! 883: */
! 884: if ((boundary = segs[0]._ds_boundary) == 0)
! 885: boundary = map->_dm_boundary;
! 886:
! 887: align = MAX(segs[0]._ds_align, PAGE_SIZE);
! 888:
! 889: /*
! 890: * Make sure that on error condition we return "no valid mappings".
! 891: */
! 892: map->dm_nsegs = 0;
! 893:
! 894: iommu_iomap_clear_pages(ims);
! 895: if (segs[0]._ds_mlist) {
! 896: struct pglist *mlist = segs[0]._ds_mlist;
! 897: struct vm_page *m;
! 898: for (m = TAILQ_FIRST(mlist); m != NULL;
! 899: m = TAILQ_NEXT(m,pageq)) {
! 900: err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m));
! 901:
! 902: if(err) {
! 903: printf("iomap insert error: %d for "
! 904: "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m));
! 905: iommu_dvmamap_print_map(t, is, map);
! 906: iommu_iomap_clear_pages(ims);
! 907: return (EFBIG);
! 908: }
! 909: }
! 910: } else {
! 911: /* Count up the total number of pages we need */
! 912: for (i = 0, left = size; left > 0 && i < nsegs; i++) {
! 913: bus_addr_t a, aend;
! 914: bus_size_t len = segs[i].ds_len;
! 915: bus_addr_t addr = segs[i].ds_addr;
! 916: int seg_len = MIN(left, len);
! 917:
! 918: if (len < 1)
! 919: continue;
! 920:
! 921: aend = round_page(addr + seg_len);
! 922: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
! 923:
! 924: err = iommu_iomap_insert_page(ims, a);
! 925: if (err) {
! 926: printf("iomap insert error: %d for "
! 927: "pa 0x%llx\n", err, a);
! 928: iommu_dvmamap_print_map(t, is, map);
! 929: iommu_iomap_clear_pages(ims);
! 930: return (EFBIG);
! 931: }
! 932: }
! 933:
! 934: left -= seg_len;
! 935: }
! 936: }
! 937: sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;
! 938:
! 939: if (flags & BUS_DMA_24BIT) {
! 940: sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
! 941: sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
! 942: } else {
! 943: sgstart = is->is_dvmamap->ex_start;
! 944: sgend = is->is_dvmamap->ex_end;
! 945: }
! 946:
! 947: /*
! 948: * If our segment size is larger than the boundary we need to
! 949: * split the transfer up into little pieces ourselves.
! 950: */
! 951: s = splhigh();
! 952: err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend,
! 953: sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
! 954: EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
! 955: splx(s);
! 956:
! 957: if (err != 0)
! 958: return (err);
! 959:
! 960: #ifdef DEBUG
! 961: if (dvmaddr == (bus_addr_t)-1) {
! 962: printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) "
! 963: "failed!\n", (int)sgsize, flags);
! 964: #ifdef DDB
! 965: if (iommudebug & IDB_BREAK)
! 966: Debugger();
! 967: #else
! 968: panic("");
! 969: #endif
! 970: }
! 971: #endif
! 972: if (dvmaddr == (bus_addr_t)-1)
! 973: return (ENOMEM);
! 974:
! 975: /* Set the active DVMA map */
! 976: map->_dm_dvmastart = dvmaddr;
! 977: map->_dm_dvmasize = sgsize;
! 978:
! 979: map->dm_mapsize = size;
! 980:
! 981: #ifdef DEBUG
! 982: iommu_dvmamap_validate_map(t, is, map);
! 983: #endif
! 984:
! 985: if (iommu_iomap_load_map(is, ims, dvmaddr, flags))
! 986: return (EFBIG);
! 987:
! 988: if (segs[0]._ds_mlist)
! 989: err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist,
! 990: flags, size, boundary);
! 991: else
! 992: err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs,
! 993: flags, size, boundary);
! 994:
! 995: if (err)
! 996: iommu_iomap_unload_map(is, ims);
! 997:
! 998: #ifdef DIAGNOSTIC
! 999: /* The map should be valid even if the load failed */
! 1000: if (iommu_dvmamap_validate_map(t, is, map)) {
! 1001: printf("load size %lld/0x%llx\n", size, size);
! 1002: if (segs[0]._ds_mlist)
! 1003: printf("mlist %p\n", segs[0]._ds_mlist);
! 1004: else {
! 1005: long tot_len = 0;
! 1006: long clip_len = 0;
! 1007: printf("segs %p nsegs %d\n", segs, nsegs);
! 1008:
! 1009: left = size;
! 1010: for(i = 0; i < nsegs; i++) {
! 1011: bus_size_t len = segs[i].ds_len;
! 1012: bus_addr_t addr = segs[i].ds_addr;
! 1013: int seg_len = MIN(left, len);
! 1014:
! 1015: printf("addr %llx len %lld/0x%llx seg_len "
! 1016: "%d/0x%x left %d/0x%x\n", addr, len, len,
! 1017: seg_len, seg_len, left, left);
! 1018:
! 1019: left -= seg_len;
! 1020:
! 1021: clip_len += seg_len;
! 1022: tot_len += segs[i].ds_len;
! 1023: }
! 1024: printf("total length %ld/0x%lx total seg. "
! 1025: "length %ld/0x%lx\n", tot_len, tot_len, clip_len,
! 1026: clip_len);
! 1027: }
! 1028:
! 1029: if (err == 0)
! 1030: err = 1;
! 1031: }
! 1032:
! 1033: #endif
! 1034:
! 1035: #ifdef DEBUG
! 1036: if (err)
! 1037: printf("**** iommu_dvmamap_load_raw failed with error %d\n",
! 1038: err);
! 1039:
! 1040: if (err || (iommudebug & IDB_PRINT_MAP)) {
! 1041: iommu_dvmamap_print_map(t, is, map);
! 1042: #ifdef DDB
! 1043: if (iommudebug & IDB_BREAK)
! 1044: Debugger();
! 1045: #endif
! 1046: }
! 1047: #endif
! 1048:
! 1049: return (err);
! 1050: }
! 1051:
! 1052: /*
! 1053: * Insert a range of addresses into a loaded map respecting the specified
! 1054: * boundary and alignment restrictions. The range is specified by its
! 1055: * physical address and length. The range cannot cross a page boundary.
! 1056: * This code (along with most of the rest of the function in this file)
! 1057: * assumes that the IOMMU page size is equal to PAGE_SIZE.
! 1058: */
! 1059: int
! 1060: iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa,
! 1061: bus_size_t length, int flags, bus_size_t boundary)
! 1062: {
! 1063: struct iommu_map_state *ims = map->_dm_cookie;
! 1064: bus_addr_t sgstart, sgend, bd_mask;
! 1065: bus_dma_segment_t *seg = NULL;
! 1066: int i = map->dm_nsegs;
! 1067:
! 1068: #ifdef DEBUG
! 1069: if (ims == NULL)
! 1070: panic("iommu_dvmamap_append_range: null map state");
! 1071: #endif
! 1072:
! 1073: sgstart = iommu_iomap_translate(ims, pa);
! 1074: sgend = sgstart + length - 1;
! 1075:
! 1076: #ifdef DIAGNOSTIC
! 1077: if (sgstart == NULL || sgstart > sgend) {
! 1078: printf("append range invalid mapping for %lx "
! 1079: "(0x%llx - 0x%llx)\n", pa, sgstart, sgend);
! 1080: map->dm_nsegs = 0;
! 1081: return (EINVAL);
! 1082: }
! 1083: #endif
! 1084:
! 1085: #ifdef DEBUG
! 1086: if (trunc_page(sgstart) != trunc_page(sgend)) {
! 1087: printf("append range crossing page boundary! "
! 1088: "pa %lx length %lld/0x%llx sgstart %llx sgend %llx\n",
! 1089: pa, length, length, sgstart, sgend);
! 1090: }
! 1091: #endif
! 1092:
! 1093: /*
! 1094: * We will attempt to merge this range with the previous entry
! 1095: * (if there is one).
! 1096: */
! 1097: if (i > 0) {
! 1098: seg = &map->dm_segs[i - 1];
! 1099: if (sgstart == seg->ds_addr + seg->ds_len) {
! 1100: length += seg->ds_len;
! 1101: sgstart = seg->ds_addr;
! 1102: sgend = sgstart + length - 1;
! 1103: } else
! 1104: seg = NULL;
! 1105: }
! 1106:
! 1107: if (seg == NULL) {
! 1108: seg = &map->dm_segs[i];
! 1109: if (++i > map->_dm_segcnt) {
! 1110: map->dm_nsegs = 0;
! 1111: return (EFBIG);
! 1112: }
! 1113: }
! 1114:
! 1115: /*
! 1116: * At this point, "i" is the index of the *next* bus_dma_segment_t
! 1117: * (the segment count, aka map->dm_nsegs) and "seg" points to the
! 1118: * *current* entry. "length", "sgstart", and "sgend" reflect what
! 1119: * we intend to put in "*seg". No assumptions should be made about
! 1120: * the contents of "*seg". Only "boundary" issue can change this
! 1121: * and "boundary" is often zero, so explicitly test for that case
! 1122: * (the test is strictly an optimization).
! 1123: */
! 1124: if (boundary != 0) {
! 1125: bd_mask = ~(boundary - 1);
! 1126:
! 1127: while ((sgstart & bd_mask) != (sgend & bd_mask)) {
! 1128: /*
! 1129: * We are crossing a boundary so fill in the current
! 1130: * segment with as much as possible, then grab a new
! 1131: * one.
! 1132: */
! 1133:
! 1134: seg->ds_addr = sgstart;
! 1135: seg->ds_len = boundary - (sgstart & bd_mask);
! 1136:
! 1137: sgstart += seg->ds_len; /* sgend stays the same */
! 1138: length -= seg->ds_len;
! 1139:
! 1140: seg = &map->dm_segs[i];
! 1141: if (++i > map->_dm_segcnt) {
! 1142: map->dm_nsegs = 0;
! 1143: return (EFBIG);
! 1144: }
! 1145: }
! 1146: }
! 1147:
! 1148: seg->ds_addr = sgstart;
! 1149: seg->ds_len = length;
! 1150: map->dm_nsegs = i;
! 1151:
! 1152: return (0);
! 1153: }
! 1154:
! 1155: /*
! 1156: * Populate the iomap from a bus_dma_segment_t array. See note for
! 1157: * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap.
! 1158: * This is less of a problem for load_seg, as the number of pages
! 1159: * is usually similar to the number of segments (nsegs).
! 1160: */
! 1161: int
! 1162: iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is,
! 1163: bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags,
! 1164: bus_size_t size, bus_size_t boundary)
! 1165: {
! 1166: int i;
! 1167: int left;
! 1168: int seg;
! 1169:
! 1170: /*
! 1171: * This segs is made up of individual physical
! 1172: * segments, probably by _bus_dmamap_load_uio() or
! 1173: * _bus_dmamap_load_mbuf(). Ignore the mlist and
! 1174: * load each one individually.
! 1175: */
! 1176:
! 1177: /*
! 1178: * Keep in mind that each segment could span
! 1179: * multiple pages and that these are not always
! 1180: * adjacent. The code is no longer adding dvma
! 1181: * aliases to the IOMMU. The STC will not cross
! 1182: * page boundaries anyway and a IOMMU table walk
! 1183: * vs. what may be a streamed PCI DMA to a ring
! 1184: * descriptor is probably a wash. It eases TLB
! 1185: * pressure and in the worst possible case, it is
! 1186: * only as bad a non-IOMMUed architecture. More
! 1187: * importantly, the code is not quite as hairy.
! 1188: * (It's bad enough as it is.)
! 1189: */
! 1190: left = size;
! 1191: seg = 0;
! 1192: for (i = 0; left > 0 && i < nsegs; i++) {
! 1193: bus_addr_t a, aend;
! 1194: bus_size_t len = segs[i].ds_len;
! 1195: bus_addr_t addr = segs[i].ds_addr;
! 1196: int seg_len = MIN(left, len);
! 1197:
! 1198: if (len < 1)
! 1199: continue;
! 1200:
! 1201: aend = round_page(addr + seg_len);
! 1202: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
! 1203: bus_addr_t pgstart;
! 1204: bus_addr_t pgend;
! 1205: int pglen;
! 1206: int err;
! 1207:
! 1208: pgstart = MAX(a, addr);
! 1209: pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1);
! 1210: pglen = pgend - pgstart + 1;
! 1211:
! 1212: if (pglen < 1)
! 1213: continue;
! 1214:
! 1215: err = iommu_dvmamap_append_range(t, map, pgstart,
! 1216: pglen, flags, boundary);
! 1217: if (err == EFBIG)
! 1218: return (err);
! 1219: if (err) {
! 1220: printf("iomap load seg page: %d for "
! 1221: "pa 0x%llx (%llx - %llx for %d/%x\n",
! 1222: err, a, pgstart, pgend, pglen, pglen);
! 1223: return (err);
! 1224: }
! 1225:
! 1226: }
! 1227:
! 1228: left -= seg_len;
! 1229: }
! 1230: return (0);
! 1231: }
! 1232:
! 1233: /*
! 1234: * Populate the iomap from an mlist. See note for iommu_dvmamap_load()
! 1235: * regarding page entry exhaustion of the iomap.
! 1236: */
! 1237: int
! 1238: iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is,
! 1239: bus_dmamap_t map, struct pglist *mlist, int flags,
! 1240: bus_size_t size, bus_size_t boundary)
! 1241: {
! 1242: struct vm_page *m;
! 1243: paddr_t pa;
! 1244: int err;
! 1245:
! 1246: /*
! 1247: * This was allocated with bus_dmamem_alloc.
! 1248: * The pages are on an `mlist'.
! 1249: */
! 1250: for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) {
! 1251: pa = VM_PAGE_TO_PHYS(m);
! 1252:
! 1253: err = iommu_dvmamap_append_range(t, map, pa, PAGE_SIZE,
! 1254: flags, boundary);
! 1255: if (err == EFBIG)
! 1256: return (err);
! 1257: if (err) {
! 1258: printf("iomap load seg page: %d for pa 0x%lx "
! 1259: "(%lx - %lx for %d/%x\n", err, pa, pa,
! 1260: pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
! 1261: return (err);
! 1262: }
! 1263: }
! 1264:
! 1265: return (0);
! 1266: }
! 1267:
! 1268: /*
! 1269: * Unload a dvmamap.
! 1270: */
! 1271: void
! 1272: iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
! 1273: {
! 1274: struct iommu_state *is;
! 1275: struct iommu_map_state *ims = map->_dm_cookie;
! 1276: bus_addr_t dvmaddr = map->_dm_dvmastart;
! 1277: bus_size_t sgsize = map->_dm_dvmasize;
! 1278: int error, s;
! 1279:
! 1280: #ifdef DEBUG
! 1281: if (ims == NULL)
! 1282: panic("iommu_dvmamap_unload: null map state");
! 1283: if (ims->ims_sb == NULL)
! 1284: panic("iommu_dvmamap_unload: null sb");
! 1285: if (ims->ims_sb->sb_iommu == NULL)
! 1286: panic("iommu_dvmamap_unload: null iommu");
! 1287: #endif /* DEBUG */
! 1288:
! 1289: is = ims->ims_sb->sb_iommu;
! 1290:
! 1291: /* Flush the iommu */
! 1292: #ifdef DEBUG
! 1293: if (dvmaddr == 0) {
! 1294: printf("iommu_dvmamap_unload: No dvmastart\n");
! 1295: #ifdef DDB
! 1296: if (iommudebug & IDB_BREAK)
! 1297: Debugger();
! 1298: #endif
! 1299: return;
! 1300: }
! 1301: iommu_dvmamap_validate_map(t, is, map);
! 1302:
! 1303: if (iommudebug & IDB_PRINT_MAP)
! 1304: iommu_dvmamap_print_map(t, is, map);
! 1305: #endif /* DEBUG */
! 1306:
! 1307: /* Remove the IOMMU entries */
! 1308: iommu_iomap_unload_map(is, ims);
! 1309:
! 1310: /* Clear the iomap */
! 1311: iommu_iomap_clear_pages(ims);
! 1312:
! 1313: bus_dmamap_unload(t->_parent, map);
! 1314:
! 1315: /* Mark the mappings as invalid. */
! 1316: map->dm_mapsize = 0;
! 1317: map->dm_nsegs = 0;
! 1318:
! 1319: s = splhigh();
! 1320: error = extent_free(is->is_dvmamap, dvmaddr,
! 1321: sgsize, EX_NOWAIT);
! 1322: map->_dm_dvmastart = 0;
! 1323: map->_dm_dvmasize = 0;
! 1324: splx(s);
! 1325: if (error != 0)
! 1326: printf("warning: %qd of DVMA space lost\n", sgsize);
! 1327: }
! 1328:
! 1329: /*
! 1330: * Perform internal consistency checking on a dvmamap.
! 1331: */
! 1332: int
! 1333: iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is,
! 1334: bus_dmamap_t map)
! 1335: {
! 1336: int err = 0;
! 1337: int seg;
! 1338:
! 1339: if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) {
! 1340: printf("**** dvmastart address not page aligned: %llx",
! 1341: map->_dm_dvmastart);
! 1342: err = 1;
! 1343: }
! 1344: if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) {
! 1345: printf("**** dvmasize not a multiple of page size: %llx",
! 1346: map->_dm_dvmasize);
! 1347: err = 1;
! 1348: }
! 1349: if (map->_dm_dvmastart < is->is_dvmabase ||
! 1350: (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) >
! 1351: is->is_dvmaend) {
! 1352: printf("dvmaddr %llx len %llx out of range %x - %x\n",
! 1353: map->_dm_dvmastart, map->_dm_dvmasize,
! 1354: is->is_dvmabase, is->is_dvmaend);
! 1355: err = 1;
! 1356: }
! 1357: for (seg = 0; seg < map->dm_nsegs; seg++) {
! 1358: if (map->dm_segs[seg].ds_addr == 0 ||
! 1359: map->dm_segs[seg].ds_len == 0) {
! 1360: printf("seg %d null segment dvmaddr %llx len %llx for "
! 1361: "range %llx len %llx\n",
! 1362: seg,
! 1363: map->dm_segs[seg].ds_addr,
! 1364: map->dm_segs[seg].ds_len,
! 1365: map->_dm_dvmastart, map->_dm_dvmasize);
! 1366: err = 1;
! 1367: } else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart ||
! 1368: round_page(map->dm_segs[seg].ds_addr +
! 1369: map->dm_segs[seg].ds_len) >
! 1370: map->_dm_dvmastart + map->_dm_dvmasize) {
! 1371: printf("seg %d dvmaddr %llx len %llx out of "
! 1372: "range %llx len %llx\n",
! 1373: seg,
! 1374: map->dm_segs[seg].ds_addr,
! 1375: map->dm_segs[seg].ds_len,
! 1376: map->_dm_dvmastart, map->_dm_dvmasize);
! 1377: err = 1;
! 1378: }
! 1379: }
! 1380:
! 1381: if (err) {
! 1382: iommu_dvmamap_print_map(t, is, map);
! 1383: #if defined(DDB) && defined(DEBUG)
! 1384: if (iommudebug & IDB_BREAK)
! 1385: Debugger();
! 1386: #endif
! 1387: }
! 1388:
! 1389: return (err);
! 1390: }
! 1391:
! 1392: void
! 1393: iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is,
! 1394: bus_dmamap_t map)
! 1395: {
! 1396: int seg, i;
! 1397: long full_len, source_len;
! 1398: struct mbuf *m;
! 1399:
! 1400: printf("DVMA %x for %x, mapping %p: dvstart %llx dvsize %llx "
! 1401: "size %lld/%llx maxsegsz %llx boundary %llx segcnt %d "
! 1402: "flags %x type %d source %p "
! 1403: "cookie %p mapsize %llx nsegs %d\n",
! 1404: is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map,
! 1405: map->_dm_dvmastart, map->_dm_dvmasize,
! 1406: map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary,
! 1407: map->_dm_segcnt, map->_dm_flags, map->_dm_type,
! 1408: map->_dm_source, map->_dm_cookie, map->dm_mapsize,
! 1409: map->dm_nsegs);
! 1410:
! 1411: full_len = 0;
! 1412: for (seg = 0; seg < map->dm_nsegs; seg++) {
! 1413: printf("seg %d dvmaddr %llx pa %lx len %llx (tte %llx)\n",
! 1414: seg, map->dm_segs[seg].ds_addr,
! 1415: is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0,
! 1416: map->dm_segs[seg].ds_len,
! 1417: is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0);
! 1418: full_len += map->dm_segs[seg].ds_len;
! 1419: }
! 1420: printf("total length = %ld/0x%lx\n", full_len, full_len);
! 1421:
! 1422: if (map->_dm_source) switch (map->_dm_type) {
! 1423: case _DM_TYPE_MBUF:
! 1424: m = map->_dm_source;
! 1425: if (m->m_flags & M_PKTHDR)
! 1426: printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n",
! 1427: m, m->m_pkthdr.len, m->m_pkthdr.len);
! 1428: else
! 1429: printf("source mbuf (%p):\n", m);
! 1430:
! 1431: source_len = 0;
! 1432: for ( ; m; m = m->m_next) {
! 1433: vaddr_t vaddr = mtod(m, vaddr_t);
! 1434: long len = m->m_len;
! 1435: paddr_t pa;
! 1436:
! 1437: if (pmap_extract(pmap_kernel(), vaddr, &pa))
! 1438: printf("kva %lx pa %lx len %ld/0x%lx\n",
! 1439: vaddr, pa, len, len);
! 1440: else
! 1441: printf("kva %lx pa <invalid> len %ld/0x%lx\n",
! 1442: vaddr, len, len);
! 1443:
! 1444: source_len += len;
! 1445: }
! 1446:
! 1447: if (full_len != source_len)
! 1448: printf("mbuf length %ld/0x%lx is %s than mapping "
! 1449: "length %ld/0x%lx\n", source_len, source_len,
! 1450: (source_len > full_len) ? "greater" : "less",
! 1451: full_len, full_len);
! 1452: else
! 1453: printf("mbuf length %ld/0x%lx\n", source_len,
! 1454: source_len);
! 1455: break;
! 1456: case _DM_TYPE_LOAD:
! 1457: case _DM_TYPE_SEGS:
! 1458: case _DM_TYPE_UIO:
! 1459: default:
! 1460: break;
! 1461: }
! 1462:
! 1463: if (map->_dm_cookie) {
! 1464: struct iommu_map_state *ims = map->_dm_cookie;
! 1465: struct iommu_page_map *ipm = &ims->ims_map;
! 1466:
! 1467: printf("page map (%p) of size %d with %d entries\n",
! 1468: ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt);
! 1469: for (i = 0; i < ipm->ipm_pagecnt; ++i) {
! 1470: struct iommu_page_entry *e = &ipm->ipm_map[i];
! 1471: printf("%d: vmaddr 0x%lx pa 0x%lx\n", i,
! 1472: e->ipe_va, e->ipe_pa);
! 1473: }
! 1474: } else
! 1475: printf("iommu map state (cookie) is NULL\n");
! 1476: }
! 1477:
! 1478: void
! 1479: _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
! 1480: bus_addr_t offset, bus_size_t len, int ops)
! 1481: {
! 1482: struct iommu_state *is;
! 1483: struct iommu_map_state *ims = map->_dm_cookie;
! 1484: struct strbuf_ctl *sb;
! 1485: bus_size_t count;
! 1486: int i, needsflush = 0;
! 1487:
! 1488: sb = ims->ims_sb;
! 1489: is = sb->sb_iommu;
! 1490:
! 1491: for (i = 0; i < map->dm_nsegs; i++) {
! 1492: if (offset < map->dm_segs[i].ds_len)
! 1493: break;
! 1494: offset -= map->dm_segs[i].ds_len;
! 1495: }
! 1496:
! 1497: if (i == map->dm_nsegs)
! 1498: panic("iommu_dvmamap_sync: too short %llu", offset);
! 1499:
! 1500: for (; len > 0 && i < map->dm_nsegs; i++) {
! 1501: count = MIN(map->dm_segs[i].ds_len - offset, len);
! 1502: if (count > 0 && iommu_dvmamap_sync_range(sb,
! 1503: map->dm_segs[i].ds_addr + offset, count))
! 1504: needsflush = 1;
! 1505: len -= count;
! 1506: }
! 1507:
! 1508: #ifdef DIAGNOSTIC
! 1509: if (i == map->dm_nsegs && len > 0)
! 1510: panic("iommu_dvmamap_sync: leftover %llu", len);
! 1511: #endif
! 1512:
! 1513: if (needsflush)
! 1514: iommu_strbuf_flush_done(ims);
! 1515: }
! 1516:
! 1517: void
! 1518: iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
! 1519: bus_addr_t offset, bus_size_t len, int ops)
! 1520: {
! 1521: struct iommu_map_state *ims = map->_dm_cookie;
! 1522:
! 1523: #ifdef DIAGNOSTIC
! 1524: if (ims == NULL)
! 1525: panic("iommu_dvmamap_sync: null map state");
! 1526: if (ims->ims_sb == NULL)
! 1527: panic("iommu_dvmamap_sync: null sb");
! 1528: if (ims->ims_sb->sb_iommu == NULL)
! 1529: panic("iommu_dvmamap_sync: null iommu");
! 1530: #endif
! 1531: if (len == 0)
! 1532: return;
! 1533:
! 1534: if (ops & BUS_DMASYNC_PREWRITE)
! 1535: membar(MemIssue);
! 1536:
! 1537: if ((ims->ims_flags & IOMMU_MAP_STREAM) &&
! 1538: (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)))
! 1539: _iommu_dvmamap_sync(t, t0, map, offset, len, ops);
! 1540:
! 1541: if (ops & BUS_DMASYNC_POSTREAD)
! 1542: membar(MemIssue);
! 1543: }
! 1544:
! 1545: /*
! 1546: * Flush an individual dma segment, returns non-zero if the streaming buffers
! 1547: * need flushing afterwards.
! 1548: */
! 1549: int
! 1550: iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
! 1551: {
! 1552: vaddr_t vaend;
! 1553: #ifdef DIAGNOSTIC
! 1554: struct iommu_state *is = sb->sb_iommu;
! 1555:
! 1556: if (va < is->is_dvmabase || va > is->is_dvmaend)
! 1557: panic("invalid va: %llx", (long long)va);
! 1558:
! 1559: if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
! 1560: printf("iommu_dvmamap_sync_range: attempting to flush "
! 1561: "non-streaming entry\n");
! 1562: return (0);
! 1563: }
! 1564: #endif
! 1565:
! 1566: vaend = (va + len + PAGE_MASK) & ~PAGE_MASK;
! 1567: va &= ~PAGE_MASK;
! 1568:
! 1569: #ifdef DIAGNOSTIC
! 1570: if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend)
! 1571: panic("invalid va range: %llx to %llx (%x to %x)",
! 1572: (long long)va, (long long)vaend,
! 1573: is->is_dvmabase,
! 1574: is->is_dvmaend);
! 1575: #endif
! 1576:
! 1577: for ( ; va <= vaend; va += PAGE_SIZE) {
! 1578: DPRINTF(IDB_BUSDMA,
! 1579: ("iommu_dvmamap_sync_range: flushing va %p\n",
! 1580: (void *)(u_long)va));
! 1581: iommu_strbuf_flush(sb, va);
! 1582: }
! 1583:
! 1584: return (1);
! 1585: }
! 1586:
! 1587: int
! 1588: iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size,
! 1589: bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs,
! 1590: int nsegs, int *rsegs, int flags)
! 1591: {
! 1592:
! 1593: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx "
! 1594: "bound %llx segp %p flags %d\n", (unsigned long long)size,
! 1595: (unsigned long long)alignment, (unsigned long long)boundary,
! 1596: segs, flags));
! 1597: BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
! 1598: return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
! 1599: segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
! 1600: }
! 1601:
! 1602: void
! 1603: iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs,
! 1604: int nsegs)
! 1605: {
! 1606:
! 1607: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
! 1608: segs, nsegs));
! 1609: BUS_DMA_FIND_PARENT(t, _dmamem_free);
! 1610: (*t->_dmamem_free)(t, t0, segs, nsegs);
! 1611: }
! 1612:
! 1613: /*
! 1614: * Map the DVMA mappings into the kernel pmap.
! 1615: * Check the flags to see whether we're streaming or coherent.
! 1616: */
! 1617: int
! 1618: iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs,
! 1619: int nsegs, size_t size, caddr_t *kvap, int flags)
! 1620: {
! 1621: struct vm_page *m;
! 1622: vaddr_t va;
! 1623: bus_addr_t addr;
! 1624: struct pglist *mlist;
! 1625: bus_addr_t cbit = 0;
! 1626:
! 1627: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
! 1628: segs, nsegs, size));
! 1629:
! 1630: /*
! 1631: * Allocate some space in the kernel map, and then map these pages
! 1632: * into this space.
! 1633: */
! 1634: size = round_page(size);
! 1635: va = uvm_km_valloc(kernel_map, size);
! 1636: if (va == 0)
! 1637: return (ENOMEM);
! 1638:
! 1639: *kvap = (caddr_t)va;
! 1640:
! 1641: /*
! 1642: * digest flags:
! 1643: */
! 1644: #if 0
! 1645: if (flags & BUS_DMA_COHERENT) /* Disable vcache */
! 1646: cbit |= PMAP_NVC;
! 1647: #endif
! 1648: if (flags & BUS_DMA_NOCACHE) /* sideffects */
! 1649: cbit |= PMAP_NC;
! 1650:
! 1651: /*
! 1652: * Now take this and map it into the CPU.
! 1653: */
! 1654: mlist = segs[0]._ds_mlist;
! 1655: TAILQ_FOREACH(m, mlist, pageq) {
! 1656: #ifdef DIAGNOSTIC
! 1657: if (size == 0)
! 1658: panic("iommu_dvmamem_map: size botch");
! 1659: #endif
! 1660: addr = VM_PAGE_TO_PHYS(m);
! 1661: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
! 1662: "mapping va %lx at %llx\n", va,
! 1663: (unsigned long long)addr | cbit));
! 1664: pmap_enter(pmap_kernel(), va, addr | cbit,
! 1665: VM_PROT_READ | VM_PROT_WRITE,
! 1666: VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED);
! 1667: va += PAGE_SIZE;
! 1668: size -= PAGE_SIZE;
! 1669: }
! 1670: pmap_update(pmap_kernel());
! 1671:
! 1672: return (0);
! 1673: }
! 1674:
! 1675: /*
! 1676: * Unmap DVMA mappings from kernel
! 1677: */
! 1678: void
! 1679: iommu_dvmamem_unmap(bus_dma_tag_t t, bus_dma_tag_t t0, caddr_t kva,
! 1680: size_t size)
! 1681: {
! 1682:
! 1683: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
! 1684: kva, size));
! 1685:
! 1686: #ifdef DIAGNOSTIC
! 1687: if ((u_long)kva & PAGE_MASK)
! 1688: panic("iommu_dvmamem_unmap");
! 1689: #endif
! 1690:
! 1691: size = round_page(size);
! 1692: pmap_remove(pmap_kernel(), (vaddr_t)kva, size);
! 1693: pmap_update(pmap_kernel());
! 1694: uvm_km_free(kernel_map, (vaddr_t)kva, size);
! 1695: }
! 1696:
! 1697: /*
! 1698: * Create a new iomap.
! 1699: */
! 1700: struct iommu_map_state *
! 1701: iommu_iomap_create(int n)
! 1702: {
! 1703: struct iommu_map_state *ims;
! 1704: struct strbuf_flush *sbf;
! 1705: vaddr_t va;
! 1706:
! 1707: /* Safety for heavily fragmented data, such as mbufs */
! 1708: n += 4;
! 1709: if (n < 16)
! 1710: n = 16;
! 1711:
! 1712: ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]),
! 1713: M_DEVBUF, M_NOWAIT);
! 1714: if (ims == NULL)
! 1715: return (NULL);
! 1716:
! 1717: memset(ims, 0, sizeof *ims);
! 1718:
! 1719: /* Initialize the map. */
! 1720: ims->ims_map.ipm_maxpage = n;
! 1721: SPLAY_INIT(&ims->ims_map.ipm_tree);
! 1722:
! 1723: /* Initialize the flush area. */
! 1724: sbf = &ims->ims_flush;
! 1725: va = (vaddr_t)&sbf->sbf_area[0x40];
! 1726: va &= ~0x3f;
! 1727: pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa);
! 1728: sbf->sbf_flush = (void *)va;
! 1729:
! 1730: return (ims);
! 1731: }
! 1732:
! 1733: /*
! 1734: * Destroy an iomap.
! 1735: */
! 1736: void
! 1737: iommu_iomap_destroy(struct iommu_map_state *ims)
! 1738: {
! 1739: #ifdef DIAGNOSTIC
! 1740: if (ims->ims_map.ipm_pagecnt > 0)
! 1741: printf("iommu_iomap_destroy: %d page entries in use\n",
! 1742: ims->ims_map.ipm_pagecnt);
! 1743: #endif
! 1744:
! 1745: free(ims, M_DEVBUF);
! 1746: }
! 1747:
! 1748: /*
! 1749: * Utility function used by splay tree to order page entries by pa.
! 1750: */
! 1751: static inline int
! 1752: iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b)
! 1753: {
! 1754: return ((a->ipe_pa > b->ipe_pa) ? 1 :
! 1755: (a->ipe_pa < b->ipe_pa) ? -1 : 0);
! 1756: }
! 1757:
! 1758: SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);
! 1759:
! 1760: SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);
! 1761:
! 1762: /*
! 1763: * Insert a pa entry in the iomap.
! 1764: */
! 1765: int
! 1766: iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa)
! 1767: {
! 1768: struct iommu_page_map *ipm = &ims->ims_map;
! 1769: struct iommu_page_entry *e;
! 1770:
! 1771: if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) {
! 1772: struct iommu_page_entry ipe;
! 1773:
! 1774: ipe.ipe_pa = pa;
! 1775: if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe))
! 1776: return (0);
! 1777:
! 1778: return (ENOMEM);
! 1779: }
! 1780:
! 1781: e = &ipm->ipm_map[ipm->ipm_pagecnt];
! 1782:
! 1783: e->ipe_pa = pa;
! 1784: e->ipe_va = NULL;
! 1785:
! 1786: e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e);
! 1787:
! 1788: /* Duplicates are okay, but only count them once. */
! 1789: if (e)
! 1790: return (0);
! 1791:
! 1792: ++ipm->ipm_pagecnt;
! 1793:
! 1794: return (0);
! 1795: }
! 1796:
! 1797: /*
! 1798: * Locate the iomap by filling in the pa->va mapping and inserting it
! 1799: * into the IOMMU tables.
! 1800: */
! 1801: int
! 1802: iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims,
! 1803: vaddr_t vmaddr, int flags)
! 1804: {
! 1805: struct iommu_page_map *ipm = &ims->ims_map;
! 1806: struct iommu_page_entry *e;
! 1807: struct strbuf_ctl *sb = ims->ims_sb;
! 1808: int i;
! 1809:
! 1810: if (sb->sb_flush == NULL)
! 1811: flags &= ~BUS_DMA_STREAMING;
! 1812:
! 1813: if (flags & BUS_DMA_STREAMING)
! 1814: ims->ims_flags |= IOMMU_MAP_STREAM;
! 1815: else
! 1816: ims->ims_flags &= ~IOMMU_MAP_STREAM;
! 1817:
! 1818: for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) {
! 1819: e->ipe_va = vmaddr;
! 1820: iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags);
! 1821: vmaddr += PAGE_SIZE;
! 1822: }
! 1823:
! 1824: return (0);
! 1825: }
! 1826:
! 1827: /*
! 1828: * Remove the iomap from the IOMMU.
! 1829: */
! 1830: int
! 1831: iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims)
! 1832: {
! 1833: struct iommu_page_map *ipm = &ims->ims_map;
! 1834: struct iommu_page_entry *e;
! 1835: struct strbuf_ctl *sb = ims->ims_sb;
! 1836: int i;
! 1837:
! 1838: for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e)
! 1839: iommu_remove(is, sb, e->ipe_va);
! 1840:
! 1841: return (0);
! 1842: }
! 1843:
! 1844: /*
! 1845: * Translate a physical address (pa) into a DVMA address.
! 1846: */
! 1847: vaddr_t
! 1848: iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa)
! 1849: {
! 1850: struct iommu_page_map *ipm = &ims->ims_map;
! 1851: struct iommu_page_entry *e;
! 1852: struct iommu_page_entry pe;
! 1853: paddr_t offset = pa & PAGE_MASK;
! 1854:
! 1855: pe.ipe_pa = trunc_page(pa);
! 1856:
! 1857: e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe);
! 1858:
! 1859: if (e == NULL)
! 1860: return (NULL);
! 1861:
! 1862: return (e->ipe_va | offset);
! 1863: }
! 1864:
! 1865: /*
! 1866: * Clear the iomap table and tree.
! 1867: */
! 1868: void
! 1869: iommu_iomap_clear_pages(struct iommu_map_state *ims)
! 1870: {
! 1871: ims->ims_map.ipm_pagecnt = 0;
! 1872: SPLAY_INIT(&ims->ims_map.ipm_tree);
! 1873: }
! 1874:
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to iommu.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / sparc64 / dev