Annotation of sys/arch/arm/arm/bus_dma.c, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: bus_dma.c,v 1.10 2007/05/29 21:00:50 jason Exp $ */
! 2: /* $NetBSD: bus_dma.c,v 1.38 2003/10/30 08:44:13 scw Exp $ */
! 3:
! 4: /*-
! 5: * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
! 6: * All rights reserved.
! 7: *
! 8: * This code is derived from software contributed to The NetBSD Foundation
! 9: * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
! 10: * NASA Ames Research Center.
! 11: *
! 12: * Redistribution and use in source and binary forms, with or without
! 13: * modification, are permitted provided that the following conditions
! 14: * are met:
! 15: * 1. Redistributions of source code must retain the above copyright
! 16: * notice, this list of conditions and the following disclaimer.
! 17: * 2. Redistributions in binary form must reproduce the above copyright
! 18: * notice, this list of conditions and the following disclaimer in the
! 19: * documentation and/or other materials provided with the distribution.
! 20: * 3. All advertising materials mentioning features or use of this software
! 21: * must display the following acknowledgement:
! 22: * This product includes software developed by the NetBSD
! 23: * Foundation, Inc. and its contributors.
! 24: * 4. Neither the name of The NetBSD Foundation nor the names of its
! 25: * contributors may be used to endorse or promote products derived
! 26: * from this software without specific prior written permission.
! 27: *
! 28: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
! 29: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
! 30: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
! 31: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
! 32: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
! 33: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
! 34: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
! 35: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
! 36: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
! 37: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
! 38: * POSSIBILITY OF SUCH DAMAGE.
! 39: */
! 40:
! 41: #define _ARM32_BUS_DMA_PRIVATE
! 42:
! 43: #include <sys/param.h>
! 44: #include <sys/systm.h>
! 45: #include <sys/kernel.h>
! 46: #include <sys/proc.h>
! 47: #include <sys/buf.h>
! 48: #include <sys/reboot.h>
! 49: #include <sys/conf.h>
! 50: #include <sys/file.h>
! 51: #include <sys/malloc.h>
! 52: #include <sys/mbuf.h>
! 53: #include <sys/vnode.h>
! 54: #include <sys/device.h>
! 55:
! 56: #include <uvm/uvm_extern.h>
! 57:
! 58: #include <machine/bus.h>
! 59: #include <machine/cpu.h>
! 60:
! 61: #include <arm/cpufunc.h>
! 62:
! 63: int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *,
! 64: bus_size_t, struct proc *, int, paddr_t *, int *, int);
! 65: struct arm32_dma_range *_bus_dma_inrange(struct arm32_dma_range *,
! 66: int, bus_addr_t);
! 67:
! 68: /*
! 69: * Check to see if the specified page is in an allowed DMA range.
! 70: */
! 71: __inline struct arm32_dma_range *
! 72: _bus_dma_inrange(struct arm32_dma_range *ranges, int nranges,
! 73: bus_addr_t curaddr)
! 74: {
! 75: struct arm32_dma_range *dr;
! 76: int i;
! 77:
! 78: for (i = 0, dr = ranges; i < nranges; i++, dr++) {
! 79: if (curaddr >= dr->dr_sysbase &&
! 80: round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len))
! 81: return (dr);
! 82: }
! 83:
! 84: return (NULL);
! 85: }
! 86:
! 87: /*
! 88: * Common function for DMA map creation. May be called by bus-specific
! 89: * DMA map creation functions.
! 90: */
! 91: int
! 92: _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
! 93: bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
! 94: {
! 95: struct arm32_bus_dmamap *map;
! 96: void *mapstore;
! 97: size_t mapsize;
! 98:
! 99: #ifdef DEBUG_DMA
! 100: printf("dmamap_create: t=%p size=%lx nseg=%x msegsz=%lx boundary=%lx flags=%x\n",
! 101: t, size, nsegments, maxsegsz, boundary, flags);
! 102: #endif /* DEBUG_DMA */
! 103:
! 104: /*
! 105: * Allocate and initialize the DMA map. The end of the map
! 106: * is a variable-sized array of segments, so we allocate enough
! 107: * room for them in one shot.
! 108: *
! 109: * Note we don't preserve the WAITOK or NOWAIT flags. Preservation
! 110: * of ALLOCNOW notifies others that we've reserved these resources,
! 111: * and they are not to be freed.
! 112: *
! 113: * The bus_dmamap_t includes one bus_dma_segment_t, hence
! 114: * the (nsegments - 1).
! 115: */
! 116: mapsize = sizeof(struct arm32_bus_dmamap) +
! 117: (sizeof(bus_dma_segment_t) * (nsegments - 1));
! 118: if ((mapstore = malloc(mapsize, M_DEVBUF,
! 119: (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL)
! 120: return (ENOMEM);
! 121:
! 122: memset(mapstore, 0, mapsize);
! 123: map = (struct arm32_bus_dmamap *)mapstore;
! 124: map->_dm_size = size;
! 125: map->_dm_segcnt = nsegments;
! 126: map->_dm_maxsegsz = maxsegsz;
! 127: map->_dm_boundary = boundary;
! 128: map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
! 129: map->_dm_origbuf = NULL;
! 130: map->_dm_buftype = ARM32_BUFTYPE_INVALID;
! 131: map->_dm_proc = NULL;
! 132: map->dm_mapsize = 0; /* no valid mappings */
! 133: map->dm_nsegs = 0;
! 134:
! 135: *dmamp = map;
! 136: #ifdef DEBUG_DMA
! 137: printf("dmamap_create:map=%p\n", map);
! 138: #endif /* DEBUG_DMA */
! 139: return (0);
! 140: }
! 141:
! 142: /*
! 143: * Common function for DMA map destruction. May be called by bus-specific
! 144: * DMA map destruction functions.
! 145: */
! 146: void
! 147: _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
! 148: {
! 149:
! 150: #ifdef DEBUG_DMA
! 151: printf("dmamap_destroy: t=%p map=%p\n", t, map);
! 152: #endif /* DEBUG_DMA */
! 153:
! 154: /*
! 155: * Explicit unload.
! 156: */
! 157: map->dm_mapsize = 0;
! 158: map->dm_nsegs = 0;
! 159: map->_dm_origbuf = NULL;
! 160: map->_dm_buftype = ARM32_BUFTYPE_INVALID;
! 161: map->_dm_proc = NULL;
! 162:
! 163: free(map, M_DEVBUF);
! 164: }
! 165:
! 166: /*
! 167: * Common function for loading a DMA map with a linear buffer. May
! 168: * be called by bus-specific DMA map load functions.
! 169: */
! 170: int
! 171: _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
! 172: bus_size_t buflen, struct proc *p, int flags)
! 173: {
! 174: paddr_t lastaddr;
! 175: int seg, error;
! 176:
! 177: #ifdef DEBUG_DMA
! 178: printf("dmamap_load: t=%p map=%p buf=%p len=%lx p=%p f=%d\n",
! 179: t, map, buf, buflen, p, flags);
! 180: #endif /* DEBUG_DMA */
! 181:
! 182: /*
! 183: * Make sure that on error condition we return "no valid mappings".
! 184: */
! 185: map->dm_mapsize = 0;
! 186: map->dm_nsegs = 0;
! 187:
! 188: if (buflen > map->_dm_size)
! 189: return (EINVAL);
! 190:
! 191: /* _bus_dmamap_load_buffer() clears this if we're not... */
! 192: map->_dm_flags |= ARM32_DMAMAP_COHERENT;
! 193:
! 194: seg = 0;
! 195: error = _bus_dmamap_load_buffer(t, map, buf, buflen, p, flags,
! 196: &lastaddr, &seg, 1);
! 197: if (error == 0) {
! 198: map->dm_mapsize = buflen;
! 199: map->dm_nsegs = seg + 1;
! 200: map->_dm_origbuf = buf;
! 201: map->_dm_buftype = ARM32_BUFTYPE_LINEAR;
! 202: map->_dm_proc = p;
! 203: }
! 204: #ifdef DEBUG_DMA
! 205: printf("dmamap_load: error=%d\n", error);
! 206: #endif /* DEBUG_DMA */
! 207: return (error);
! 208: }
! 209:
! 210: /*
! 211: * Like _bus_dmamap_load(), but for mbufs.
! 212: */
! 213: int
! 214: _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0,
! 215: int flags)
! 216: {
! 217: #if 0
! 218: struct arm32_dma_range *dr;
! 219: #endif
! 220: paddr_t lastaddr;
! 221: int seg, error, first;
! 222: struct mbuf *m;
! 223:
! 224: #ifdef DEBUG_DMA
! 225: printf("dmamap_load_mbuf: t=%p map=%p m0=%p f=%d\n",
! 226: t, map, m0, flags);
! 227: #endif /* DEBUG_DMA */
! 228:
! 229: /*
! 230: * Make sure that on error condition we return "no valid mappings."
! 231: */
! 232: map->dm_mapsize = 0;
! 233: map->dm_nsegs = 0;
! 234:
! 235: #ifdef DIAGNOSTIC
! 236: if ((m0->m_flags & M_PKTHDR) == 0)
! 237: panic("_bus_dmamap_load_mbuf: no packet header");
! 238: #endif /* DIAGNOSTIC */
! 239:
! 240: if (m0->m_pkthdr.len > map->_dm_size)
! 241: return (EINVAL);
! 242:
! 243: /*
! 244: * Mbuf chains should almost never have coherent (i.e.
! 245: * un-cached) mappings, so clear that flag now.
! 246: */
! 247: map->_dm_flags &= ~ARM32_DMAMAP_COHERENT;
! 248:
! 249: first = 1;
! 250: seg = 0;
! 251: error = 0;
! 252: for (m = m0; m != NULL && error == 0; m = m->m_next) {
! 253: if (m->m_len == 0)
! 254: continue;
! 255: error = _bus_dmamap_load_buffer(t, map, m->m_data, m->m_len,
! 256: NULL, flags, &lastaddr, &seg, first);
! 257: first = 0;
! 258: }
! 259: if (error == 0) {
! 260: map->dm_mapsize = m0->m_pkthdr.len;
! 261: map->dm_nsegs = seg + 1;
! 262: map->_dm_origbuf = m0;
! 263: map->_dm_buftype = ARM32_BUFTYPE_MBUF;
! 264: map->_dm_proc = NULL; /* always kernel */
! 265: }
! 266: #ifdef DEBUG_DMA
! 267: printf("dmamap_load_mbuf: error=%d\n", error);
! 268: #endif /* DEBUG_DMA */
! 269: return (error);
! 270: }
! 271:
! 272: /*
! 273: * Like _bus_dmamap_load(), but for uios.
! 274: */
! 275: int
! 276: _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio,
! 277: int flags)
! 278: {
! 279: paddr_t lastaddr;
! 280: int seg, i, error, first;
! 281: bus_size_t minlen, resid;
! 282: struct proc *p = NULL;
! 283: struct iovec *iov;
! 284: caddr_t addr;
! 285:
! 286: /*
! 287: * Make sure that on error condition we return "no valid mappings."
! 288: */
! 289: map->dm_mapsize = 0;
! 290: map->dm_nsegs = 0;
! 291:
! 292: resid = uio->uio_resid;
! 293: iov = uio->uio_iov;
! 294:
! 295: if (uio->uio_segflg == UIO_USERSPACE) {
! 296: p = uio->uio_procp;
! 297: #ifdef DIAGNOSTIC
! 298: if (p == NULL)
! 299: panic("_bus_dmamap_load_uio: USERSPACE but no proc");
! 300: #endif
! 301: }
! 302:
! 303: /* _bus_dmamap_load_buffer() clears this if we're not... */
! 304: map->_dm_flags |= ARM32_DMAMAP_COHERENT;
! 305:
! 306: first = 1;
! 307: seg = 0;
! 308: error = 0;
! 309: for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
! 310: /*
! 311: * Now at the first iovec to load. Load each iovec
! 312: * until we have exhausted the residual count.
! 313: */
! 314: minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
! 315: addr = (caddr_t)iov[i].iov_base;
! 316:
! 317: error = _bus_dmamap_load_buffer(t, map, addr, minlen,
! 318: p, flags, &lastaddr, &seg, first);
! 319: first = 0;
! 320:
! 321: resid -= minlen;
! 322: }
! 323: if (error == 0) {
! 324: map->dm_mapsize = uio->uio_resid;
! 325: map->dm_nsegs = seg + 1;
! 326: map->_dm_origbuf = uio;
! 327: map->_dm_buftype = ARM32_BUFTYPE_UIO;
! 328: map->_dm_proc = p;
! 329: }
! 330: return (error);
! 331: }
! 332:
! 333: /*
! 334: * Like _bus_dmamap_load(), but for raw memory allocated with
! 335: * bus_dmamem_alloc().
! 336: */
! 337: int
! 338: _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
! 339: bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
! 340: {
! 341:
! 342: panic("_bus_dmamap_load_raw: not implemented");
! 343: }
! 344:
! 345: /*
! 346: * Common function for unloading a DMA map. May be called by
! 347: * bus-specific DMA map unload functions.
! 348: */
! 349: void
! 350: _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
! 351: {
! 352:
! 353: #ifdef DEBUG_DMA
! 354: printf("dmamap_unload: t=%p map=%p\n", t, map);
! 355: #endif /* DEBUG_DMA */
! 356:
! 357: /*
! 358: * No resources to free; just mark the mappings as
! 359: * invalid.
! 360: */
! 361: map->dm_mapsize = 0;
! 362: map->dm_nsegs = 0;
! 363: map->_dm_origbuf = NULL;
! 364: map->_dm_buftype = ARM32_BUFTYPE_INVALID;
! 365: map->_dm_proc = NULL;
! 366: }
! 367:
! 368: static __inline void
! 369: _bus_dmamap_sync_linear(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
! 370: bus_size_t len, int ops)
! 371: {
! 372: vaddr_t addr = (vaddr_t) map->_dm_origbuf;
! 373:
! 374: addr += offset;
! 375:
! 376: switch (ops) {
! 377: case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
! 378: cpu_dcache_wbinv_range(addr, len);
! 379: break;
! 380:
! 381: case BUS_DMASYNC_PREREAD:
! 382: if (((addr | len) & arm_dcache_align_mask) == 0)
! 383: cpu_dcache_inv_range(addr, len);
! 384: else
! 385: cpu_dcache_wbinv_range(addr, len);
! 386: break;
! 387:
! 388: case BUS_DMASYNC_PREWRITE:
! 389: cpu_dcache_wb_range(addr, len);
! 390: break;
! 391:
! 392: case BUS_DMASYNC_POSTREAD:
! 393: cpu_dcache_inv_range(addr, len);
! 394: break;
! 395: }
! 396: }
! 397:
! 398: static __inline void
! 399: _bus_dmamap_sync_mbuf(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
! 400: bus_size_t len, int ops)
! 401: {
! 402: struct mbuf *m, *m0 = map->_dm_origbuf;
! 403: bus_size_t minlen, moff;
! 404: vaddr_t maddr;
! 405:
! 406: for (moff = offset, m = m0; m != NULL && len != 0;
! 407: m = m->m_next) {
! 408: /* Find the beginning mbuf. */
! 409: if (moff >= m->m_len) {
! 410: moff -= m->m_len;
! 411: continue;
! 412: }
! 413:
! 414: /*
! 415: * Now at the first mbuf to sync; nail each one until
! 416: * we have exhausted the length.
! 417: */
! 418: minlen = m->m_len - moff;
! 419: if (len < minlen)
! 420: minlen = len;
! 421:
! 422: maddr = mtod(m, vaddr_t);
! 423: maddr += moff;
! 424:
! 425: /*
! 426: * We can save a lot of work here if we know the mapping
! 427: * is read-only at the MMU:
! 428: *
! 429: * If a mapping is read-only, no dirty cache blocks will
! 430: * exist for it. If a writable mapping was made read-only,
! 431: * we know any dirty cache lines for the range will have
! 432: * been cleaned for us already. Therefore, if the upper
! 433: * layer can tell us we have a read-only mapping, we can
! 434: * skip all cache cleaning.
! 435: *
! 436: * NOTE: This only works if we know the pmap cleans pages
! 437: * before making a read-write -> read-only transition. If
! 438: * this ever becomes non-true (e.g. Physically Indexed
! 439: * cache), this will have to be revisited.
! 440: */
! 441: switch (ops) {
! 442: case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
! 443: /* if (! M_ROMAP(m)) */{
! 444: cpu_dcache_wbinv_range(maddr, minlen);
! 445: break;
! 446: }
! 447: /* else FALLTHROUGH */
! 448:
! 449: case BUS_DMASYNC_PREREAD:
! 450: if (((maddr | minlen) & arm_dcache_align_mask) == 0)
! 451: cpu_dcache_inv_range(maddr, minlen);
! 452: else
! 453: cpu_dcache_wbinv_range(maddr, minlen);
! 454: break;
! 455:
! 456: case BUS_DMASYNC_PREWRITE:
! 457: /* if (! M_ROMAP(m)) */
! 458: cpu_dcache_wb_range(maddr, minlen);
! 459: break;
! 460: }
! 461: moff = 0;
! 462: len -= minlen;
! 463: }
! 464: }
! 465:
! 466: static __inline void
! 467: _bus_dmamap_sync_uio(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
! 468: bus_size_t len, int ops)
! 469: {
! 470: struct uio *uio = map->_dm_origbuf;
! 471: struct iovec *iov;
! 472: bus_size_t minlen, ioff;
! 473: vaddr_t addr;
! 474:
! 475: for (iov = uio->uio_iov, ioff = offset; len != 0; iov++) {
! 476: /* Find the beginning iovec. */
! 477: if (ioff >= iov->iov_len) {
! 478: ioff -= iov->iov_len;
! 479: continue;
! 480: }
! 481:
! 482: /*
! 483: * Now at the first iovec to sync; nail each one until
! 484: * we have exhausted the length.
! 485: */
! 486: minlen = iov->iov_len - ioff;
! 487: if (len < minlen)
! 488: minlen = len;
! 489:
! 490: addr = (vaddr_t) iov->iov_base;
! 491: addr += ioff;
! 492:
! 493: switch (ops) {
! 494: case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
! 495: cpu_dcache_wbinv_range(addr, minlen);
! 496: break;
! 497:
! 498: case BUS_DMASYNC_PREREAD:
! 499: if (((addr | minlen) & arm_dcache_align_mask) == 0)
! 500: cpu_dcache_inv_range(addr, minlen);
! 501: else
! 502: cpu_dcache_wbinv_range(addr, minlen);
! 503: break;
! 504:
! 505: case BUS_DMASYNC_PREWRITE:
! 506: cpu_dcache_wb_range(addr, minlen);
! 507: break;
! 508: }
! 509: ioff = 0;
! 510: len -= minlen;
! 511: }
! 512: }
! 513:
! 514: /*
! 515: * Common function for DMA map synchronization. May be called
! 516: * by bus-specific DMA map synchronization functions.
! 517: *
! 518: * This version works for the Virtually Indexed Virtually Tagged
! 519: * cache found on 32-bit ARM processors.
! 520: *
! 521: * XXX Should have separate versions for write-through vs.
! 522: * XXX write-back caches. We currently assume write-back
! 523: * XXX here, which is not as efficient as it could be for
! 524: * XXX the write-through case.
! 525: */
! 526: void
! 527: _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
! 528: bus_size_t len, int ops)
! 529: {
! 530:
! 531: #ifdef DEBUG_DMA
! 532: printf("dmamap_sync: t=%p map=%p offset=%lx len=%lx ops=%x\n",
! 533: t, map, offset, len, ops);
! 534: #endif /* DEBUG_DMA */
! 535:
! 536: /*
! 537: * Mixing of PRE and POST operations is not allowed.
! 538: */
! 539: if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
! 540: (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
! 541: panic("_bus_dmamap_sync: mix PRE and POST");
! 542:
! 543: #ifdef DIAGNOSTIC
! 544: if (offset >= map->dm_mapsize)
! 545: panic("_bus_dmamap_sync: bad offset %lu (map size is %lu)",
! 546: offset, map->dm_mapsize);
! 547: if (len == 0 || (offset + len) > map->dm_mapsize)
! 548: panic("_bus_dmamap_sync: bad length");
! 549: #endif
! 550:
! 551: /*
! 552: * For a virtually-indexed write-back cache, we need
! 553: * to do the following things:
! 554: *
! 555: * PREREAD -- Invalidate the D-cache. We do this
! 556: * here in case a write-back is required by the back-end.
! 557: *
! 558: * PREWRITE -- Write-back the D-cache. Note that if
! 559: * we are doing a PREREAD|PREWRITE, we can collapse
! 560: * the whole thing into a single Wb-Inv.
! 561: *
! 562: * POSTREAD -- Invalidate the D-Cache. Contents of
! 563: * the cache could be from before a device wrote
! 564: * to the memory.
! 565: *
! 566: * POSTWRITE -- Nothing.
! 567: */
! 568:
! 569: /* Skip cache frobbing if mapping was COHERENT. */
! 570: if (map->_dm_flags & ARM32_DMAMAP_COHERENT) {
! 571: /* Drain the write buffer. */
! 572: cpu_drain_writebuf();
! 573: return;
! 574: }
! 575:
! 576: /*
! 577: * If the mapping belongs to a non-kernel vmspace, and the
! 578: * vmspace has not been active since the last time a full
! 579: * cache flush was performed, we don't need to do anything.
! 580: */
! 581: if (__predict_false(map->_dm_proc != NULL &&
! 582: map->_dm_proc->p_vmspace->vm_map.pmap->pm_cstate.cs_cache_d == 0))
! 583: return;
! 584:
! 585: switch (map->_dm_buftype) {
! 586: case ARM32_BUFTYPE_LINEAR:
! 587: _bus_dmamap_sync_linear(t, map, offset, len, ops);
! 588: break;
! 589:
! 590: case ARM32_BUFTYPE_MBUF:
! 591: _bus_dmamap_sync_mbuf(t, map, offset, len, ops);
! 592: break;
! 593:
! 594: case ARM32_BUFTYPE_UIO:
! 595: _bus_dmamap_sync_uio(t, map, offset, len, ops);
! 596: break;
! 597:
! 598: case ARM32_BUFTYPE_INVALID:
! 599: panic("_bus_dmamap_sync: ARM32_BUFTYPE_INVALID");
! 600: break;
! 601:
! 602: default:
! 603: printf("unknown buffer type %d\n", map->_dm_buftype);
! 604: panic("_bus_dmamap_sync");
! 605: }
! 606:
! 607: /* Drain the write buffer. */
! 608: cpu_drain_writebuf();
! 609: }
! 610:
! 611: /*
! 612: * Common function for DMA-safe memory allocation. May be called
! 613: * by bus-specific DMA memory allocation functions.
! 614: */
! 615:
! 616: extern paddr_t physical_start;
! 617: extern paddr_t physical_end;
! 618:
! 619: int
! 620: _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
! 621: bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
! 622: int flags)
! 623: {
! 624: struct arm32_dma_range *dr;
! 625: int error, i;
! 626:
! 627: #ifdef DEBUG_DMA
! 628: printf("dmamem_alloc t=%p size=%lx align=%lx boundary=%lx "
! 629: "segs=%p nsegs=%x rsegs=%p flags=%x\n", t, size, alignment,
! 630: boundary, segs, nsegs, rsegs, flags);
! 631: #endif
! 632:
! 633: if ((dr = t->_ranges) != NULL) {
! 634: error = ENOMEM;
! 635: for (i = 0; i < t->_nranges; i++, dr++) {
! 636: if (dr->dr_len == 0)
! 637: continue;
! 638: error = _bus_dmamem_alloc_range(t, size, alignment,
! 639: boundary, segs, nsegs, rsegs, flags,
! 640: trunc_page(dr->dr_sysbase),
! 641: trunc_page(dr->dr_sysbase + dr->dr_len));
! 642: if (error == 0)
! 643: break;
! 644: }
! 645: } else {
! 646: error = _bus_dmamem_alloc_range(t, size, alignment, boundary,
! 647: segs, nsegs, rsegs, flags, trunc_page(physical_start),
! 648: trunc_page(physical_end));
! 649: }
! 650:
! 651: #ifdef DEBUG_DMA
! 652: printf("dmamem_alloc: =%d\n", error);
! 653: #endif
! 654:
! 655: return(error);
! 656: }
! 657:
! 658: /*
! 659: * Common function for freeing DMA-safe memory. May be called by
! 660: * bus-specific DMA memory free functions.
! 661: */
! 662: void
! 663: _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
! 664: {
! 665: struct vm_page *m;
! 666: bus_addr_t addr;
! 667: struct pglist mlist;
! 668: int curseg;
! 669:
! 670: #ifdef DEBUG_DMA
! 671: printf("dmamem_free: t=%p segs=%p nsegs=%x\n", t, segs, nsegs);
! 672: #endif /* DEBUG_DMA */
! 673:
! 674: /*
! 675: * Build a list of pages to free back to the VM system.
! 676: */
! 677: TAILQ_INIT(&mlist);
! 678: for (curseg = 0; curseg < nsegs; curseg++) {
! 679: for (addr = segs[curseg].ds_addr;
! 680: addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
! 681: addr += PAGE_SIZE) {
! 682: m = PHYS_TO_VM_PAGE(addr);
! 683: TAILQ_INSERT_TAIL(&mlist, m, pageq);
! 684: }
! 685: }
! 686: uvm_pglistfree(&mlist);
! 687: }
! 688:
! 689: /*
! 690: * Common function for mapping DMA-safe memory. May be called by
! 691: * bus-specific DMA memory map functions.
! 692: */
! 693: int
! 694: _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
! 695: size_t size, caddr_t *kvap, int flags)
! 696: {
! 697: vaddr_t va;
! 698: bus_addr_t addr;
! 699: int curseg;
! 700: pt_entry_t *ptep/*, pte*/;
! 701:
! 702: #ifdef DEBUG_DMA
! 703: printf("dmamem_map: t=%p segs=%p nsegs=%x size=%lx flags=%x\n", t,
! 704: segs, nsegs, (unsigned long)size, flags);
! 705: #endif /* DEBUG_DMA */
! 706:
! 707: size = round_page(size);
! 708: va = uvm_km_valloc(kernel_map, size);
! 709:
! 710: if (va == 0)
! 711: return (ENOMEM);
! 712:
! 713: *kvap = (caddr_t)va;
! 714:
! 715: for (curseg = 0; curseg < nsegs; curseg++) {
! 716: for (addr = segs[curseg].ds_addr;
! 717: addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
! 718: addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
! 719: #ifdef DEBUG_DMA
! 720: printf("wiring p%lx to v%lx", addr, va);
! 721: #endif /* DEBUG_DMA */
! 722: if (size == 0)
! 723: panic("_bus_dmamem_map: size botch");
! 724: pmap_enter(pmap_kernel(), va, addr,
! 725: VM_PROT_READ | VM_PROT_WRITE,
! 726: VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED);
! 727: /*
! 728: * If the memory must remain coherent with the
! 729: * cache then we must make the memory uncacheable
! 730: * in order to maintain virtual cache coherency.
! 731: * We must also guarantee the cache does not already
! 732: * contain the virtual addresses we are making
! 733: * uncacheable.
! 734: */
! 735: if (flags & BUS_DMA_COHERENT) {
! 736: cpu_dcache_wbinv_range(va, PAGE_SIZE);
! 737: cpu_drain_writebuf();
! 738: ptep = vtopte(va);
! 739: *ptep &= ~L2_S_CACHE_MASK;
! 740: PTE_SYNC(ptep);
! 741: tlb_flush();
! 742: }
! 743: #ifdef DEBUG_DMA
! 744: ptep = vtopte(va);
! 745: printf(" pte=v%p *pte=%x\n", ptep, *ptep);
! 746: #endif /* DEBUG_DMA */
! 747: }
! 748: }
! 749: pmap_update(pmap_kernel());
! 750: #ifdef DEBUG_DMA
! 751: printf("dmamem_map: =%p\n", *kvap);
! 752: #endif /* DEBUG_DMA */
! 753: return (0);
! 754: }
! 755:
! 756: /*
! 757: * Common function for unmapping DMA-safe memory. May be called by
! 758: * bus-specific DMA memory unmapping functions.
! 759: */
! 760: void
! 761: _bus_dmamem_unmap(bus_dma_tag_t t, caddr_t kva, size_t size)
! 762: {
! 763:
! 764: #ifdef DEBUG_DMA
! 765: printf("dmamem_unmap: t=%p kva=%p size=%lx\n", t, kva,
! 766: (unsigned long)size);
! 767: #endif /* DEBUG_DMA */
! 768: #ifdef DIAGNOSTIC
! 769: if ((u_long)kva & PGOFSET)
! 770: panic("_bus_dmamem_unmap");
! 771: #endif /* DIAGNOSTIC */
! 772:
! 773: size = round_page(size);
! 774: uvm_km_free(kernel_map, (vaddr_t)kva, size);
! 775: }
! 776:
! 777: /*
! 778: * Common function for mmap(2)'ing DMA-safe memory. May be called by
! 779: * bus-specific DMA mmap(2)'ing functions.
! 780: */
! 781: paddr_t
! 782: _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
! 783: off_t off, int prot, int flags)
! 784: {
! 785: int i;
! 786:
! 787: for (i = 0; i < nsegs; i++) {
! 788: #ifdef DIAGNOSTIC
! 789: if (off & PGOFSET)
! 790: panic("_bus_dmamem_mmap: offset unaligned");
! 791: if (segs[i].ds_addr & PGOFSET)
! 792: panic("_bus_dmamem_mmap: segment unaligned");
! 793: if (segs[i].ds_len & PGOFSET)
! 794: panic("_bus_dmamem_mmap: segment size not multiple"
! 795: " of page size");
! 796: #endif /* DIAGNOSTIC */
! 797: if (off >= segs[i].ds_len) {
! 798: off -= segs[i].ds_len;
! 799: continue;
! 800: }
! 801:
! 802: return (atop(segs[i].ds_addr + off));
! 803: }
! 804:
! 805: /* Page not found. */
! 806: return (-1);
! 807: }
! 808:
! 809: /**********************************************************************
! 810: * DMA utility functions
! 811: **********************************************************************/
! 812:
! 813: /*
! 814: * Utility function to load a linear buffer. lastaddrp holds state
! 815: * between invocations (for multiple-buffer loads). segp contains
! 816: * the starting segment on entrance, and the ending segment on exit.
! 817: * first indicates if this is the first invocation of this function.
! 818: */
! 819: int
! 820: _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
! 821: bus_size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp,
! 822: int *segp, int first)
! 823: {
! 824: struct arm32_dma_range *dr;
! 825: bus_size_t sgsize;
! 826: bus_addr_t curaddr, lastaddr, baddr, bmask;
! 827: vaddr_t vaddr = (vaddr_t)buf;
! 828: pd_entry_t *pde;
! 829: pt_entry_t pte;
! 830: int seg;
! 831: pmap_t pmap;
! 832: pt_entry_t *ptep;
! 833:
! 834: #ifdef DEBUG_DMA
! 835: printf("_bus_dmamem_load_buffer(buf=%p, len=%lx, flags=%d, 1st=%d)\n",
! 836: buf, buflen, flags, first);
! 837: #endif /* DEBUG_DMA */
! 838:
! 839: if (p != NULL)
! 840: pmap = p->p_vmspace->vm_map.pmap;
! 841: else
! 842: pmap = pmap_kernel();
! 843:
! 844: lastaddr = *lastaddrp;
! 845: bmask = ~(map->_dm_boundary - 1);
! 846:
! 847: for (seg = *segp; buflen > 0; ) {
! 848: /*
! 849: * Get the physical address for this segment.
! 850: *
! 851: * XXX Don't support checking for coherent mappings
! 852: * XXX in user address space.
! 853: */
! 854: if (__predict_true(pmap == pmap_kernel())) {
! 855: (void) pmap_get_pde_pte(pmap, vaddr, &pde, &ptep);
! 856: if (__predict_false(pmap_pde_section(pde))) {
! 857: curaddr = (*pde & L1_S_FRAME) |
! 858: (vaddr & L1_S_OFFSET);
! 859: if (*pde & L1_S_CACHE_MASK) {
! 860: map->_dm_flags &=
! 861: ~ARM32_DMAMAP_COHERENT;
! 862: }
! 863: } else {
! 864: pte = *ptep;
! 865: KDASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV);
! 866: if (__predict_false((pte & L2_TYPE_MASK)
! 867: == L2_TYPE_L)) {
! 868: curaddr = (pte & L2_L_FRAME) |
! 869: (vaddr & L2_L_OFFSET);
! 870: if (pte & L2_L_CACHE_MASK) {
! 871: map->_dm_flags &=
! 872: ~ARM32_DMAMAP_COHERENT;
! 873: }
! 874: } else {
! 875: curaddr = (pte & L2_S_FRAME) |
! 876: (vaddr & L2_S_OFFSET);
! 877: if (pte & L2_S_CACHE_MASK) {
! 878: map->_dm_flags &=
! 879: ~ARM32_DMAMAP_COHERENT;
! 880: }
! 881: }
! 882: }
! 883: } else {
! 884: (void) pmap_extract(pmap, vaddr, &curaddr);
! 885: map->_dm_flags &= ~ARM32_DMAMAP_COHERENT;
! 886: }
! 887:
! 888: /*
! 889: * Make sure we're in an allowed DMA range.
! 890: */
! 891: if (t->_ranges != NULL) {
! 892: /* XXX cache last result? */
! 893: dr = _bus_dma_inrange(t->_ranges, t->_nranges,
! 894: curaddr);
! 895: if (dr == NULL)
! 896: return (EINVAL);
! 897:
! 898: /*
! 899: * In a valid DMA range. Translate the physical
! 900: * memory address to an address in the DMA window.
! 901: */
! 902: curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase;
! 903: }
! 904:
! 905: /*
! 906: * Compute the segment size, and adjust counts.
! 907: */
! 908: sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
! 909: if (buflen < sgsize)
! 910: sgsize = buflen;
! 911:
! 912: /*
! 913: * Make sure we don't cross any boundaries.
! 914: */
! 915: if (map->_dm_boundary > 0) {
! 916: baddr = (curaddr + map->_dm_boundary) & bmask;
! 917: if (sgsize > (baddr - curaddr))
! 918: sgsize = (baddr - curaddr);
! 919: }
! 920:
! 921: /*
! 922: * Insert chunk into a segment, coalescing with
! 923: * previous segment if possible.
! 924: */
! 925: if (first) {
! 926: map->dm_segs[seg].ds_addr = curaddr;
! 927: map->dm_segs[seg].ds_len = sgsize;
! 928: first = 0;
! 929: } else {
! 930: if (curaddr == lastaddr &&
! 931: (map->dm_segs[seg].ds_len + sgsize) <=
! 932: map->_dm_maxsegsz &&
! 933: (map->_dm_boundary == 0 ||
! 934: (map->dm_segs[seg].ds_addr & bmask) ==
! 935: (curaddr & bmask)))
! 936: map->dm_segs[seg].ds_len += sgsize;
! 937: else {
! 938: if (++seg >= map->_dm_segcnt)
! 939: break;
! 940: map->dm_segs[seg].ds_addr = curaddr;
! 941: map->dm_segs[seg].ds_len = sgsize;
! 942: }
! 943: }
! 944:
! 945: lastaddr = curaddr + sgsize;
! 946: vaddr += sgsize;
! 947: buflen -= sgsize;
! 948: }
! 949:
! 950: *segp = seg;
! 951: *lastaddrp = lastaddr;
! 952:
! 953: /*
! 954: * Did we fit?
! 955: */
! 956: if (buflen != 0)
! 957: return (EFBIG); /* XXX better return value here? */
! 958: return (0);
! 959: }
! 960:
! 961: /*
! 962: * Allocate physical memory from the given physical address range.
! 963: * Called by DMA-safe memory allocation methods.
! 964: */
! 965: int
! 966: _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
! 967: bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
! 968: int flags, paddr_t low, paddr_t high)
! 969: {
! 970: paddr_t curaddr, lastaddr;
! 971: struct vm_page *m;
! 972: struct pglist mlist;
! 973: int curseg, error;
! 974:
! 975: #ifdef DEBUG_DMA
! 976: printf("alloc_range: t=%p size=%lx align=%lx boundary=%lx segs=%p nsegs=%x rsegs=%p flags=%x lo=%lx hi=%lx\n",
! 977: t, size, alignment, boundary, segs, nsegs, rsegs, flags, low, high);
! 978: #endif /* DEBUG_DMA */
! 979:
! 980: /* Always round the size. */
! 981: size = round_page(size);
! 982:
! 983: TAILQ_INIT(&mlist);
! 984: /*
! 985: * Allocate pages from the VM system.
! 986: */
! 987: error = uvm_pglistalloc(size, low, high, alignment, boundary,
! 988: &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
! 989: if (error)
! 990: return (error);
! 991:
! 992: /*
! 993: * Compute the location, size, and number of segments actually
! 994: * returned by the VM code.
! 995: */
! 996: m = TAILQ_FIRST(&mlist);
! 997: curseg = 0;
! 998: lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m);
! 999: segs[curseg].ds_len = PAGE_SIZE;
! 1000: #ifdef DEBUG_DMA
! 1001: printf("alloc: page %lx\n", lastaddr);
! 1002: #endif /* DEBUG_DMA */
! 1003: m = TAILQ_NEXT(m, pageq);
! 1004:
! 1005: for (; m != TAILQ_END(&mlist); m = TAILQ_NEXT(m, pageq)) {
! 1006: curaddr = VM_PAGE_TO_PHYS(m);
! 1007: #ifdef DIAGNOSTIC
! 1008: if (curaddr < low || curaddr >= high) {
! 1009: printf("uvm_pglistalloc returned non-sensical"
! 1010: " address 0x%lx\n", curaddr);
! 1011: panic("_bus_dmamem_alloc_range");
! 1012: }
! 1013: #endif /* DIAGNOSTIC */
! 1014: #ifdef DEBUG_DMA
! 1015: printf("alloc: page %lx\n", curaddr);
! 1016: #endif /* DEBUG_DMA */
! 1017: if (curaddr == (lastaddr + PAGE_SIZE))
! 1018: segs[curseg].ds_len += PAGE_SIZE;
! 1019: else {
! 1020: curseg++;
! 1021: segs[curseg].ds_addr = curaddr;
! 1022: segs[curseg].ds_len = PAGE_SIZE;
! 1023: }
! 1024: lastaddr = curaddr;
! 1025: }
! 1026:
! 1027: *rsegs = curseg + 1;
! 1028:
! 1029: return (0);
! 1030: }
! 1031:
! 1032: /*
! 1033: * Check if a memory region intersects with a DMA range, and return the
! 1034: * page-rounded intersection if it does.
! 1035: */
! 1036: int
! 1037: arm32_dma_range_intersect(struct arm32_dma_range *ranges, int nranges,
! 1038: paddr_t pa, psize_t size, paddr_t *pap, psize_t *sizep)
! 1039: {
! 1040: struct arm32_dma_range *dr;
! 1041: int i;
! 1042:
! 1043: if (ranges == NULL)
! 1044: return (0);
! 1045:
! 1046: for (i = 0, dr = ranges; i < nranges; i++, dr++) {
! 1047: if (dr->dr_sysbase <= pa &&
! 1048: pa < (dr->dr_sysbase + dr->dr_len)) {
! 1049: /*
! 1050: * Beginning of region intersects with this range.
! 1051: */
! 1052: *pap = trunc_page(pa);
! 1053: *sizep = round_page(min(pa + size,
! 1054: dr->dr_sysbase + dr->dr_len) - pa);
! 1055: return (1);
! 1056: }
! 1057: if (pa < dr->dr_sysbase && dr->dr_sysbase < (pa + size)) {
! 1058: /*
! 1059: * End of region intersects with this range.
! 1060: */
! 1061: *pap = trunc_page(dr->dr_sysbase);
! 1062: *sizep = round_page(min((pa + size) - dr->dr_sysbase,
! 1063: dr->dr_len));
! 1064: return (1);
! 1065: }
! 1066: }
! 1067:
! 1068: /* No intersection found. */
! 1069: return (0);
! 1070: }
! 1071:
! 1072: /*
! 1073: * probably should be ppc_space_copy
! 1074: */
! 1075:
! 1076: #define _CONCAT(A,B) A ## B
! 1077: #define __C(A,B) _CONCAT(A,B)
! 1078:
! 1079: #define BUS_SPACE_READ_RAW_MULTI_N(BYTES,SHIFT,TYPE) \
! 1080: void \
! 1081: __C(bus_space_read_raw_multi_,BYTES)(bus_space_tag_t bst, \
! 1082: bus_space_handle_t h, bus_addr_t o, u_int8_t *dst, bus_size_t size) \
! 1083: { \
! 1084: TYPE *rdst = (TYPE *)dst; \
! 1085: int i; \
! 1086: int count = size >> SHIFT; \
! 1087: \
! 1088: for (i = 0; i < count; i++) { \
! 1089: rdst[i] = __bs_rs(BYTES, bst, h, o); \
! 1090: } \
! 1091: }
! 1092: BUS_SPACE_READ_RAW_MULTI_N(2,1,u_int16_t)
! 1093: BUS_SPACE_READ_RAW_MULTI_N(4,2,u_int32_t)
! 1094:
! 1095: #define BUS_SPACE_WRITE_RAW_MULTI_N(BYTES,SHIFT,TYPE) \
! 1096: void \
! 1097: __C(bus_space_write_raw_multi_,BYTES)( bus_space_tag_t bst, \
! 1098: bus_space_handle_t h, bus_addr_t o, const u_int8_t *src, \
! 1099: bus_size_t size) \
! 1100: { \
! 1101: int i; \
! 1102: TYPE *rsrc = (TYPE *)src; \
! 1103: int count = size >> SHIFT; \
! 1104: \
! 1105: for (i = 0; i < count; i++) { \
! 1106: __bs_ws(BYTES, bst, h, o, rsrc[i]); \
! 1107: } \
! 1108: }
! 1109:
! 1110: BUS_SPACE_WRITE_RAW_MULTI_N(2,1,u_int16_t)
! 1111: BUS_SPACE_WRITE_RAW_MULTI_N(4,2,u_int32_t)
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to bus_dma.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / arm / arm