Annotation of sys/arch/sparc64/dev/iommu.c, Revision 1.1.1.1
1.1 nbrk 1: /* $OpenBSD: iommu.c,v 1.47 2007/05/29 09:53:59 sobrado Exp $ */
2: /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */
3:
4: /*
5: * Copyright (c) 2003 Henric Jungheim
6: * Copyright (c) 2001, 2002 Eduardo Horvath
7: * Copyright (c) 1999, 2000 Matthew R. Green
8: * All rights reserved.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. The name of the author may not be used to endorse or promote products
19: * derived from this software without specific prior written permission.
20: *
21: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
22: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
23: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
25: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26: * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27: * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28: * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29: * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31: * SUCH DAMAGE.
32: */
33:
34: /*
35: * UltraSPARC IOMMU support; used by both the sbus and pci code.
36: */
37: #include <sys/param.h>
38: #include <sys/extent.h>
39: #include <sys/malloc.h>
40: #include <sys/systm.h>
41: #include <sys/device.h>
42: #include <sys/mbuf.h>
43:
44: #include <uvm/uvm_extern.h>
45:
46: #include <machine/bus.h>
47: #include <sparc64/sparc64/cache.h>
48: #include <sparc64/dev/iommureg.h>
49: #include <sparc64/dev/iommuvar.h>
50:
51: #include <machine/autoconf.h>
52: #include <machine/cpu.h>
53:
54: #ifdef DDB
55: #include <machine/db_machdep.h>
56: #include <ddb/db_sym.h>
57: #include <ddb/db_extern.h>
58: #endif
59:
60: #ifdef DEBUG
61: #define IDB_BUSDMA 0x1
62: #define IDB_IOMMU 0x2
63: #define IDB_INFO 0x4
64: #define IDB_SYNC 0x8
65: #define IDB_XXX 0x10
66: #define IDB_PRINT_MAP 0x20
67: #define IDB_BREAK 0x40
68: int iommudebug = IDB_INFO;
69: #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0)
70: #else
71: #define DPRINTF(l, s)
72: #endif
73:
74: void iommu_enter(struct iommu_state *, struct strbuf_ctl *, vaddr_t, paddr_t,
75: int);
76: void iommu_remove(struct iommu_state *, struct strbuf_ctl *, vaddr_t);
77: int iommu_dvmamap_sync_range(struct strbuf_ctl*, vaddr_t, bus_size_t);
78: int iommu_strbuf_flush_done(struct iommu_map_state *);
79: int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *,
80: bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t);
81: int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *,
82: bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t);
83: int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *,
84: bus_dmamap_t);
85: void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *,
86: bus_dmamap_t);
87: int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t,
88: bus_size_t, int, bus_size_t);
89: int64_t iommu_tsb_entry(struct iommu_state *, vaddr_t);
90: void strbuf_reset(struct strbuf_ctl *);
91: int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t);
92: vaddr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t);
93: int iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *,
94: vaddr_t, int);
95: int iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *);
96: struct iommu_map_state *iommu_iomap_create(int);
97: void iommu_iomap_destroy(struct iommu_map_state *);
98: void iommu_iomap_clear_pages(struct iommu_map_state *);
99: void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
100: bus_addr_t, bus_size_t, int);
101:
102: /*
103: * Initiate an STC entry flush.
104: */
105: static inline void
106: iommu_strbuf_flush(struct strbuf_ctl *sb, vaddr_t va)
107: {
108: #ifdef DEBUG
109: if (sb->sb_flush == NULL) {
110: printf("iommu_strbuf_flush: attempting to flush w/o STC\n");
111: return;
112: }
113: #endif
114:
115: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
116: STRBUFREG(strbuf_pgflush), va);
117: }
118:
119: /*
120: * initialise the UltraSPARC IOMMU (SBus or PCI):
121: * - allocate and setup the iotsb.
122: * - enable the IOMMU
123: * - initialise the streaming buffers (if they exist)
124: * - create a private DVMA map.
125: */
126: void
127: iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
128: {
129: psize_t size;
130: vaddr_t va;
131: paddr_t pa;
132: struct vm_page *m;
133: struct pglist mlist;
134:
135: /*
136: * Setup the iommu.
137: *
138: * The sun4u iommu is part of the SBus or PCI controller so we will
139: * deal with it here..
140: *
141: * For sysio and psycho/psycho+ the IOMMU address space always ends at
142: * 0xffffe000, but the starting address depends on the size of the
143: * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each
144: * entry is 8 bytes. The start of the map can be calculated by
145: * (0xffffe000 << (8 + is->is_tsbsize)).
146: *
147: * But sabre and hummingbird use a different scheme that seems to
148: * be hard-wired, so we read the start and size from the PROM and
149: * just use those values.
150: */
151: is->is_cr = IOMMUCR_EN;
152: is->is_tsbsize = tsbsize;
153: if (iovabase == (u_int32_t)-1) {
154: is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
155: is->is_dvmaend = IOTSB_VEND;
156: } else {
157: is->is_dvmabase = iovabase;
158: is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1;
159: }
160:
161: /*
162: * Allocate memory for I/O pagetables. They need to be physically
163: * contiguous.
164: */
165:
166: size = PAGE_SIZE << is->is_tsbsize;
167: TAILQ_INIT(&mlist);
168: if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
169: (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, 0) != 0)
170: panic("iommu_init: no memory");
171:
172: va = uvm_km_valloc(kernel_map, size);
173: if (va == 0)
174: panic("iommu_init: no memory");
175: is->is_tsb = (int64_t *)va;
176:
177: m = TAILQ_FIRST(&mlist);
178: is->is_ptsb = VM_PAGE_TO_PHYS(m);
179:
180: /* Map the pages */
181: for (; m != NULL; m = TAILQ_NEXT(m,pageq)) {
182: pa = VM_PAGE_TO_PHYS(m);
183: pmap_enter(pmap_kernel(), va, pa | PMAP_NVC,
184: VM_PROT_READ|VM_PROT_WRITE,
185: VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
186: va += PAGE_SIZE;
187: }
188: pmap_update(pmap_kernel());
189: memset(is->is_tsb, 0, size);
190:
191: #ifdef DEBUG
192: if (iommudebug & IDB_INFO) {
193: /* Probe the iommu */
194: /* The address or contents of the regs...? */
195: printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
196: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
197: IOMMUREG(iommu_cr),
198: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
199: IOMMUREG(iommu_tsb),
200: (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) +
201: IOMMUREG(iommu_flush));
202: printf("iommu cr=%llx tsb=%llx\n",
203: IOMMUREG_READ(is, iommu_cr),
204: IOMMUREG_READ(is, iommu_tsb));
205: printf("TSB base %p phys %llx\n",
206: (void *)is->is_tsb, (unsigned long long)is->is_ptsb);
207: delay(1000000); /* 1 s */
208: }
209: #endif
210:
211: /*
212: * Now all the hardware's working we need to allocate a dvma map.
213: */
214: printf("dvma map %x-%x, ", is->is_dvmabase, is->is_dvmaend);
215: printf("iotdb %llx-%llx",
216: (unsigned long long)is->is_ptsb,
217: (unsigned long long)(is->is_ptsb + size));
218: is->is_dvmamap = extent_create(name,
219: is->is_dvmabase, (u_long)is->is_dvmaend + 1,
220: M_DEVBUF, 0, 0, EX_NOWAIT);
221:
222: /*
223: * Set the TSB size. The relevant bits were moved to the TSB
224: * base register in the PCIe host bridges.
225: */
226: if (strncmp(name, "pyro", 4) == 0)
227: is->is_ptsb |= is->is_tsbsize;
228: else
229: is->is_cr |= (is->is_tsbsize << 16);
230:
231: /*
232: * Now actually start up the IOMMU.
233: */
234: iommu_reset(is);
235: printf("\n");
236: }
237:
238: /*
239: * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have
240: * detected that already and disabled them. If not, we will notice that
241: * they aren't there when the STRBUF_EN bit does not remain.
242: */
243: void
244: iommu_reset(struct iommu_state *is)
245: {
246: int i;
247:
248: IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
249:
250: /* Enable IOMMU */
251: IOMMUREG_WRITE(is, iommu_cr, is->is_cr);
252:
253: for (i = 0; i < 2; ++i) {
254: struct strbuf_ctl *sb = is->is_sb[i];
255:
256: if (sb == NULL)
257: continue;
258:
259: sb->sb_iommu = is;
260: strbuf_reset(sb);
261:
262: if (sb->sb_flush)
263: printf(", STC%d enabled", i);
264: }
265: }
266:
267: /*
268: * Initialize one STC.
269: */
270: void
271: strbuf_reset(struct strbuf_ctl *sb)
272: {
273: if(sb->sb_flush == NULL)
274: return;
275:
276: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
277: STRBUFREG(strbuf_ctl), STRBUF_EN);
278:
279: membar(Lookaside);
280:
281: /* No streaming buffers? Disable them */
282: if (bus_space_read_8(sb->sb_bustag, sb->sb_sb,
283: STRBUFREG(strbuf_ctl)) == 0) {
284: sb->sb_flush = NULL;
285: } else {
286: /*
287: * locate the pa of the flush buffer
288: */
289: if (pmap_extract(pmap_kernel(),
290: (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE)
291: sb->sb_flush = NULL;
292: }
293: }
294:
295: /*
296: * Add an entry to the IOMMU table.
297: *
298: * The entry is marked streaming if an STC was detected and
299: * the BUS_DMA_STREAMING flag is set.
300: */
301: void
302: iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va,
303: paddr_t pa, int flags)
304: {
305: int64_t tte;
306: volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)];
307:
308: #ifdef DIAGNOSTIC
309: if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
310: panic("iommu_enter: va %#lx not in DVMA space", va);
311:
312: tte = *tte_ptr;
313:
314: if (tte & IOTTE_V) {
315: printf("Overwriting valid tte entry (dva %lx pa %lx "
316: "&tte %p tte %llx)\n", va, pa, tte_ptr, tte);
317: extent_print(is->is_dvmamap);
318: panic("IOMMU overwrite");
319: }
320: #endif
321:
322: tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
323: !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING));
324:
325: DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
326: (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));
327:
328: *tte_ptr = tte;
329:
330: /*
331: * Why bother to flush this va? It should only be relevant for
332: * V ==> V or V ==> non-V transitions. The former is illegal and
333: * the latter is never done here. It is true that this provides
334: * some protection against a misbehaving master using an address
335: * after it should. The IOMMU documentations specifically warns
336: * that the consequences of a simultaneous IOMMU flush and DVMA
337: * access to the same address are undefined. (By that argument,
338: * the STC should probably be flushed as well.) Note that if
339: * a bus master keeps using a memory region after it has been
340: * unmapped, the specific behavior of the IOMMU is likely to
341: * be the least of our worries.
342: */
343: IOMMUREG_WRITE(is, iommu_flush, va);
344:
345: DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
346: va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize),
347: (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
348: (u_long)tte));
349: }
350:
351: /*
352: * Remove an entry from the IOMMU table.
353: *
354: * The entry is flushed from the STC if an STC is detected and the TSB
355: * entry has the IOTTE_STREAM flags set. It should be impossible for
356: * the TSB entry to have this flag set without the BUS_DMA_STREAMING
357: * flag, but better to be safe. (The IOMMU will be ignored as long
358: * as an STC entry exists.)
359: */
360: void
361: iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va)
362: {
363: int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)];
364: int64_t tte;
365:
366: #ifdef DIAGNOSTIC
367: if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend)
368: panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
369: if (va != trunc_page(va)) {
370: printf("iommu_remove: unaligned va: %lx\n", va);
371: va = trunc_page(va);
372: }
373: #endif
374: tte = *tte_ptr;
375:
376: DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n",
377: va, tte, tte_ptr));
378:
379: #ifdef DIAGNOSTIC
380: if ((tte & IOTTE_V) == 0) {
381: printf("Removing invalid tte entry (dva %lx &tte %p "
382: "tte %llx)\n", va, tte_ptr, tte);
383: extent_print(is->is_dvmamap);
384: panic("IOMMU remove overwrite");
385: }
386: #endif
387:
388: *tte_ptr = tte & ~IOTTE_V;
389:
390: /*
391: * IO operations are strongly ordered WRT each other. It is
392: * unclear how they relate to normal memory accesses.
393: */
394: membar(StoreStore);
395:
396: IOMMUREG_WRITE(is, iommu_flush, va);
397:
398: if (sb && (tte & IOTTE_STREAM))
399: iommu_strbuf_flush(sb, va);
400:
401: /* Should we sync the iommu and stc here? */
402: }
403:
404: /*
405: * Find the physical address of a DVMA address (debug routine).
406: */
407: paddr_t
408: iommu_extract(struct iommu_state *is, vaddr_t dva)
409: {
410: int64_t tte = 0;
411:
412: if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
413: tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
414:
415: return (tte & IOTTE_PAMASK);
416: }
417:
418: /*
419: * Lookup a TSB entry for a given DVMA (debug routine).
420: */
421: int64_t
422: iommu_lookup_tte(struct iommu_state *is, vaddr_t dva)
423: {
424: int64_t tte = 0;
425:
426: if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
427: tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
428:
429: return (tte);
430: }
431:
432: /*
433: * Lookup a TSB entry at a given physical address (debug routine).
434: */
435: int64_t
436: iommu_fetch_tte(struct iommu_state *is, paddr_t pa)
437: {
438: int64_t tte = 0;
439:
440: if (pa >= is->is_ptsb && pa < is->is_ptsb +
441: (PAGE_SIZE << is->is_tsbsize))
442: tte = ldxa(pa, ASI_PHYS_CACHED);
443:
444: return (tte);
445: }
446:
447: /*
448: * Fetch a TSB entry with some sanity checking.
449: */
450: int64_t
451: iommu_tsb_entry(struct iommu_state *is, vaddr_t dva)
452: {
453: int64_t tte;
454:
455: if (dva < is->is_dvmabase || dva > is->is_dvmaend)
456: panic("invalid dva: %llx", (long long)dva);
457:
458: tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)];
459:
460: if ((tte & IOTTE_V) == 0)
461: panic("iommu_tsb_entry: invalid entry %lx", dva);
462:
463: return (tte);
464: }
465:
466: /*
467: * Initiate and then block until an STC flush synchronization has completed.
468: */
469: int
470: iommu_strbuf_flush_done(struct iommu_map_state *ims)
471: {
472: struct strbuf_ctl *sb = ims->ims_sb;
473: struct strbuf_flush *sf = &ims->ims_flush;
474: struct timeval cur, flushtimeout;
475: struct timeval to = { 0, 500000 };
476: u_int64_t flush;
477: int timeout_started = 0;
478:
479: #ifdef DIAGNOSTIC
480: if (sb == NULL) {
481: panic("iommu_strbuf_flush_done: invalid flush buffer");
482: }
483: #endif
484:
485: /*
486: * Streaming buffer flushes:
487: *
488: * 1 Tell strbuf to flush by storing va to strbuf_pgflush.
489: * 2 Store 0 in flag
490: * 3 Store pointer to flag in flushsync
491: * 4 wait till flushsync becomes 0x1
492: *
493: * If it takes more than .5 sec, something went very, very wrong.
494: */
495:
496: /*
497: * If we're reading from ASI_PHYS_CACHED, then we'll write to
498: * it too. No need to tempt fate or learn about Si bugs or such.
499: * FreeBSD just uses normal "volatile" reads/writes...
500: */
501:
502: stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0);
503:
504: /*
505: * Insure any previous strbuf operations are complete and that
506: * memory is initialized before the IOMMU uses it.
507: * Is this Needed? How are IO and memory operations ordered?
508: */
509: membar(StoreStore);
510:
511: bus_space_write_8(sb->sb_bustag, sb->sb_sb,
512: STRBUFREG(strbuf_flushsync), sf->sbf_flushpa);
513:
514: DPRINTF(IDB_IOMMU,
515: ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n",
516: ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa));
517:
518: membar(StoreLoad | Lookaside);
519:
520: for(;;) {
521: int i;
522:
523: /*
524: * Try to shave a few instruction cycles off the average
525: * latency by only checking the elapsed time every few
526: * fetches.
527: */
528: for (i = 0; i < 1000; ++i) {
529: membar(LoadLoad);
530: /* Bypass non-coherent D$ */
531: /* non-coherent...? Huh? */
532: flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED);
533:
534: if (flush) {
535: DPRINTF(IDB_IOMMU,
536: ("iommu_strbuf_flush_done: flushed\n"));
537: return (0);
538: }
539: }
540:
541: microtime(&cur);
542:
543: if (timeout_started) {
544: if (timercmp(&cur, &flushtimeout, >))
545: panic("STC timeout at %lx (%lld)",
546: sf->sbf_flushpa, flush);
547: } else {
548: timeradd(&cur, &to, &flushtimeout);
549:
550: timeout_started = 1;
551:
552: DPRINTF(IDB_IOMMU,
553: ("iommu_strbuf_flush_done: flush = %llx pa = %lx "
554: "now=%lx:%lx until = %lx:%lx\n",
555: ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED),
556: sf->sbf_flushpa, cur.tv_sec, cur.tv_usec,
557: flushtimeout.tv_sec, flushtimeout.tv_usec));
558: }
559: }
560: }
561:
562: /*
563: * IOMMU DVMA operations, common to SBus and PCI.
564: */
565:
566: #define BUS_DMA_FIND_PARENT(t, fn) \
567: if (t->_parent == NULL) \
568: panic("null bus_dma parent (" #fn ")"); \
569: for (t = t->_parent; t->fn == NULL; t = t->_parent) \
570: if (t->_parent == NULL) \
571: panic("no bus_dma " #fn " located");
572:
573: int
574: iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb,
575: bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary,
576: int flags, bus_dmamap_t *dmamap)
577: {
578: int ret;
579: bus_dmamap_t map;
580: struct iommu_map_state *ims;
581:
582: BUS_DMA_FIND_PARENT(t, _dmamap_create);
583: ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary,
584: flags, &map);
585:
586: if (ret)
587: return (ret);
588:
589: ims = iommu_iomap_create(atop(round_page(size)));
590:
591: if (ims == NULL) {
592: bus_dmamap_destroy(t0, map);
593: return (ENOMEM);
594: }
595:
596: ims->ims_sb = sb;
597: map->_dm_cookie = ims;
598:
599: #ifdef DIAGNOSTIC
600: if (ims->ims_sb == NULL)
601: panic("iommu_dvmamap_create: null sb");
602: if (ims->ims_sb->sb_iommu == NULL)
603: panic("iommu_dvmamap_create: null iommu");
604: #endif
605: *dmamap = map;
606:
607: return (0);
608: }
609:
610: void
611: iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
612: {
613: /*
614: * The specification (man page) requires a loaded
615: * map to be unloaded before it is destroyed.
616: */
617: if (map->dm_nsegs)
618: bus_dmamap_unload(t0, map);
619:
620: if (map->_dm_cookie)
621: iommu_iomap_destroy(map->_dm_cookie);
622: map->_dm_cookie = NULL;
623:
624: BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
625: (*t->_dmamap_destroy)(t, t0, map);
626: }
627:
628: /*
629: * Load a contiguous kva buffer into a dmamap. The physical pages are
630: * not assumed to be contiguous. Two passes are made through the buffer
631: * and both call pmap_extract() for the same va->pa translations. It
632: * is possible to run out of pa->dvma mappings; the code should be smart
633: * enough to resize the iomap (when the "flags" permit allocation). It
634: * is trivial to compute the number of entries required (round the length
635: * up to the page size and then divide by the page size)...
636: */
637: int
638: iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
639: void *buf, bus_size_t buflen, struct proc *p, int flags)
640: {
641: int s;
642: int err = 0;
643: bus_size_t sgsize;
644: u_long dvmaddr, sgstart, sgend;
645: bus_size_t align, boundary;
646: struct iommu_state *is;
647: struct iommu_map_state *ims = map->_dm_cookie;
648: pmap_t pmap;
649:
650: #ifdef DIAGNOSTIC
651: if (ims == NULL)
652: panic("iommu_dvmamap_load: null map state");
653: #endif
654: #ifdef DEBUG
655: if (ims->ims_sb == NULL)
656: panic("iommu_dvmamap_load: null sb");
657: if (ims->ims_sb->sb_iommu == NULL)
658: panic("iommu_dvmamap_load: null iommu");
659: #endif /* DEBUG */
660: is = ims->ims_sb->sb_iommu;
661:
662: if (map->dm_nsegs) {
663: /*
664: * Is it still in use? _bus_dmamap_load should have taken care
665: * of this.
666: */
667: #ifdef DIAGNOSTIC
668: panic("iommu_dvmamap_load: map still in use");
669: #endif
670: bus_dmamap_unload(t0, map);
671: }
672:
673: /*
674: * Make sure that on error condition we return "no valid mappings".
675: */
676: map->dm_nsegs = 0;
677:
678: if (buflen < 1 || buflen > map->_dm_size) {
679: DPRINTF(IDB_BUSDMA,
680: ("iommu_dvmamap_load(): error %d > %d -- "
681: "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
682: return (EINVAL);
683: }
684:
685: /*
686: * A boundary presented to bus_dmamem_alloc() takes precedence
687: * over boundary in the map.
688: */
689: if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
690: boundary = map->_dm_boundary;
691: align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE);
692:
693: pmap = p ? p->p_vmspace->vm_map.pmap : pmap = pmap_kernel();
694:
695: /* Count up the total number of pages we need */
696: iommu_iomap_clear_pages(ims);
697: { /* Scope */
698: bus_addr_t a, aend;
699: bus_addr_t addr = (vaddr_t)buf;
700: int seg_len = buflen;
701:
702: aend = round_page(addr + seg_len);
703: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
704: paddr_t pa;
705:
706: if (pmap_extract(pmap, a, &pa) == FALSE) {
707: printf("iomap pmap error addr 0x%llx\n", a);
708: iommu_iomap_clear_pages(ims);
709: return (EFBIG);
710: }
711:
712: err = iommu_iomap_insert_page(ims, pa);
713: if (err) {
714: printf("iomap insert error: %d for "
715: "va 0x%llx pa 0x%lx "
716: "(buf %p len %lld/%llx)\n",
717: err, a, pa, buf, buflen, buflen);
718: iommu_dvmamap_print_map(t, is, map);
719: iommu_iomap_clear_pages(ims);
720: return (EFBIG);
721: }
722: }
723: }
724: sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;
725:
726: if (flags & BUS_DMA_24BIT) {
727: sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
728: sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
729: } else {
730: sgstart = is->is_dvmamap->ex_start;
731: sgend = is->is_dvmamap->ex_end;
732: }
733:
734: /*
735: * If our segment size is larger than the boundary we need to
736: * split the transfer up into little pieces ourselves.
737: */
738: s = splhigh();
739: err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend,
740: sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
741: EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
742: splx(s);
743:
744: #ifdef DEBUG
745: if (err || (dvmaddr == (bus_addr_t)-1)) {
746: printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
747: (int)sgsize, flags);
748: #ifdef DDB
749: if (iommudebug & IDB_BREAK)
750: Debugger();
751: #endif
752: }
753: #endif
754: if (err != 0)
755: return (err);
756:
757: if (dvmaddr == (bus_addr_t)-1)
758: return (ENOMEM);
759:
760: /* Set the active DVMA map */
761: map->_dm_dvmastart = dvmaddr;
762: map->_dm_dvmasize = sgsize;
763:
764: map->dm_mapsize = buflen;
765:
766: #ifdef DEBUG
767: iommu_dvmamap_validate_map(t, is, map);
768: #endif
769:
770: if (iommu_iomap_load_map(is, ims, dvmaddr, flags))
771: return (EFBIG);
772:
773: { /* Scope */
774: bus_addr_t a, aend;
775: bus_addr_t addr = (vaddr_t)buf;
776: int seg_len = buflen;
777:
778: aend = round_page(addr + seg_len);
779: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
780: bus_addr_t pgstart;
781: bus_addr_t pgend;
782: paddr_t pa;
783: int pglen;
784:
785: /* Yuck... Redoing the same pmap_extract... */
786: if (pmap_extract(pmap, a, &pa) == FALSE) {
787: printf("iomap pmap error addr 0x%llx\n", a);
788: iommu_iomap_clear_pages(ims);
789: return (EFBIG);
790: }
791:
792: pgstart = pa | (MAX(a, addr) & PAGE_MASK);
793: pgend = pa | (MIN(a + PAGE_SIZE - 1,
794: addr + seg_len - 1) & PAGE_MASK);
795: pglen = pgend - pgstart + 1;
796:
797: if (pglen < 1)
798: continue;
799:
800: err = iommu_dvmamap_append_range(t, map, pgstart,
801: pglen, flags, boundary);
802: if (err == EFBIG)
803: return (err);
804: if (err) {
805: printf("iomap load seg page: %d for "
806: "va 0x%llx pa %lx (%llx - %llx) "
807: "for %d/0x%x\n",
808: err, a, pa, pgstart, pgend, pglen, pglen);
809: return (err);
810: }
811: }
812: }
813:
814: #ifdef DIAGNOSTIC
815: iommu_dvmamap_validate_map(t, is, map);
816: #endif
817:
818: #ifdef DEBUG
819: if (err)
820: printf("**** iommu_dvmamap_load failed with error %d\n",
821: err);
822:
823: if (err || (iommudebug & IDB_PRINT_MAP)) {
824: iommu_dvmamap_print_map(t, is, map);
825: #ifdef DDB
826: if (iommudebug & IDB_BREAK)
827: Debugger();
828: #endif
829: }
830: #endif
831:
832: return (err);
833: }
834:
835: /*
836: * Load a dvmamap from an array of segs or an mlist (if the first
837: * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs()
838: * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the
839: * mapping. This is ugly. A better solution would probably be to have
840: * function pointers for implementing the traversal. That way, there
841: * could be one core load routine for each of the three required algorithms
842: * (buffer, seg, and mlist). That would also mean that the traversal
843: * algorithm would then only need one implementation for each algorithm
844: * instead of two (one for populating the iomap and one for populating
845: * the dvma map).
846: */
847: int
848: iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
849: bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
850: {
851: int i, s;
852: int left;
853: int err = 0;
854: bus_size_t sgsize;
855: bus_size_t boundary, align;
856: u_long dvmaddr, sgstart, sgend;
857: struct iommu_state *is;
858: struct iommu_map_state *ims = map->_dm_cookie;
859:
860: #ifdef DIAGNOSTIC
861: if (ims == NULL)
862: panic("iommu_dvmamap_load_raw: null map state");
863: #endif
864: #ifdef DEBUG
865: if (ims->ims_sb == NULL)
866: panic("iommu_dvmamap_load_raw: null sb");
867: if (ims->ims_sb->sb_iommu == NULL)
868: panic("iommu_dvmamap_load_raw: null iommu");
869: #endif /* DEBUG */
870: is = ims->ims_sb->sb_iommu;
871:
872: if (map->dm_nsegs) {
873: /* Already in use?? */
874: #ifdef DIAGNOSTIC
875: panic("iommu_dvmamap_load_raw: map still in use");
876: #endif
877: bus_dmamap_unload(t0, map);
878: }
879:
880: /*
881: * A boundary presented to bus_dmamem_alloc() takes precedence
882: * over boundary in the map.
883: */
884: if ((boundary = segs[0]._ds_boundary) == 0)
885: boundary = map->_dm_boundary;
886:
887: align = MAX(segs[0]._ds_align, PAGE_SIZE);
888:
889: /*
890: * Make sure that on error condition we return "no valid mappings".
891: */
892: map->dm_nsegs = 0;
893:
894: iommu_iomap_clear_pages(ims);
895: if (segs[0]._ds_mlist) {
896: struct pglist *mlist = segs[0]._ds_mlist;
897: struct vm_page *m;
898: for (m = TAILQ_FIRST(mlist); m != NULL;
899: m = TAILQ_NEXT(m,pageq)) {
900: err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m));
901:
902: if(err) {
903: printf("iomap insert error: %d for "
904: "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m));
905: iommu_dvmamap_print_map(t, is, map);
906: iommu_iomap_clear_pages(ims);
907: return (EFBIG);
908: }
909: }
910: } else {
911: /* Count up the total number of pages we need */
912: for (i = 0, left = size; left > 0 && i < nsegs; i++) {
913: bus_addr_t a, aend;
914: bus_size_t len = segs[i].ds_len;
915: bus_addr_t addr = segs[i].ds_addr;
916: int seg_len = MIN(left, len);
917:
918: if (len < 1)
919: continue;
920:
921: aend = round_page(addr + seg_len);
922: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
923:
924: err = iommu_iomap_insert_page(ims, a);
925: if (err) {
926: printf("iomap insert error: %d for "
927: "pa 0x%llx\n", err, a);
928: iommu_dvmamap_print_map(t, is, map);
929: iommu_iomap_clear_pages(ims);
930: return (EFBIG);
931: }
932: }
933:
934: left -= seg_len;
935: }
936: }
937: sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE;
938:
939: if (flags & BUS_DMA_24BIT) {
940: sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000);
941: sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff);
942: } else {
943: sgstart = is->is_dvmamap->ex_start;
944: sgend = is->is_dvmamap->ex_end;
945: }
946:
947: /*
948: * If our segment size is larger than the boundary we need to
949: * split the transfer up into little pieces ourselves.
950: */
951: s = splhigh();
952: err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend,
953: sgsize, align, 0, (sgsize > boundary) ? 0 : boundary,
954: EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
955: splx(s);
956:
957: if (err != 0)
958: return (err);
959:
960: #ifdef DEBUG
961: if (dvmaddr == (bus_addr_t)-1) {
962: printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) "
963: "failed!\n", (int)sgsize, flags);
964: #ifdef DDB
965: if (iommudebug & IDB_BREAK)
966: Debugger();
967: #else
968: panic("");
969: #endif
970: }
971: #endif
972: if (dvmaddr == (bus_addr_t)-1)
973: return (ENOMEM);
974:
975: /* Set the active DVMA map */
976: map->_dm_dvmastart = dvmaddr;
977: map->_dm_dvmasize = sgsize;
978:
979: map->dm_mapsize = size;
980:
981: #ifdef DEBUG
982: iommu_dvmamap_validate_map(t, is, map);
983: #endif
984:
985: if (iommu_iomap_load_map(is, ims, dvmaddr, flags))
986: return (EFBIG);
987:
988: if (segs[0]._ds_mlist)
989: err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist,
990: flags, size, boundary);
991: else
992: err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs,
993: flags, size, boundary);
994:
995: if (err)
996: iommu_iomap_unload_map(is, ims);
997:
998: #ifdef DIAGNOSTIC
999: /* The map should be valid even if the load failed */
1000: if (iommu_dvmamap_validate_map(t, is, map)) {
1001: printf("load size %lld/0x%llx\n", size, size);
1002: if (segs[0]._ds_mlist)
1003: printf("mlist %p\n", segs[0]._ds_mlist);
1004: else {
1005: long tot_len = 0;
1006: long clip_len = 0;
1007: printf("segs %p nsegs %d\n", segs, nsegs);
1008:
1009: left = size;
1010: for(i = 0; i < nsegs; i++) {
1011: bus_size_t len = segs[i].ds_len;
1012: bus_addr_t addr = segs[i].ds_addr;
1013: int seg_len = MIN(left, len);
1014:
1015: printf("addr %llx len %lld/0x%llx seg_len "
1016: "%d/0x%x left %d/0x%x\n", addr, len, len,
1017: seg_len, seg_len, left, left);
1018:
1019: left -= seg_len;
1020:
1021: clip_len += seg_len;
1022: tot_len += segs[i].ds_len;
1023: }
1024: printf("total length %ld/0x%lx total seg. "
1025: "length %ld/0x%lx\n", tot_len, tot_len, clip_len,
1026: clip_len);
1027: }
1028:
1029: if (err == 0)
1030: err = 1;
1031: }
1032:
1033: #endif
1034:
1035: #ifdef DEBUG
1036: if (err)
1037: printf("**** iommu_dvmamap_load_raw failed with error %d\n",
1038: err);
1039:
1040: if (err || (iommudebug & IDB_PRINT_MAP)) {
1041: iommu_dvmamap_print_map(t, is, map);
1042: #ifdef DDB
1043: if (iommudebug & IDB_BREAK)
1044: Debugger();
1045: #endif
1046: }
1047: #endif
1048:
1049: return (err);
1050: }
1051:
1052: /*
1053: * Insert a range of addresses into a loaded map respecting the specified
1054: * boundary and alignment restrictions. The range is specified by its
1055: * physical address and length. The range cannot cross a page boundary.
1056: * This code (along with most of the rest of the function in this file)
1057: * assumes that the IOMMU page size is equal to PAGE_SIZE.
1058: */
1059: int
1060: iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa,
1061: bus_size_t length, int flags, bus_size_t boundary)
1062: {
1063: struct iommu_map_state *ims = map->_dm_cookie;
1064: bus_addr_t sgstart, sgend, bd_mask;
1065: bus_dma_segment_t *seg = NULL;
1066: int i = map->dm_nsegs;
1067:
1068: #ifdef DEBUG
1069: if (ims == NULL)
1070: panic("iommu_dvmamap_append_range: null map state");
1071: #endif
1072:
1073: sgstart = iommu_iomap_translate(ims, pa);
1074: sgend = sgstart + length - 1;
1075:
1076: #ifdef DIAGNOSTIC
1077: if (sgstart == NULL || sgstart > sgend) {
1078: printf("append range invalid mapping for %lx "
1079: "(0x%llx - 0x%llx)\n", pa, sgstart, sgend);
1080: map->dm_nsegs = 0;
1081: return (EINVAL);
1082: }
1083: #endif
1084:
1085: #ifdef DEBUG
1086: if (trunc_page(sgstart) != trunc_page(sgend)) {
1087: printf("append range crossing page boundary! "
1088: "pa %lx length %lld/0x%llx sgstart %llx sgend %llx\n",
1089: pa, length, length, sgstart, sgend);
1090: }
1091: #endif
1092:
1093: /*
1094: * We will attempt to merge this range with the previous entry
1095: * (if there is one).
1096: */
1097: if (i > 0) {
1098: seg = &map->dm_segs[i - 1];
1099: if (sgstart == seg->ds_addr + seg->ds_len) {
1100: length += seg->ds_len;
1101: sgstart = seg->ds_addr;
1102: sgend = sgstart + length - 1;
1103: } else
1104: seg = NULL;
1105: }
1106:
1107: if (seg == NULL) {
1108: seg = &map->dm_segs[i];
1109: if (++i > map->_dm_segcnt) {
1110: map->dm_nsegs = 0;
1111: return (EFBIG);
1112: }
1113: }
1114:
1115: /*
1116: * At this point, "i" is the index of the *next* bus_dma_segment_t
1117: * (the segment count, aka map->dm_nsegs) and "seg" points to the
1118: * *current* entry. "length", "sgstart", and "sgend" reflect what
1119: * we intend to put in "*seg". No assumptions should be made about
1120: * the contents of "*seg". Only "boundary" issue can change this
1121: * and "boundary" is often zero, so explicitly test for that case
1122: * (the test is strictly an optimization).
1123: */
1124: if (boundary != 0) {
1125: bd_mask = ~(boundary - 1);
1126:
1127: while ((sgstart & bd_mask) != (sgend & bd_mask)) {
1128: /*
1129: * We are crossing a boundary so fill in the current
1130: * segment with as much as possible, then grab a new
1131: * one.
1132: */
1133:
1134: seg->ds_addr = sgstart;
1135: seg->ds_len = boundary - (sgstart & bd_mask);
1136:
1137: sgstart += seg->ds_len; /* sgend stays the same */
1138: length -= seg->ds_len;
1139:
1140: seg = &map->dm_segs[i];
1141: if (++i > map->_dm_segcnt) {
1142: map->dm_nsegs = 0;
1143: return (EFBIG);
1144: }
1145: }
1146: }
1147:
1148: seg->ds_addr = sgstart;
1149: seg->ds_len = length;
1150: map->dm_nsegs = i;
1151:
1152: return (0);
1153: }
1154:
1155: /*
1156: * Populate the iomap from a bus_dma_segment_t array. See note for
1157: * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap.
1158: * This is less of a problem for load_seg, as the number of pages
1159: * is usually similar to the number of segments (nsegs).
1160: */
1161: int
1162: iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is,
1163: bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags,
1164: bus_size_t size, bus_size_t boundary)
1165: {
1166: int i;
1167: int left;
1168: int seg;
1169:
1170: /*
1171: * This segs is made up of individual physical
1172: * segments, probably by _bus_dmamap_load_uio() or
1173: * _bus_dmamap_load_mbuf(). Ignore the mlist and
1174: * load each one individually.
1175: */
1176:
1177: /*
1178: * Keep in mind that each segment could span
1179: * multiple pages and that these are not always
1180: * adjacent. The code is no longer adding dvma
1181: * aliases to the IOMMU. The STC will not cross
1182: * page boundaries anyway and a IOMMU table walk
1183: * vs. what may be a streamed PCI DMA to a ring
1184: * descriptor is probably a wash. It eases TLB
1185: * pressure and in the worst possible case, it is
1186: * only as bad a non-IOMMUed architecture. More
1187: * importantly, the code is not quite as hairy.
1188: * (It's bad enough as it is.)
1189: */
1190: left = size;
1191: seg = 0;
1192: for (i = 0; left > 0 && i < nsegs; i++) {
1193: bus_addr_t a, aend;
1194: bus_size_t len = segs[i].ds_len;
1195: bus_addr_t addr = segs[i].ds_addr;
1196: int seg_len = MIN(left, len);
1197:
1198: if (len < 1)
1199: continue;
1200:
1201: aend = round_page(addr + seg_len);
1202: for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
1203: bus_addr_t pgstart;
1204: bus_addr_t pgend;
1205: int pglen;
1206: int err;
1207:
1208: pgstart = MAX(a, addr);
1209: pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1);
1210: pglen = pgend - pgstart + 1;
1211:
1212: if (pglen < 1)
1213: continue;
1214:
1215: err = iommu_dvmamap_append_range(t, map, pgstart,
1216: pglen, flags, boundary);
1217: if (err == EFBIG)
1218: return (err);
1219: if (err) {
1220: printf("iomap load seg page: %d for "
1221: "pa 0x%llx (%llx - %llx for %d/%x\n",
1222: err, a, pgstart, pgend, pglen, pglen);
1223: return (err);
1224: }
1225:
1226: }
1227:
1228: left -= seg_len;
1229: }
1230: return (0);
1231: }
1232:
1233: /*
1234: * Populate the iomap from an mlist. See note for iommu_dvmamap_load()
1235: * regarding page entry exhaustion of the iomap.
1236: */
1237: int
1238: iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is,
1239: bus_dmamap_t map, struct pglist *mlist, int flags,
1240: bus_size_t size, bus_size_t boundary)
1241: {
1242: struct vm_page *m;
1243: paddr_t pa;
1244: int err;
1245:
1246: /*
1247: * This was allocated with bus_dmamem_alloc.
1248: * The pages are on an `mlist'.
1249: */
1250: for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) {
1251: pa = VM_PAGE_TO_PHYS(m);
1252:
1253: err = iommu_dvmamap_append_range(t, map, pa, PAGE_SIZE,
1254: flags, boundary);
1255: if (err == EFBIG)
1256: return (err);
1257: if (err) {
1258: printf("iomap load seg page: %d for pa 0x%lx "
1259: "(%lx - %lx for %d/%x\n", err, pa, pa,
1260: pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1261: return (err);
1262: }
1263: }
1264:
1265: return (0);
1266: }
1267:
1268: /*
1269: * Unload a dvmamap.
1270: */
1271: void
1272: iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
1273: {
1274: struct iommu_state *is;
1275: struct iommu_map_state *ims = map->_dm_cookie;
1276: bus_addr_t dvmaddr = map->_dm_dvmastart;
1277: bus_size_t sgsize = map->_dm_dvmasize;
1278: int error, s;
1279:
1280: #ifdef DEBUG
1281: if (ims == NULL)
1282: panic("iommu_dvmamap_unload: null map state");
1283: if (ims->ims_sb == NULL)
1284: panic("iommu_dvmamap_unload: null sb");
1285: if (ims->ims_sb->sb_iommu == NULL)
1286: panic("iommu_dvmamap_unload: null iommu");
1287: #endif /* DEBUG */
1288:
1289: is = ims->ims_sb->sb_iommu;
1290:
1291: /* Flush the iommu */
1292: #ifdef DEBUG
1293: if (dvmaddr == 0) {
1294: printf("iommu_dvmamap_unload: No dvmastart\n");
1295: #ifdef DDB
1296: if (iommudebug & IDB_BREAK)
1297: Debugger();
1298: #endif
1299: return;
1300: }
1301: iommu_dvmamap_validate_map(t, is, map);
1302:
1303: if (iommudebug & IDB_PRINT_MAP)
1304: iommu_dvmamap_print_map(t, is, map);
1305: #endif /* DEBUG */
1306:
1307: /* Remove the IOMMU entries */
1308: iommu_iomap_unload_map(is, ims);
1309:
1310: /* Clear the iomap */
1311: iommu_iomap_clear_pages(ims);
1312:
1313: bus_dmamap_unload(t->_parent, map);
1314:
1315: /* Mark the mappings as invalid. */
1316: map->dm_mapsize = 0;
1317: map->dm_nsegs = 0;
1318:
1319: s = splhigh();
1320: error = extent_free(is->is_dvmamap, dvmaddr,
1321: sgsize, EX_NOWAIT);
1322: map->_dm_dvmastart = 0;
1323: map->_dm_dvmasize = 0;
1324: splx(s);
1325: if (error != 0)
1326: printf("warning: %qd of DVMA space lost\n", sgsize);
1327: }
1328:
1329: /*
1330: * Perform internal consistency checking on a dvmamap.
1331: */
1332: int
1333: iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is,
1334: bus_dmamap_t map)
1335: {
1336: int err = 0;
1337: int seg;
1338:
1339: if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) {
1340: printf("**** dvmastart address not page aligned: %llx",
1341: map->_dm_dvmastart);
1342: err = 1;
1343: }
1344: if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) {
1345: printf("**** dvmasize not a multiple of page size: %llx",
1346: map->_dm_dvmasize);
1347: err = 1;
1348: }
1349: if (map->_dm_dvmastart < is->is_dvmabase ||
1350: (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) >
1351: is->is_dvmaend) {
1352: printf("dvmaddr %llx len %llx out of range %x - %x\n",
1353: map->_dm_dvmastart, map->_dm_dvmasize,
1354: is->is_dvmabase, is->is_dvmaend);
1355: err = 1;
1356: }
1357: for (seg = 0; seg < map->dm_nsegs; seg++) {
1358: if (map->dm_segs[seg].ds_addr == 0 ||
1359: map->dm_segs[seg].ds_len == 0) {
1360: printf("seg %d null segment dvmaddr %llx len %llx for "
1361: "range %llx len %llx\n",
1362: seg,
1363: map->dm_segs[seg].ds_addr,
1364: map->dm_segs[seg].ds_len,
1365: map->_dm_dvmastart, map->_dm_dvmasize);
1366: err = 1;
1367: } else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart ||
1368: round_page(map->dm_segs[seg].ds_addr +
1369: map->dm_segs[seg].ds_len) >
1370: map->_dm_dvmastart + map->_dm_dvmasize) {
1371: printf("seg %d dvmaddr %llx len %llx out of "
1372: "range %llx len %llx\n",
1373: seg,
1374: map->dm_segs[seg].ds_addr,
1375: map->dm_segs[seg].ds_len,
1376: map->_dm_dvmastart, map->_dm_dvmasize);
1377: err = 1;
1378: }
1379: }
1380:
1381: if (err) {
1382: iommu_dvmamap_print_map(t, is, map);
1383: #if defined(DDB) && defined(DEBUG)
1384: if (iommudebug & IDB_BREAK)
1385: Debugger();
1386: #endif
1387: }
1388:
1389: return (err);
1390: }
1391:
1392: void
1393: iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is,
1394: bus_dmamap_t map)
1395: {
1396: int seg, i;
1397: long full_len, source_len;
1398: struct mbuf *m;
1399:
1400: printf("DVMA %x for %x, mapping %p: dvstart %llx dvsize %llx "
1401: "size %lld/%llx maxsegsz %llx boundary %llx segcnt %d "
1402: "flags %x type %d source %p "
1403: "cookie %p mapsize %llx nsegs %d\n",
1404: is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map,
1405: map->_dm_dvmastart, map->_dm_dvmasize,
1406: map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary,
1407: map->_dm_segcnt, map->_dm_flags, map->_dm_type,
1408: map->_dm_source, map->_dm_cookie, map->dm_mapsize,
1409: map->dm_nsegs);
1410:
1411: full_len = 0;
1412: for (seg = 0; seg < map->dm_nsegs; seg++) {
1413: printf("seg %d dvmaddr %llx pa %lx len %llx (tte %llx)\n",
1414: seg, map->dm_segs[seg].ds_addr,
1415: is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0,
1416: map->dm_segs[seg].ds_len,
1417: is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0);
1418: full_len += map->dm_segs[seg].ds_len;
1419: }
1420: printf("total length = %ld/0x%lx\n", full_len, full_len);
1421:
1422: if (map->_dm_source) switch (map->_dm_type) {
1423: case _DM_TYPE_MBUF:
1424: m = map->_dm_source;
1425: if (m->m_flags & M_PKTHDR)
1426: printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n",
1427: m, m->m_pkthdr.len, m->m_pkthdr.len);
1428: else
1429: printf("source mbuf (%p):\n", m);
1430:
1431: source_len = 0;
1432: for ( ; m; m = m->m_next) {
1433: vaddr_t vaddr = mtod(m, vaddr_t);
1434: long len = m->m_len;
1435: paddr_t pa;
1436:
1437: if (pmap_extract(pmap_kernel(), vaddr, &pa))
1438: printf("kva %lx pa %lx len %ld/0x%lx\n",
1439: vaddr, pa, len, len);
1440: else
1441: printf("kva %lx pa <invalid> len %ld/0x%lx\n",
1442: vaddr, len, len);
1443:
1444: source_len += len;
1445: }
1446:
1447: if (full_len != source_len)
1448: printf("mbuf length %ld/0x%lx is %s than mapping "
1449: "length %ld/0x%lx\n", source_len, source_len,
1450: (source_len > full_len) ? "greater" : "less",
1451: full_len, full_len);
1452: else
1453: printf("mbuf length %ld/0x%lx\n", source_len,
1454: source_len);
1455: break;
1456: case _DM_TYPE_LOAD:
1457: case _DM_TYPE_SEGS:
1458: case _DM_TYPE_UIO:
1459: default:
1460: break;
1461: }
1462:
1463: if (map->_dm_cookie) {
1464: struct iommu_map_state *ims = map->_dm_cookie;
1465: struct iommu_page_map *ipm = &ims->ims_map;
1466:
1467: printf("page map (%p) of size %d with %d entries\n",
1468: ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt);
1469: for (i = 0; i < ipm->ipm_pagecnt; ++i) {
1470: struct iommu_page_entry *e = &ipm->ipm_map[i];
1471: printf("%d: vmaddr 0x%lx pa 0x%lx\n", i,
1472: e->ipe_va, e->ipe_pa);
1473: }
1474: } else
1475: printf("iommu map state (cookie) is NULL\n");
1476: }
1477:
1478: void
1479: _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
1480: bus_addr_t offset, bus_size_t len, int ops)
1481: {
1482: struct iommu_state *is;
1483: struct iommu_map_state *ims = map->_dm_cookie;
1484: struct strbuf_ctl *sb;
1485: bus_size_t count;
1486: int i, needsflush = 0;
1487:
1488: sb = ims->ims_sb;
1489: is = sb->sb_iommu;
1490:
1491: for (i = 0; i < map->dm_nsegs; i++) {
1492: if (offset < map->dm_segs[i].ds_len)
1493: break;
1494: offset -= map->dm_segs[i].ds_len;
1495: }
1496:
1497: if (i == map->dm_nsegs)
1498: panic("iommu_dvmamap_sync: too short %llu", offset);
1499:
1500: for (; len > 0 && i < map->dm_nsegs; i++) {
1501: count = MIN(map->dm_segs[i].ds_len - offset, len);
1502: if (count > 0 && iommu_dvmamap_sync_range(sb,
1503: map->dm_segs[i].ds_addr + offset, count))
1504: needsflush = 1;
1505: len -= count;
1506: }
1507:
1508: #ifdef DIAGNOSTIC
1509: if (i == map->dm_nsegs && len > 0)
1510: panic("iommu_dvmamap_sync: leftover %llu", len);
1511: #endif
1512:
1513: if (needsflush)
1514: iommu_strbuf_flush_done(ims);
1515: }
1516:
1517: void
1518: iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
1519: bus_addr_t offset, bus_size_t len, int ops)
1520: {
1521: struct iommu_map_state *ims = map->_dm_cookie;
1522:
1523: #ifdef DIAGNOSTIC
1524: if (ims == NULL)
1525: panic("iommu_dvmamap_sync: null map state");
1526: if (ims->ims_sb == NULL)
1527: panic("iommu_dvmamap_sync: null sb");
1528: if (ims->ims_sb->sb_iommu == NULL)
1529: panic("iommu_dvmamap_sync: null iommu");
1530: #endif
1531: if (len == 0)
1532: return;
1533:
1534: if (ops & BUS_DMASYNC_PREWRITE)
1535: membar(MemIssue);
1536:
1537: if ((ims->ims_flags & IOMMU_MAP_STREAM) &&
1538: (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)))
1539: _iommu_dvmamap_sync(t, t0, map, offset, len, ops);
1540:
1541: if (ops & BUS_DMASYNC_POSTREAD)
1542: membar(MemIssue);
1543: }
1544:
1545: /*
1546: * Flush an individual dma segment, returns non-zero if the streaming buffers
1547: * need flushing afterwards.
1548: */
1549: int
1550: iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len)
1551: {
1552: vaddr_t vaend;
1553: #ifdef DIAGNOSTIC
1554: struct iommu_state *is = sb->sb_iommu;
1555:
1556: if (va < is->is_dvmabase || va > is->is_dvmaend)
1557: panic("invalid va: %llx", (long long)va);
1558:
1559: if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) {
1560: printf("iommu_dvmamap_sync_range: attempting to flush "
1561: "non-streaming entry\n");
1562: return (0);
1563: }
1564: #endif
1565:
1566: vaend = (va + len + PAGE_MASK) & ~PAGE_MASK;
1567: va &= ~PAGE_MASK;
1568:
1569: #ifdef DIAGNOSTIC
1570: if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend)
1571: panic("invalid va range: %llx to %llx (%x to %x)",
1572: (long long)va, (long long)vaend,
1573: is->is_dvmabase,
1574: is->is_dvmaend);
1575: #endif
1576:
1577: for ( ; va <= vaend; va += PAGE_SIZE) {
1578: DPRINTF(IDB_BUSDMA,
1579: ("iommu_dvmamap_sync_range: flushing va %p\n",
1580: (void *)(u_long)va));
1581: iommu_strbuf_flush(sb, va);
1582: }
1583:
1584: return (1);
1585: }
1586:
1587: int
1588: iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size,
1589: bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs,
1590: int nsegs, int *rsegs, int flags)
1591: {
1592:
1593: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx "
1594: "bound %llx segp %p flags %d\n", (unsigned long long)size,
1595: (unsigned long long)alignment, (unsigned long long)boundary,
1596: segs, flags));
1597: BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
1598: return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
1599: segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
1600: }
1601:
1602: void
1603: iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs,
1604: int nsegs)
1605: {
1606:
1607: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
1608: segs, nsegs));
1609: BUS_DMA_FIND_PARENT(t, _dmamem_free);
1610: (*t->_dmamem_free)(t, t0, segs, nsegs);
1611: }
1612:
1613: /*
1614: * Map the DVMA mappings into the kernel pmap.
1615: * Check the flags to see whether we're streaming or coherent.
1616: */
1617: int
1618: iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs,
1619: int nsegs, size_t size, caddr_t *kvap, int flags)
1620: {
1621: struct vm_page *m;
1622: vaddr_t va;
1623: bus_addr_t addr;
1624: struct pglist *mlist;
1625: bus_addr_t cbit = 0;
1626:
1627: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
1628: segs, nsegs, size));
1629:
1630: /*
1631: * Allocate some space in the kernel map, and then map these pages
1632: * into this space.
1633: */
1634: size = round_page(size);
1635: va = uvm_km_valloc(kernel_map, size);
1636: if (va == 0)
1637: return (ENOMEM);
1638:
1639: *kvap = (caddr_t)va;
1640:
1641: /*
1642: * digest flags:
1643: */
1644: #if 0
1645: if (flags & BUS_DMA_COHERENT) /* Disable vcache */
1646: cbit |= PMAP_NVC;
1647: #endif
1648: if (flags & BUS_DMA_NOCACHE) /* sideffects */
1649: cbit |= PMAP_NC;
1650:
1651: /*
1652: * Now take this and map it into the CPU.
1653: */
1654: mlist = segs[0]._ds_mlist;
1655: TAILQ_FOREACH(m, mlist, pageq) {
1656: #ifdef DIAGNOSTIC
1657: if (size == 0)
1658: panic("iommu_dvmamem_map: size botch");
1659: #endif
1660: addr = VM_PAGE_TO_PHYS(m);
1661: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
1662: "mapping va %lx at %llx\n", va,
1663: (unsigned long long)addr | cbit));
1664: pmap_enter(pmap_kernel(), va, addr | cbit,
1665: VM_PROT_READ | VM_PROT_WRITE,
1666: VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED);
1667: va += PAGE_SIZE;
1668: size -= PAGE_SIZE;
1669: }
1670: pmap_update(pmap_kernel());
1671:
1672: return (0);
1673: }
1674:
1675: /*
1676: * Unmap DVMA mappings from kernel
1677: */
1678: void
1679: iommu_dvmamem_unmap(bus_dma_tag_t t, bus_dma_tag_t t0, caddr_t kva,
1680: size_t size)
1681: {
1682:
1683: DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
1684: kva, size));
1685:
1686: #ifdef DIAGNOSTIC
1687: if ((u_long)kva & PAGE_MASK)
1688: panic("iommu_dvmamem_unmap");
1689: #endif
1690:
1691: size = round_page(size);
1692: pmap_remove(pmap_kernel(), (vaddr_t)kva, size);
1693: pmap_update(pmap_kernel());
1694: uvm_km_free(kernel_map, (vaddr_t)kva, size);
1695: }
1696:
1697: /*
1698: * Create a new iomap.
1699: */
1700: struct iommu_map_state *
1701: iommu_iomap_create(int n)
1702: {
1703: struct iommu_map_state *ims;
1704: struct strbuf_flush *sbf;
1705: vaddr_t va;
1706:
1707: /* Safety for heavily fragmented data, such as mbufs */
1708: n += 4;
1709: if (n < 16)
1710: n = 16;
1711:
1712: ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]),
1713: M_DEVBUF, M_NOWAIT);
1714: if (ims == NULL)
1715: return (NULL);
1716:
1717: memset(ims, 0, sizeof *ims);
1718:
1719: /* Initialize the map. */
1720: ims->ims_map.ipm_maxpage = n;
1721: SPLAY_INIT(&ims->ims_map.ipm_tree);
1722:
1723: /* Initialize the flush area. */
1724: sbf = &ims->ims_flush;
1725: va = (vaddr_t)&sbf->sbf_area[0x40];
1726: va &= ~0x3f;
1727: pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa);
1728: sbf->sbf_flush = (void *)va;
1729:
1730: return (ims);
1731: }
1732:
1733: /*
1734: * Destroy an iomap.
1735: */
1736: void
1737: iommu_iomap_destroy(struct iommu_map_state *ims)
1738: {
1739: #ifdef DIAGNOSTIC
1740: if (ims->ims_map.ipm_pagecnt > 0)
1741: printf("iommu_iomap_destroy: %d page entries in use\n",
1742: ims->ims_map.ipm_pagecnt);
1743: #endif
1744:
1745: free(ims, M_DEVBUF);
1746: }
1747:
1748: /*
1749: * Utility function used by splay tree to order page entries by pa.
1750: */
1751: static inline int
1752: iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b)
1753: {
1754: return ((a->ipe_pa > b->ipe_pa) ? 1 :
1755: (a->ipe_pa < b->ipe_pa) ? -1 : 0);
1756: }
1757:
1758: SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);
1759:
1760: SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare);
1761:
1762: /*
1763: * Insert a pa entry in the iomap.
1764: */
1765: int
1766: iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa)
1767: {
1768: struct iommu_page_map *ipm = &ims->ims_map;
1769: struct iommu_page_entry *e;
1770:
1771: if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) {
1772: struct iommu_page_entry ipe;
1773:
1774: ipe.ipe_pa = pa;
1775: if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe))
1776: return (0);
1777:
1778: return (ENOMEM);
1779: }
1780:
1781: e = &ipm->ipm_map[ipm->ipm_pagecnt];
1782:
1783: e->ipe_pa = pa;
1784: e->ipe_va = NULL;
1785:
1786: e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e);
1787:
1788: /* Duplicates are okay, but only count them once. */
1789: if (e)
1790: return (0);
1791:
1792: ++ipm->ipm_pagecnt;
1793:
1794: return (0);
1795: }
1796:
1797: /*
1798: * Locate the iomap by filling in the pa->va mapping and inserting it
1799: * into the IOMMU tables.
1800: */
1801: int
1802: iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims,
1803: vaddr_t vmaddr, int flags)
1804: {
1805: struct iommu_page_map *ipm = &ims->ims_map;
1806: struct iommu_page_entry *e;
1807: struct strbuf_ctl *sb = ims->ims_sb;
1808: int i;
1809:
1810: if (sb->sb_flush == NULL)
1811: flags &= ~BUS_DMA_STREAMING;
1812:
1813: if (flags & BUS_DMA_STREAMING)
1814: ims->ims_flags |= IOMMU_MAP_STREAM;
1815: else
1816: ims->ims_flags &= ~IOMMU_MAP_STREAM;
1817:
1818: for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) {
1819: e->ipe_va = vmaddr;
1820: iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags);
1821: vmaddr += PAGE_SIZE;
1822: }
1823:
1824: return (0);
1825: }
1826:
1827: /*
1828: * Remove the iomap from the IOMMU.
1829: */
1830: int
1831: iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims)
1832: {
1833: struct iommu_page_map *ipm = &ims->ims_map;
1834: struct iommu_page_entry *e;
1835: struct strbuf_ctl *sb = ims->ims_sb;
1836: int i;
1837:
1838: for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e)
1839: iommu_remove(is, sb, e->ipe_va);
1840:
1841: return (0);
1842: }
1843:
1844: /*
1845: * Translate a physical address (pa) into a DVMA address.
1846: */
1847: vaddr_t
1848: iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa)
1849: {
1850: struct iommu_page_map *ipm = &ims->ims_map;
1851: struct iommu_page_entry *e;
1852: struct iommu_page_entry pe;
1853: paddr_t offset = pa & PAGE_MASK;
1854:
1855: pe.ipe_pa = trunc_page(pa);
1856:
1857: e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe);
1858:
1859: if (e == NULL)
1860: return (NULL);
1861:
1862: return (e->ipe_va | offset);
1863: }
1864:
1865: /*
1866: * Clear the iomap table and tree.
1867: */
1868: void
1869: iommu_iomap_clear_pages(struct iommu_map_state *ims)
1870: {
1871: ims->ims_map.ipm_pagecnt = 0;
1872: SPLAY_INIT(&ims->ims_map.ipm_tree);
1873: }
1874:
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to iommu.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / arch / sparc64 / dev