Annotation of sys/kern/subr_pool.c, Revision 1.1.1.1
1.1 nbrk 1: /* $OpenBSD: subr_pool.c,v 1.55 2007/08/16 15:18:54 art Exp $ */
2: /* $NetBSD: subr_pool.c,v 1.61 2001/09/26 07:14:56 chs Exp $ */
3:
4: /*-
5: * Copyright (c) 1997, 1999, 2000 The NetBSD Foundation, Inc.
6: * All rights reserved.
7: *
8: * This code is derived from software contributed to The NetBSD Foundation
9: * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
10: * Simulation Facility, NASA Ames Research Center.
11: *
12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: * 3. All advertising materials mentioning features or use of this software
21: * must display the following acknowledgement:
22: * This product includes software developed by the NetBSD
23: * Foundation, Inc. and its contributors.
24: * 4. Neither the name of The NetBSD Foundation nor the names of its
25: * contributors may be used to endorse or promote products derived
26: * from this software without specific prior written permission.
27: *
28: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
29: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
30: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
31: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
32: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38: * POSSIBILITY OF SUCH DAMAGE.
39: */
40:
41: #include <sys/param.h>
42: #include <sys/systm.h>
43: #include <sys/proc.h>
44: #include <sys/errno.h>
45: #include <sys/kernel.h>
46: #include <sys/malloc.h>
47: #include <sys/lock.h>
48: #include <sys/pool.h>
49: #include <sys/syslog.h>
50: #include <sys/sysctl.h>
51:
52: #include <uvm/uvm.h>
53:
54: /*
55: * XXX - for now.
56: */
57: #ifdef LOCKDEBUG
58: #define simple_lock_freecheck(a, s) do { /* nothing */ } while (0)
59: #define simple_lock_only_held(lkp, str) do { /* nothing */ } while (0)
60: #endif
61:
62: /*
63: * Pool resource management utility.
64: *
65: * Memory is allocated in pages which are split into pieces according to
66: * the pool item size. Each page is kept on one of three lists in the
67: * pool structure: `pr_emptypages', `pr_fullpages' and `pr_partpages',
68: * for empty, full and partially-full pages respectively. The individual
69: * pool items are on a linked list headed by `ph_itemlist' in each page
70: * header. The memory for building the page list is either taken from
71: * the allocated pages themselves (for small pool items) or taken from
72: * an internal pool of page headers (`phpool').
73: */
74:
75: /* List of all pools */
76: TAILQ_HEAD(,pool) pool_head = TAILQ_HEAD_INITIALIZER(pool_head);
77:
78: /* Private pool for page header structures */
79: static struct pool phpool;
80:
81: /* This spin lock protects both pool_head */
82: struct simplelock pool_head_slock;
83:
84: struct pool_item_header {
85: /* Page headers */
86: LIST_ENTRY(pool_item_header)
87: ph_pagelist; /* pool page list */
88: TAILQ_HEAD(,pool_item) ph_itemlist; /* chunk list for this page */
89: SPLAY_ENTRY(pool_item_header)
90: ph_node; /* Off-page page headers */
91: int ph_nmissing; /* # of chunks in use */
92: caddr_t ph_page; /* this page's address */
93: };
94:
95: struct pool_item {
96: #ifdef DIAGNOSTIC
97: int pi_magic;
98: #endif
99: #ifdef DEADBEEF1
100: #define PI_MAGIC DEADBEEF1
101: #else
102: #define PI_MAGIC 0xdeafbeef
103: #endif
104: /* Other entries use only this list entry */
105: TAILQ_ENTRY(pool_item) pi_list;
106: };
107:
108: #define POOL_NEEDS_CATCHUP(pp) \
109: ((pp)->pr_nitems < (pp)->pr_minitems)
110:
111: /*
112: * Every pool gets a unique serial number assigned to it. If this counter
113: * wraps, we're screwed, but we shouldn't create so many pools anyway.
114: */
115: unsigned int pool_serial;
116:
117: /*
118: * Pool cache management.
119: *
120: * Pool caches provide a way for constructed objects to be cached by the
121: * pool subsystem. This can lead to performance improvements by avoiding
122: * needless object construction/destruction; it is deferred until absolutely
123: * necessary.
124: *
125: * Caches are grouped into cache groups. Each cache group references
126: * up to 16 constructed objects. When a cache allocates an object
127: * from the pool, it calls the object's constructor and places it into
128: * a cache group. When a cache group frees an object back to the pool,
129: * it first calls the object's destructor. This allows the object to
130: * persist in constructed form while freed to the cache.
131: *
132: * Multiple caches may exist for each pool. This allows a single
133: * object type to have multiple constructed forms. The pool references
134: * each cache, so that when a pool is drained by the pagedaemon, it can
135: * drain each individual cache as well. Each time a cache is drained,
136: * the most idle cache group is freed to the pool in its entirety.
137: *
138: * Pool caches are layed on top of pools. By layering them, we can avoid
139: * the complexity of cache management for pools which would not benefit
140: * from it.
141: */
142:
143: /* The cache group pool. */
144: static struct pool pcgpool;
145:
146: /* The pool cache group. */
147: #define PCG_NOBJECTS 16
148: struct pool_cache_group {
149: TAILQ_ENTRY(pool_cache_group)
150: pcg_list; /* link in the pool cache's group list */
151: u_int pcg_avail; /* # available objects */
152: /* pointers to the objects */
153: void *pcg_objects[PCG_NOBJECTS];
154: };
155:
156: void pool_cache_reclaim(struct pool_cache *);
157: void pool_cache_do_invalidate(struct pool_cache *, int,
158: void (*)(struct pool *, void *));
159:
160: int pool_catchup(struct pool *);
161: void pool_prime_page(struct pool *, caddr_t, struct pool_item_header *);
162: void pool_update_curpage(struct pool *);
163: void pool_do_put(struct pool *, void *);
164: void pr_rmpage(struct pool *, struct pool_item_header *,
165: struct pool_pagelist *);
166: int pool_chk_page(struct pool *, const char *, struct pool_item_header *);
167:
168: void *pool_allocator_alloc(struct pool *, int);
169: void pool_allocator_free(struct pool *, void *);
170:
171: #ifdef DDB
172: void pool_print_pagelist(struct pool_pagelist *, int (*)(const char *, ...));
173: void pool_print1(struct pool *, const char *, int (*)(const char *, ...));
174: #endif
175:
176:
177: /*
178: * Pool log entry. An array of these is allocated in pool_init().
179: */
180: struct pool_log {
181: const char *pl_file;
182: long pl_line;
183: int pl_action;
184: #define PRLOG_GET 1
185: #define PRLOG_PUT 2
186: void *pl_addr;
187: };
188:
189: /* Number of entries in pool log buffers */
190: #ifndef POOL_LOGSIZE
191: #define POOL_LOGSIZE 10
192: #endif
193:
194: int pool_logsize = POOL_LOGSIZE;
195:
196: #ifdef POOL_DIAGNOSTIC
197: static __inline void
198: pr_log(struct pool *pp, void *v, int action, const char *file, long line)
199: {
200: int n = pp->pr_curlogentry;
201: struct pool_log *pl;
202:
203: if ((pp->pr_roflags & PR_LOGGING) == 0)
204: return;
205:
206: /*
207: * Fill in the current entry. Wrap around and overwrite
208: * the oldest entry if necessary.
209: */
210: pl = &pp->pr_log[n];
211: pl->pl_file = file;
212: pl->pl_line = line;
213: pl->pl_action = action;
214: pl->pl_addr = v;
215: if (++n >= pp->pr_logsize)
216: n = 0;
217: pp->pr_curlogentry = n;
218: }
219:
220: static void
221: pr_printlog(struct pool *pp, struct pool_item *pi,
222: int (*pr)(const char *, ...))
223: {
224: int i = pp->pr_logsize;
225: int n = pp->pr_curlogentry;
226:
227: if ((pp->pr_roflags & PR_LOGGING) == 0)
228: return;
229:
230: /*
231: * Print all entries in this pool's log.
232: */
233: while (i-- > 0) {
234: struct pool_log *pl = &pp->pr_log[n];
235: if (pl->pl_action != 0) {
236: if (pi == NULL || pi == pl->pl_addr) {
237: (*pr)("\tlog entry %d:\n", i);
238: (*pr)("\t\taction = %s, addr = %p\n",
239: pl->pl_action == PRLOG_GET ? "get" : "put",
240: pl->pl_addr);
241: (*pr)("\t\tfile: %s at line %lu\n",
242: pl->pl_file, pl->pl_line);
243: }
244: }
245: if (++n >= pp->pr_logsize)
246: n = 0;
247: }
248: }
249:
250: static __inline void
251: pr_enter(struct pool *pp, const char *file, long line)
252: {
253:
254: if (__predict_false(pp->pr_entered_file != NULL)) {
255: printf("pool %s: reentrancy at file %s line %ld\n",
256: pp->pr_wchan, file, line);
257: printf(" previous entry at file %s line %ld\n",
258: pp->pr_entered_file, pp->pr_entered_line);
259: panic("pr_enter");
260: }
261:
262: pp->pr_entered_file = file;
263: pp->pr_entered_line = line;
264: }
265:
266: static __inline void
267: pr_leave(struct pool *pp)
268: {
269:
270: if (__predict_false(pp->pr_entered_file == NULL)) {
271: printf("pool %s not entered?\n", pp->pr_wchan);
272: panic("pr_leave");
273: }
274:
275: pp->pr_entered_file = NULL;
276: pp->pr_entered_line = 0;
277: }
278:
279: static __inline void
280: pr_enter_check(struct pool *pp, int (*pr)(const char *, ...))
281: {
282:
283: if (pp->pr_entered_file != NULL)
284: (*pr)("\n\tcurrently entered from file %s line %ld\n",
285: pp->pr_entered_file, pp->pr_entered_line);
286: }
287: #else
288: #define pr_log(pp, v, action, file, line)
289: #define pr_printlog(pp, pi, pr)
290: #define pr_enter(pp, file, line)
291: #define pr_leave(pp)
292: #define pr_enter_check(pp, pr)
293: #endif /* POOL_DIAGNOSTIC */
294:
295: static __inline int
296: phtree_compare(struct pool_item_header *a, struct pool_item_header *b)
297: {
298: if (a->ph_page < b->ph_page)
299: return (-1);
300: else if (a->ph_page > b->ph_page)
301: return (1);
302: else
303: return (0);
304: }
305:
306: SPLAY_PROTOTYPE(phtree, pool_item_header, ph_node, phtree_compare);
307: SPLAY_GENERATE(phtree, pool_item_header, ph_node, phtree_compare);
308:
309: /*
310: * Return the pool page header based on page address.
311: */
312: static __inline struct pool_item_header *
313: pr_find_pagehead(struct pool *pp, caddr_t page)
314: {
315: struct pool_item_header *ph, tmp;
316:
317: if ((pp->pr_roflags & PR_PHINPAGE) != 0)
318: return ((struct pool_item_header *)(page + pp->pr_phoffset));
319:
320: tmp.ph_page = page;
321: ph = SPLAY_FIND(phtree, &pp->pr_phtree, &tmp);
322: return ph;
323: }
324:
325: /*
326: * Remove a page from the pool.
327: */
328: void
329: pr_rmpage(struct pool *pp, struct pool_item_header *ph,
330: struct pool_pagelist *pq)
331: {
332: int s;
333:
334: /*
335: * If the page was idle, decrement the idle page count.
336: */
337: if (ph->ph_nmissing == 0) {
338: #ifdef DIAGNOSTIC
339: if (pp->pr_nidle == 0)
340: panic("pr_rmpage: nidle inconsistent");
341: if (pp->pr_nitems < pp->pr_itemsperpage)
342: panic("pr_rmpage: nitems inconsistent");
343: #endif
344: pp->pr_nidle--;
345: }
346:
347: pp->pr_nitems -= pp->pr_itemsperpage;
348:
349: /*
350: * Unlink a page from the pool and release it (or queue it for release).
351: */
352: LIST_REMOVE(ph, ph_pagelist);
353: if (pq) {
354: LIST_INSERT_HEAD(pq, ph, ph_pagelist);
355: } else {
356: pool_allocator_free(pp, ph->ph_page);
357: if ((pp->pr_roflags & PR_PHINPAGE) == 0) {
358: SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
359: s = splhigh();
360: pool_put(&phpool, ph);
361: splx(s);
362: }
363: }
364: pp->pr_npages--;
365: pp->pr_npagefree++;
366:
367: pool_update_curpage(pp);
368: }
369:
370: /*
371: * Initialize the given pool resource structure.
372: *
373: * We export this routine to allow other kernel parts to declare
374: * static pools that must be initialized before malloc() is available.
375: */
376: void
377: pool_init(struct pool *pp, size_t size, u_int align, u_int ioff, int flags,
378: const char *wchan, struct pool_allocator *palloc)
379: {
380: int off, slack;
381:
382: #ifdef POOL_DIAGNOSTIC
383: /*
384: * Always log if POOL_DIAGNOSTIC is defined.
385: */
386: if (pool_logsize != 0)
387: flags |= PR_LOGGING;
388: #endif
389:
390: #ifdef MALLOC_DEBUG
391: if ((flags & PR_DEBUG) && (ioff != 0 || align != 0))
392: flags &= ~PR_DEBUG;
393: #endif
394: /*
395: * Check arguments and construct default values.
396: */
397: if (palloc == NULL)
398: palloc = &pool_allocator_nointr;
399: if ((palloc->pa_flags & PA_INITIALIZED) == 0) {
400: if (palloc->pa_pagesz == 0)
401: palloc->pa_pagesz = PAGE_SIZE;
402:
403: TAILQ_INIT(&palloc->pa_list);
404:
405: simple_lock_init(&palloc->pa_slock);
406: palloc->pa_pagemask = ~(palloc->pa_pagesz - 1);
407: palloc->pa_pageshift = ffs(palloc->pa_pagesz) - 1;
408: palloc->pa_flags |= PA_INITIALIZED;
409: }
410:
411: if (align == 0)
412: align = ALIGN(1);
413:
414: if (size < sizeof(struct pool_item))
415: size = sizeof(struct pool_item);
416:
417: size = roundup(size, align);
418: #ifdef DIAGNOSTIC
419: if (size > palloc->pa_pagesz)
420: panic("pool_init: pool item size (%lu) too large",
421: (u_long)size);
422: #endif
423:
424: /*
425: * Initialize the pool structure.
426: */
427: LIST_INIT(&pp->pr_emptypages);
428: LIST_INIT(&pp->pr_fullpages);
429: LIST_INIT(&pp->pr_partpages);
430: TAILQ_INIT(&pp->pr_cachelist);
431: pp->pr_curpage = NULL;
432: pp->pr_npages = 0;
433: pp->pr_minitems = 0;
434: pp->pr_minpages = 0;
435: pp->pr_maxpages = 8;
436: pp->pr_roflags = flags;
437: pp->pr_flags = 0;
438: pp->pr_size = size;
439: pp->pr_align = align;
440: pp->pr_wchan = wchan;
441: pp->pr_alloc = palloc;
442: pp->pr_nitems = 0;
443: pp->pr_nout = 0;
444: pp->pr_hardlimit = UINT_MAX;
445: pp->pr_hardlimit_warning = NULL;
446: pp->pr_hardlimit_ratecap.tv_sec = 0;
447: pp->pr_hardlimit_ratecap.tv_usec = 0;
448: pp->pr_hardlimit_warning_last.tv_sec = 0;
449: pp->pr_hardlimit_warning_last.tv_usec = 0;
450: pp->pr_serial = ++pool_serial;
451: if (pool_serial == 0)
452: panic("pool_init: too much uptime");
453:
454: /*
455: * Decide whether to put the page header off page to avoid
456: * wasting too large a part of the page. Off-page page headers
457: * go on a hash table, so we can match a returned item
458: * with its header based on the page address.
459: * We use 1/16 of the page size as the threshold (XXX: tune)
460: */
461: if (pp->pr_size < palloc->pa_pagesz/16) {
462: /* Use the end of the page for the page header */
463: pp->pr_roflags |= PR_PHINPAGE;
464: pp->pr_phoffset = off = palloc->pa_pagesz -
465: ALIGN(sizeof(struct pool_item_header));
466: } else {
467: /* The page header will be taken from our page header pool */
468: pp->pr_phoffset = 0;
469: off = palloc->pa_pagesz;
470: SPLAY_INIT(&pp->pr_phtree);
471: }
472:
473: /*
474: * Alignment is to take place at `ioff' within the item. This means
475: * we must reserve up to `align - 1' bytes on the page to allow
476: * appropriate positioning of each item.
477: *
478: * Silently enforce `0 <= ioff < align'.
479: */
480: pp->pr_itemoffset = ioff = ioff % align;
481: pp->pr_itemsperpage = (off - ((align - ioff) % align)) / pp->pr_size;
482: KASSERT(pp->pr_itemsperpage != 0);
483:
484: /*
485: * Use the slack between the chunks and the page header
486: * for "cache coloring".
487: */
488: slack = off - pp->pr_itemsperpage * pp->pr_size;
489: pp->pr_maxcolor = (slack / align) * align;
490: pp->pr_curcolor = 0;
491:
492: pp->pr_nget = 0;
493: pp->pr_nfail = 0;
494: pp->pr_nput = 0;
495: pp->pr_npagealloc = 0;
496: pp->pr_npagefree = 0;
497: pp->pr_hiwat = 0;
498: pp->pr_nidle = 0;
499:
500: #ifdef POOL_DIAGNOSTIC
501: if (flags & PR_LOGGING) {
502: if (kmem_map == NULL ||
503: (pp->pr_log = malloc(pool_logsize * sizeof(struct pool_log),
504: M_TEMP, M_NOWAIT)) == NULL)
505: pp->pr_roflags &= ~PR_LOGGING;
506: pp->pr_curlogentry = 0;
507: pp->pr_logsize = pool_logsize;
508: }
509: #endif
510:
511: pp->pr_entered_file = NULL;
512: pp->pr_entered_line = 0;
513:
514: simple_lock_init(&pp->pr_slock);
515:
516: pp->pr_ipl = -1;
517:
518: /*
519: * Initialize private page header pool and cache magazine pool if we
520: * haven't done so yet.
521: * XXX LOCKING.
522: */
523: if (phpool.pr_size == 0) {
524: pool_init(&phpool, sizeof(struct pool_item_header), 0, 0,
525: 0, "phpool", NULL);
526: pool_init(&pcgpool, sizeof(struct pool_cache_group), 0, 0,
527: 0, "pcgpool", NULL);
528: }
529:
530: simple_lock_init(&pool_head_slock);
531:
532: /* Insert this into the list of all pools. */
533: simple_lock(&pool_head_slock);
534: TAILQ_INSERT_TAIL(&pool_head, pp, pr_poollist);
535: simple_unlock(&pool_head_slock);
536:
537: /* Insert into the list of pools using this allocator. */
538: simple_lock(&palloc->pa_slock);
539: TAILQ_INSERT_TAIL(&palloc->pa_list, pp, pr_alloc_list);
540: simple_unlock(&palloc->pa_slock);
541: }
542:
543: #ifdef DIAGNOSTIC
544: void
545: pool_setipl(struct pool *pp, int ipl)
546: {
547: pp->pr_ipl = ipl;
548: }
549: #endif
550:
551: /*
552: * Decommission a pool resource.
553: */
554: void
555: pool_destroy(struct pool *pp)
556: {
557: struct pool_item_header *ph;
558: struct pool_cache *pc;
559:
560: /* Locking order: pool_allocator -> pool */
561: simple_lock(&pp->pr_alloc->pa_slock);
562: TAILQ_REMOVE(&pp->pr_alloc->pa_list, pp, pr_alloc_list);
563: simple_unlock(&pp->pr_alloc->pa_slock);
564:
565: /* Destroy all caches for this pool. */
566: while ((pc = TAILQ_FIRST(&pp->pr_cachelist)) != NULL)
567: pool_cache_destroy(pc);
568:
569: #ifdef DIAGNOSTIC
570: if (pp->pr_nout != 0) {
571: pr_printlog(pp, NULL, printf);
572: panic("pool_destroy: pool busy: still out: %u",
573: pp->pr_nout);
574: }
575: #endif
576:
577: /* Remove all pages */
578: while ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
579: pr_rmpage(pp, ph, NULL);
580: KASSERT(LIST_EMPTY(&pp->pr_fullpages));
581: KASSERT(LIST_EMPTY(&pp->pr_partpages));
582:
583: /* Remove from global pool list */
584: simple_lock(&pool_head_slock);
585: TAILQ_REMOVE(&pool_head, pp, pr_poollist);
586: simple_unlock(&pool_head_slock);
587:
588: #ifdef POOL_DIAGNOSTIC
589: if ((pp->pr_roflags & PR_LOGGING) != 0)
590: free(pp->pr_log, M_TEMP);
591: #endif
592: }
593:
594: static struct pool_item_header *
595: pool_alloc_item_header(struct pool *pp, caddr_t storage, int flags)
596: {
597: struct pool_item_header *ph;
598: int s;
599:
600: LOCK_ASSERT(simple_lock_held(&pp->pr_slock) == 0);
601:
602: if ((pp->pr_roflags & PR_PHINPAGE) != 0)
603: ph = (struct pool_item_header *) (storage + pp->pr_phoffset);
604: else {
605: s = splhigh();
606: ph = pool_get(&phpool, flags);
607: splx(s);
608: }
609:
610: return (ph);
611: }
612:
613: /*
614: * Grab an item from the pool; must be called at appropriate spl level
615: */
616: void *
617: #ifdef POOL_DIAGNOSTIC
618: _pool_get(struct pool *pp, int flags, const char *file, long line)
619: #else
620: pool_get(struct pool *pp, int flags)
621: #endif
622: {
623: struct pool_item *pi;
624: struct pool_item_header *ph;
625: void *v;
626:
627: #ifdef DIAGNOSTIC
628: if ((flags & PR_WAITOK) != 0)
629: splassert(IPL_NONE);
630: if (pp->pr_ipl != -1)
631: splassert(pp->pr_ipl);
632: if (__predict_false(curproc == NULL && /* doing_shutdown == 0 && XXX*/
633: (flags & PR_WAITOK) != 0))
634: panic("pool_get: %s:must have NOWAIT", pp->pr_wchan);
635:
636: #ifdef LOCKDEBUG
637: if (flags & PR_WAITOK)
638: simple_lock_only_held(NULL, "pool_get(PR_WAITOK)");
639: #endif
640: #endif /* DIAGNOSTIC */
641:
642: #ifdef MALLOC_DEBUG
643: if (pp->pr_roflags & PR_DEBUG) {
644: void *addr;
645:
646: addr = NULL;
647: debug_malloc(pp->pr_size, M_DEBUG,
648: (flags & PR_WAITOK) ? M_WAITOK : M_NOWAIT, &addr);
649: return (addr);
650: }
651: #endif
652:
653: simple_lock(&pp->pr_slock);
654: pr_enter(pp, file, line);
655:
656: startover:
657: /*
658: * Check to see if we've reached the hard limit. If we have,
659: * and we can wait, then wait until an item has been returned to
660: * the pool.
661: */
662: #ifdef DIAGNOSTIC
663: if (__predict_false(pp->pr_nout > pp->pr_hardlimit)) {
664: pr_leave(pp);
665: simple_unlock(&pp->pr_slock);
666: panic("pool_get: %s: crossed hard limit", pp->pr_wchan);
667: }
668: #endif
669: if (__predict_false(pp->pr_nout == pp->pr_hardlimit)) {
670: if ((flags & PR_WAITOK) && !(flags & PR_LIMITFAIL)) {
671: /*
672: * XXX: A warning isn't logged in this case. Should
673: * it be?
674: */
675: pp->pr_flags |= PR_WANTED;
676: pr_leave(pp);
677: ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
678: pr_enter(pp, file, line);
679: goto startover;
680: }
681:
682: /*
683: * Log a message that the hard limit has been hit.
684: */
685: if (pp->pr_hardlimit_warning != NULL &&
686: ratecheck(&pp->pr_hardlimit_warning_last,
687: &pp->pr_hardlimit_ratecap))
688: log(LOG_ERR, "%s\n", pp->pr_hardlimit_warning);
689:
690: pp->pr_nfail++;
691:
692: pr_leave(pp);
693: simple_unlock(&pp->pr_slock);
694: return (NULL);
695: }
696:
697: /*
698: * The convention we use is that if `curpage' is not NULL, then
699: * it points at a non-empty bucket. In particular, `curpage'
700: * never points at a page header which has PR_PHINPAGE set and
701: * has no items in its bucket.
702: */
703: if ((ph = pp->pr_curpage) == NULL) {
704: #ifdef DIAGNOSTIC
705: if (pp->pr_nitems != 0) {
706: simple_unlock(&pp->pr_slock);
707: printf("pool_get: %s: curpage NULL, nitems %u\n",
708: pp->pr_wchan, pp->pr_nitems);
709: panic("pool_get: nitems inconsistent");
710: }
711: #endif
712:
713: /*
714: * Call the back-end page allocator for more memory.
715: * Release the pool lock, as the back-end page allocator
716: * may block.
717: */
718: pr_leave(pp);
719: simple_unlock(&pp->pr_slock);
720: v = pool_allocator_alloc(pp, flags);
721: if (__predict_true(v != NULL))
722: ph = pool_alloc_item_header(pp, v, flags);
723: simple_lock(&pp->pr_slock);
724: pr_enter(pp, file, line);
725:
726: if (__predict_false(v == NULL || ph == NULL)) {
727: if (v != NULL)
728: pool_allocator_free(pp, v);
729:
730: /*
731: * We were unable to allocate a page or item
732: * header, but we released the lock during
733: * allocation, so perhaps items were freed
734: * back to the pool. Check for this case.
735: */
736: if (pp->pr_curpage != NULL)
737: goto startover;
738:
739: if ((flags & PR_WAITOK) == 0) {
740: pp->pr_nfail++;
741: pr_leave(pp);
742: simple_unlock(&pp->pr_slock);
743: return (NULL);
744: }
745:
746: /*
747: * Wait for items to be returned to this pool.
748: *
749: * XXX: maybe we should wake up once a second and
750: * try again?
751: */
752: pp->pr_flags |= PR_WANTED;
753: /* PA_WANTED is already set on the allocator. */
754: pr_leave(pp);
755: ltsleep(pp, PSWP, pp->pr_wchan, 0, &pp->pr_slock);
756: pr_enter(pp, file, line);
757: goto startover;
758: }
759:
760: /* We have more memory; add it to the pool */
761: pool_prime_page(pp, v, ph);
762: pp->pr_npagealloc++;
763:
764: /* Start the allocation process over. */
765: goto startover;
766: }
767: if (__predict_false((v = pi = TAILQ_FIRST(&ph->ph_itemlist)) == NULL)) {
768: pr_leave(pp);
769: simple_unlock(&pp->pr_slock);
770: panic("pool_get: %s: page empty", pp->pr_wchan);
771: }
772: #ifdef DIAGNOSTIC
773: if (__predict_false(pp->pr_nitems == 0)) {
774: pr_leave(pp);
775: simple_unlock(&pp->pr_slock);
776: printf("pool_get: %s: items on itemlist, nitems %u\n",
777: pp->pr_wchan, pp->pr_nitems);
778: panic("pool_get: nitems inconsistent");
779: }
780: #endif
781:
782: #ifdef POOL_DIAGNOSTIC
783: pr_log(pp, v, PRLOG_GET, file, line);
784: #endif
785:
786: #ifdef DIAGNOSTIC
787: if (__predict_false(pi->pi_magic != PI_MAGIC)) {
788: pr_printlog(pp, pi, printf);
789: panic("pool_get(%s): free list modified: magic=%x; page %p;"
790: " item addr %p",
791: pp->pr_wchan, pi->pi_magic, ph->ph_page, pi);
792: }
793: #endif
794:
795: /*
796: * Remove from item list.
797: */
798: TAILQ_REMOVE(&ph->ph_itemlist, pi, pi_list);
799: pp->pr_nitems--;
800: pp->pr_nout++;
801: if (ph->ph_nmissing == 0) {
802: #ifdef DIAGNOSTIC
803: if (__predict_false(pp->pr_nidle == 0))
804: panic("pool_get: nidle inconsistent");
805: #endif
806: pp->pr_nidle--;
807:
808: /*
809: * This page was previously empty. Move it to the list of
810: * partially-full pages. This page is already curpage.
811: */
812: LIST_REMOVE(ph, ph_pagelist);
813: LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
814: }
815: ph->ph_nmissing++;
816: if (TAILQ_EMPTY(&ph->ph_itemlist)) {
817: #ifdef DIAGNOSTIC
818: if (__predict_false(ph->ph_nmissing != pp->pr_itemsperpage)) {
819: pr_leave(pp);
820: simple_unlock(&pp->pr_slock);
821: panic("pool_get: %s: nmissing inconsistent",
822: pp->pr_wchan);
823: }
824: #endif
825: /*
826: * This page is now full. Move it to the full list
827: * and select a new current page.
828: */
829: LIST_REMOVE(ph, ph_pagelist);
830: LIST_INSERT_HEAD(&pp->pr_fullpages, ph, ph_pagelist);
831: pool_update_curpage(pp);
832: }
833:
834: pp->pr_nget++;
835:
836: /*
837: * If we have a low water mark and we are now below that low
838: * water mark, add more items to the pool.
839: */
840: if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
841: /*
842: * XXX: Should we log a warning? Should we set up a timeout
843: * to try again in a second or so? The latter could break
844: * a caller's assumptions about interrupt protection, etc.
845: */
846: }
847:
848: pr_leave(pp);
849: simple_unlock(&pp->pr_slock);
850: return (v);
851: }
852:
853: /*
854: * Internal version of pool_put(). Pool is already locked/entered.
855: */
856: void
857: pool_do_put(struct pool *pp, void *v)
858: {
859: struct pool_item *pi = v;
860: struct pool_item_header *ph;
861: caddr_t page;
862:
863: #ifdef MALLOC_DEBUG
864: if (pp->pr_roflags & PR_DEBUG) {
865: debug_free(v, M_DEBUG);
866: return;
867: }
868: #endif
869:
870: LOCK_ASSERT(simple_lock_held(&pp->pr_slock));
871:
872: page = (caddr_t)((vaddr_t)v & pp->pr_alloc->pa_pagemask);
873:
874: #ifdef DIAGNOSTIC
875: if (pp->pr_ipl != -1)
876: splassert(pp->pr_ipl);
877:
878: if (__predict_false(pp->pr_nout == 0)) {
879: printf("pool %s: putting with none out\n",
880: pp->pr_wchan);
881: panic("pool_put");
882: }
883: #endif
884:
885: if (__predict_false((ph = pr_find_pagehead(pp, page)) == NULL)) {
886: pr_printlog(pp, NULL, printf);
887: panic("pool_put: %s: page header missing", pp->pr_wchan);
888: }
889:
890: #ifdef LOCKDEBUG
891: /*
892: * Check if we're freeing a locked simple lock.
893: */
894: simple_lock_freecheck((caddr_t)pi, ((caddr_t)pi) + pp->pr_size);
895: #endif
896:
897: /*
898: * Return to item list.
899: */
900: #ifdef DIAGNOSTIC
901: pi->pi_magic = PI_MAGIC;
902: #endif
903: #ifdef DEBUG
904: {
905: int i, *ip = v;
906:
907: for (i = 0; i < pp->pr_size / sizeof(int); i++) {
908: *ip++ = PI_MAGIC;
909: }
910: }
911: #endif
912:
913: TAILQ_INSERT_HEAD(&ph->ph_itemlist, pi, pi_list);
914: ph->ph_nmissing--;
915: pp->pr_nput++;
916: pp->pr_nitems++;
917: pp->pr_nout--;
918:
919: /* Cancel "pool empty" condition if it exists */
920: if (pp->pr_curpage == NULL)
921: pp->pr_curpage = ph;
922:
923: if (pp->pr_flags & PR_WANTED) {
924: pp->pr_flags &= ~PR_WANTED;
925: if (ph->ph_nmissing == 0)
926: pp->pr_nidle++;
927: wakeup(pp);
928: return;
929: }
930:
931: /*
932: * If this page is now empty, do one of two things:
933: *
934: * (1) If we have more pages than the page high water mark,
935: * free the page back to the system.
936: *
937: * (2) Otherwise, move the page to the empty page list.
938: *
939: * Either way, select a new current page (so we use a partially-full
940: * page if one is available).
941: */
942: if (ph->ph_nmissing == 0) {
943: pp->pr_nidle++;
944: if (pp->pr_nidle > pp->pr_maxpages ||
945: (pp->pr_alloc->pa_flags & PA_WANT) != 0) {
946: pr_rmpage(pp, ph, NULL);
947: } else {
948: LIST_REMOVE(ph, ph_pagelist);
949: LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
950: }
951: pool_update_curpage(pp);
952: }
953:
954: /*
955: * If the page was previously completely full, move it to the
956: * partially-full list and make it the current page. The next
957: * allocation will get the item from this page, instead of
958: * further fragmenting the pool.
959: */
960: else if (ph->ph_nmissing == (pp->pr_itemsperpage - 1)) {
961: LIST_REMOVE(ph, ph_pagelist);
962: LIST_INSERT_HEAD(&pp->pr_partpages, ph, ph_pagelist);
963: pp->pr_curpage = ph;
964: }
965: }
966:
967: /*
968: * Return resource to the pool; must be called at appropriate spl level
969: */
970: #ifdef POOL_DIAGNOSTIC
971: void
972: _pool_put(struct pool *pp, void *v, const char *file, long line)
973: {
974:
975: simple_lock(&pp->pr_slock);
976: pr_enter(pp, file, line);
977:
978: pr_log(pp, v, PRLOG_PUT, file, line);
979:
980: pool_do_put(pp, v);
981:
982: pr_leave(pp);
983: simple_unlock(&pp->pr_slock);
984: }
985: #undef pool_put
986: #endif /* POOL_DIAGNOSTIC */
987:
988: void
989: pool_put(struct pool *pp, void *v)
990: {
991:
992: simple_lock(&pp->pr_slock);
993:
994: pool_do_put(pp, v);
995:
996: simple_unlock(&pp->pr_slock);
997: }
998:
999: #ifdef POOL_DIAGNOSTIC
1000: #define pool_put(h, v) _pool_put((h), (v), __FILE__, __LINE__)
1001: #endif
1002:
1003: /*
1004: * Add N items to the pool.
1005: */
1006: int
1007: pool_prime(struct pool *pp, int n)
1008: {
1009: struct pool_item_header *ph;
1010: caddr_t cp;
1011: int newpages;
1012:
1013: simple_lock(&pp->pr_slock);
1014:
1015: newpages = roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1016:
1017: while (newpages-- > 0) {
1018: simple_unlock(&pp->pr_slock);
1019: cp = pool_allocator_alloc(pp, PR_NOWAIT);
1020: if (__predict_true(cp != NULL))
1021: ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
1022: simple_lock(&pp->pr_slock);
1023:
1024: if (__predict_false(cp == NULL || ph == NULL)) {
1025: if (cp != NULL)
1026: pool_allocator_free(pp, cp);
1027: break;
1028: }
1029:
1030: pool_prime_page(pp, cp, ph);
1031: pp->pr_npagealloc++;
1032: pp->pr_minpages++;
1033: }
1034:
1035: if (pp->pr_minpages >= pp->pr_maxpages)
1036: pp->pr_maxpages = pp->pr_minpages + 1; /* XXX */
1037:
1038: simple_unlock(&pp->pr_slock);
1039: return (0);
1040: }
1041:
1042: /*
1043: * Add a page worth of items to the pool.
1044: *
1045: * Note, we must be called with the pool descriptor LOCKED.
1046: */
1047: void
1048: pool_prime_page(struct pool *pp, caddr_t storage, struct pool_item_header *ph)
1049: {
1050: struct pool_item *pi;
1051: caddr_t cp = storage;
1052: unsigned int align = pp->pr_align;
1053: unsigned int ioff = pp->pr_itemoffset;
1054: int n;
1055:
1056: #ifdef DIAGNOSTIC
1057: if (((u_long)cp & (pp->pr_alloc->pa_pagesz - 1)) != 0)
1058: panic("pool_prime_page: %s: unaligned page", pp->pr_wchan);
1059: #endif
1060:
1061: /*
1062: * Insert page header.
1063: */
1064: LIST_INSERT_HEAD(&pp->pr_emptypages, ph, ph_pagelist);
1065: TAILQ_INIT(&ph->ph_itemlist);
1066: ph->ph_page = storage;
1067: ph->ph_nmissing = 0;
1068: if ((pp->pr_roflags & PR_PHINPAGE) == 0)
1069: SPLAY_INSERT(phtree, &pp->pr_phtree, ph);
1070:
1071: pp->pr_nidle++;
1072:
1073: /*
1074: * Color this page.
1075: */
1076: cp = (caddr_t)(cp + pp->pr_curcolor);
1077: if ((pp->pr_curcolor += align) > pp->pr_maxcolor)
1078: pp->pr_curcolor = 0;
1079:
1080: /*
1081: * Adjust storage to apply aligment to `pr_itemoffset' in each item.
1082: */
1083: if (ioff != 0)
1084: cp = (caddr_t)(cp + (align - ioff));
1085:
1086: /*
1087: * Insert remaining chunks on the bucket list.
1088: */
1089: n = pp->pr_itemsperpage;
1090: pp->pr_nitems += n;
1091:
1092: while (n--) {
1093: pi = (struct pool_item *)cp;
1094:
1095: KASSERT(((((vaddr_t)pi) + ioff) & (align - 1)) == 0);
1096:
1097: /* Insert on page list */
1098: TAILQ_INSERT_TAIL(&ph->ph_itemlist, pi, pi_list);
1099: #ifdef DIAGNOSTIC
1100: pi->pi_magic = PI_MAGIC;
1101: #endif
1102: cp = (caddr_t)(cp + pp->pr_size);
1103: }
1104:
1105: /*
1106: * If the pool was depleted, point at the new page.
1107: */
1108: if (pp->pr_curpage == NULL)
1109: pp->pr_curpage = ph;
1110:
1111: if (++pp->pr_npages > pp->pr_hiwat)
1112: pp->pr_hiwat = pp->pr_npages;
1113: }
1114:
1115: /*
1116: * Used by pool_get() when nitems drops below the low water mark. This
1117: * is used to catch up pr_nitems with the low water mark.
1118: *
1119: * Note 1, we never wait for memory here, we let the caller decide what to do.
1120: *
1121: * Note 2, we must be called with the pool already locked, and we return
1122: * with it locked.
1123: */
1124: int
1125: pool_catchup(struct pool *pp)
1126: {
1127: struct pool_item_header *ph;
1128: caddr_t cp;
1129: int error = 0;
1130:
1131: while (POOL_NEEDS_CATCHUP(pp)) {
1132: /*
1133: * Call the page back-end allocator for more memory.
1134: *
1135: * XXX: We never wait, so should we bother unlocking
1136: * the pool descriptor?
1137: */
1138: simple_unlock(&pp->pr_slock);
1139: cp = pool_allocator_alloc(pp, PR_NOWAIT);
1140: if (__predict_true(cp != NULL))
1141: ph = pool_alloc_item_header(pp, cp, PR_NOWAIT);
1142: simple_lock(&pp->pr_slock);
1143: if (__predict_false(cp == NULL || ph == NULL)) {
1144: if (cp != NULL)
1145: pool_allocator_free(pp, cp);
1146: error = ENOMEM;
1147: break;
1148: }
1149: pool_prime_page(pp, cp, ph);
1150: pp->pr_npagealloc++;
1151: }
1152:
1153: return (error);
1154: }
1155:
1156: void
1157: pool_update_curpage(struct pool *pp)
1158: {
1159:
1160: pp->pr_curpage = LIST_FIRST(&pp->pr_partpages);
1161: if (pp->pr_curpage == NULL) {
1162: pp->pr_curpage = LIST_FIRST(&pp->pr_emptypages);
1163: }
1164: }
1165:
1166: void
1167: pool_setlowat(struct pool *pp, int n)
1168: {
1169:
1170: simple_lock(&pp->pr_slock);
1171:
1172: pp->pr_minitems = n;
1173: pp->pr_minpages = (n == 0)
1174: ? 0
1175: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1176:
1177: /* Make sure we're caught up with the newly-set low water mark. */
1178: if (POOL_NEEDS_CATCHUP(pp) && pool_catchup(pp) != 0) {
1179: /*
1180: * XXX: Should we log a warning? Should we set up a timeout
1181: * to try again in a second or so? The latter could break
1182: * a caller's assumptions about interrupt protection, etc.
1183: */
1184: }
1185:
1186: simple_unlock(&pp->pr_slock);
1187: }
1188:
1189: void
1190: pool_sethiwat(struct pool *pp, int n)
1191: {
1192:
1193: simple_lock(&pp->pr_slock);
1194:
1195: pp->pr_maxpages = (n == 0)
1196: ? 0
1197: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1198:
1199: simple_unlock(&pp->pr_slock);
1200: }
1201:
1202: int
1203: pool_sethardlimit(struct pool *pp, unsigned n, const char *warnmess, int ratecap)
1204: {
1205: int error = 0;
1206:
1207: simple_lock(&pp->pr_slock);
1208:
1209: if (n < pp->pr_nout) {
1210: error = EINVAL;
1211: goto done;
1212: }
1213:
1214: pp->pr_hardlimit = n;
1215: pp->pr_hardlimit_warning = warnmess;
1216: pp->pr_hardlimit_ratecap.tv_sec = ratecap;
1217: pp->pr_hardlimit_warning_last.tv_sec = 0;
1218: pp->pr_hardlimit_warning_last.tv_usec = 0;
1219:
1220: /*
1221: * In-line version of pool_sethiwat(), because we don't want to
1222: * release the lock.
1223: */
1224: pp->pr_maxpages = (n == 0 || n == UINT_MAX)
1225: ? n
1226: : roundup(n, pp->pr_itemsperpage) / pp->pr_itemsperpage;
1227:
1228: done:
1229: simple_unlock(&pp->pr_slock);
1230:
1231: return (error);
1232: }
1233:
1234: /*
1235: * Release all complete pages that have not been used recently.
1236: *
1237: * Returns non-zero if any pages have been reclaimed.
1238: */
1239: int
1240: #ifdef POOL_DIAGNOSTIC
1241: _pool_reclaim(struct pool *pp, const char *file, long line)
1242: #else
1243: pool_reclaim(struct pool *pp)
1244: #endif
1245: {
1246: struct pool_item_header *ph, *phnext;
1247: struct pool_cache *pc;
1248: struct pool_pagelist pq;
1249: int s;
1250:
1251: if (simple_lock_try(&pp->pr_slock) == 0)
1252: return (0);
1253: pr_enter(pp, file, line);
1254:
1255: LIST_INIT(&pq);
1256:
1257: /*
1258: * Reclaim items from the pool's caches.
1259: */
1260: TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist)
1261: pool_cache_reclaim(pc);
1262:
1263: for (ph = LIST_FIRST(&pp->pr_emptypages); ph != NULL; ph = phnext) {
1264: phnext = LIST_NEXT(ph, ph_pagelist);
1265:
1266: /* Check our minimum page claim */
1267: if (pp->pr_npages <= pp->pr_minpages)
1268: break;
1269:
1270: KASSERT(ph->ph_nmissing == 0);
1271:
1272: /*
1273: * If freeing this page would put us below
1274: * the low water mark, stop now.
1275: */
1276: if ((pp->pr_nitems - pp->pr_itemsperpage) <
1277: pp->pr_minitems)
1278: break;
1279:
1280: pr_rmpage(pp, ph, &pq);
1281: }
1282:
1283: pr_leave(pp);
1284: simple_unlock(&pp->pr_slock);
1285: if (LIST_EMPTY(&pq))
1286: return (0);
1287: while ((ph = LIST_FIRST(&pq)) != NULL) {
1288: LIST_REMOVE(ph, ph_pagelist);
1289: pool_allocator_free(pp, ph->ph_page);
1290: if (pp->pr_roflags & PR_PHINPAGE) {
1291: continue;
1292: }
1293: SPLAY_REMOVE(phtree, &pp->pr_phtree, ph);
1294: s = splhigh();
1295: pool_put(&phpool, ph);
1296: splx(s);
1297: }
1298:
1299: return (1);
1300: }
1301:
1302: #ifdef DDB
1303: #include <machine/db_machdep.h>
1304: #include <ddb/db_interface.h>
1305: #include <ddb/db_output.h>
1306:
1307: /*
1308: * Diagnostic helpers.
1309: */
1310: void
1311: pool_printit(struct pool *pp, const char *modif, int (*pr)(const char *, ...))
1312: {
1313: int s;
1314:
1315: s = splvm();
1316: if (simple_lock_try(&pp->pr_slock) == 0) {
1317: pr("pool %s is locked; try again later\n",
1318: pp->pr_wchan);
1319: splx(s);
1320: return;
1321: }
1322: pool_print1(pp, modif, pr);
1323: simple_unlock(&pp->pr_slock);
1324: splx(s);
1325: }
1326:
1327: void
1328: pool_print_pagelist(struct pool_pagelist *pl, int (*pr)(const char *, ...))
1329: {
1330: struct pool_item_header *ph;
1331: #ifdef DIAGNOSTIC
1332: struct pool_item *pi;
1333: #endif
1334:
1335: LIST_FOREACH(ph, pl, ph_pagelist) {
1336: (*pr)("\t\tpage %p, nmissing %d\n",
1337: ph->ph_page, ph->ph_nmissing);
1338: #ifdef DIAGNOSTIC
1339: TAILQ_FOREACH(pi, &ph->ph_itemlist, pi_list) {
1340: if (pi->pi_magic != PI_MAGIC) {
1341: (*pr)("\t\t\titem %p, magic 0x%x\n",
1342: pi, pi->pi_magic);
1343: }
1344: }
1345: #endif
1346: }
1347: }
1348:
1349: void
1350: pool_print1(struct pool *pp, const char *modif, int (*pr)(const char *, ...))
1351: {
1352: struct pool_item_header *ph;
1353: struct pool_cache *pc;
1354: struct pool_cache_group *pcg;
1355: int i, print_log = 0, print_pagelist = 0, print_cache = 0;
1356: char c;
1357:
1358: while ((c = *modif++) != '\0') {
1359: if (c == 'l')
1360: print_log = 1;
1361: if (c == 'p')
1362: print_pagelist = 1;
1363: if (c == 'c')
1364: print_cache = 1;
1365: modif++;
1366: }
1367:
1368: (*pr)("POOL %s: size %u, align %u, ioff %u, roflags 0x%08x\n",
1369: pp->pr_wchan, pp->pr_size, pp->pr_align, pp->pr_itemoffset,
1370: pp->pr_roflags);
1371: (*pr)("\talloc %p\n", pp->pr_alloc);
1372: (*pr)("\tminitems %u, minpages %u, maxpages %u, npages %u\n",
1373: pp->pr_minitems, pp->pr_minpages, pp->pr_maxpages, pp->pr_npages);
1374: (*pr)("\titemsperpage %u, nitems %u, nout %u, hardlimit %u\n",
1375: pp->pr_itemsperpage, pp->pr_nitems, pp->pr_nout, pp->pr_hardlimit);
1376:
1377: (*pr)("\n\tnget %lu, nfail %lu, nput %lu\n",
1378: pp->pr_nget, pp->pr_nfail, pp->pr_nput);
1379: (*pr)("\tnpagealloc %lu, npagefree %lu, hiwat %u, nidle %lu\n",
1380: pp->pr_npagealloc, pp->pr_npagefree, pp->pr_hiwat, pp->pr_nidle);
1381:
1382: if (print_pagelist == 0)
1383: goto skip_pagelist;
1384:
1385: if ((ph = LIST_FIRST(&pp->pr_emptypages)) != NULL)
1386: (*pr)("\n\tempty page list:\n");
1387: pool_print_pagelist(&pp->pr_emptypages, pr);
1388: if ((ph = LIST_FIRST(&pp->pr_fullpages)) != NULL)
1389: (*pr)("\n\tfull page list:\n");
1390: pool_print_pagelist(&pp->pr_fullpages, pr);
1391: if ((ph = LIST_FIRST(&pp->pr_partpages)) != NULL)
1392: (*pr)("\n\tpartial-page list:\n");
1393: pool_print_pagelist(&pp->pr_partpages, pr);
1394:
1395: if (pp->pr_curpage == NULL)
1396: (*pr)("\tno current page\n");
1397: else
1398: (*pr)("\tcurpage %p\n", pp->pr_curpage->ph_page);
1399:
1400: skip_pagelist:
1401: if (print_log == 0)
1402: goto skip_log;
1403:
1404: (*pr)("\n");
1405: if ((pp->pr_roflags & PR_LOGGING) == 0)
1406: (*pr)("\tno log\n");
1407: else
1408: pr_printlog(pp, NULL, pr);
1409:
1410: skip_log:
1411: if (print_cache == 0)
1412: goto skip_cache;
1413:
1414: TAILQ_FOREACH(pc, &pp->pr_cachelist, pc_poollist) {
1415: (*pr)("\tcache %p: allocfrom %p freeto %p\n", pc,
1416: pc->pc_allocfrom, pc->pc_freeto);
1417: (*pr)("\t hits %lu misses %lu ngroups %lu nitems %lu\n",
1418: pc->pc_hits, pc->pc_misses, pc->pc_ngroups, pc->pc_nitems);
1419: TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1420: (*pr)("\t\tgroup %p: avail %d\n", pcg, pcg->pcg_avail);
1421: for (i = 0; i < PCG_NOBJECTS; i++)
1422: (*pr)("\t\t\t%p\n", pcg->pcg_objects[i]);
1423: }
1424: }
1425:
1426: skip_cache:
1427: pr_enter_check(pp, pr);
1428: }
1429:
1430: void
1431: db_show_all_pools(db_expr_t expr, int haddr, db_expr_t count, char *modif)
1432: {
1433: struct pool *pp;
1434: char maxp[16];
1435: int ovflw;
1436: char mode;
1437:
1438: mode = modif[0];
1439: if (mode != '\0' && mode != 'a') {
1440: db_printf("usage: show all pools [/a]\n");
1441: return;
1442: }
1443:
1444: if (mode == '\0')
1445: db_printf("%-10s%4s%9s%5s%9s%6s%6s%6s%6s%6s%6s%5s\n",
1446: "Name",
1447: "Size",
1448: "Requests",
1449: "Fail",
1450: "Releases",
1451: "Pgreq",
1452: "Pgrel",
1453: "Npage",
1454: "Hiwat",
1455: "Minpg",
1456: "Maxpg",
1457: "Idle");
1458: else
1459: db_printf("%-10s %18s %18s\n",
1460: "Name", "Address", "Allocator");
1461:
1462: TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
1463: if (mode == 'a') {
1464: db_printf("%-10s %18p %18p\n", pp->pr_wchan, pp,
1465: pp->pr_alloc);
1466: continue;
1467: }
1468:
1469: if (!pp->pr_nget)
1470: continue;
1471:
1472: if (pp->pr_maxpages == UINT_MAX)
1473: snprintf(maxp, sizeof maxp, "inf");
1474: else
1475: snprintf(maxp, sizeof maxp, "%u", pp->pr_maxpages);
1476:
1477: #define PRWORD(ovflw, fmt, width, fixed, val) do { \
1478: (ovflw) += db_printf((fmt), \
1479: (width) - (fixed) - (ovflw) > 0 ? \
1480: (width) - (fixed) - (ovflw) : 0, \
1481: (val)) - (width); \
1482: if ((ovflw) < 0) \
1483: (ovflw) = 0; \
1484: } while (/* CONSTCOND */0)
1485:
1486: ovflw = 0;
1487: PRWORD(ovflw, "%-*s", 10, 0, pp->pr_wchan);
1488: PRWORD(ovflw, " %*u", 4, 1, pp->pr_size);
1489: PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nget);
1490: PRWORD(ovflw, " %*lu", 5, 1, pp->pr_nfail);
1491: PRWORD(ovflw, " %*lu", 9, 1, pp->pr_nput);
1492: PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagealloc);
1493: PRWORD(ovflw, " %*lu", 6, 1, pp->pr_npagefree);
1494: PRWORD(ovflw, " %*d", 6, 1, pp->pr_npages);
1495: PRWORD(ovflw, " %*d", 6, 1, pp->pr_hiwat);
1496: PRWORD(ovflw, " %*d", 6, 1, pp->pr_minpages);
1497: PRWORD(ovflw, " %*s", 6, 1, maxp);
1498: PRWORD(ovflw, " %*lu\n", 5, 1, pp->pr_nidle);
1499: }
1500: }
1501:
1502: int
1503: pool_chk_page(struct pool *pp, const char *label, struct pool_item_header *ph)
1504: {
1505: struct pool_item *pi;
1506: caddr_t page;
1507: int n;
1508:
1509: page = (caddr_t)((u_long)ph & pp->pr_alloc->pa_pagemask);
1510: if (page != ph->ph_page &&
1511: (pp->pr_roflags & PR_PHINPAGE) != 0) {
1512: if (label != NULL)
1513: printf("%s: ", label);
1514: printf("pool(%p:%s): page inconsistency: page %p;"
1515: " at page head addr %p (p %p)\n", pp,
1516: pp->pr_wchan, ph->ph_page,
1517: ph, page);
1518: return 1;
1519: }
1520:
1521: for (pi = TAILQ_FIRST(&ph->ph_itemlist), n = 0;
1522: pi != NULL;
1523: pi = TAILQ_NEXT(pi,pi_list), n++) {
1524:
1525: #ifdef DIAGNOSTIC
1526: if (pi->pi_magic != PI_MAGIC) {
1527: if (label != NULL)
1528: printf("%s: ", label);
1529: printf("pool(%s): free list modified: magic=%x;"
1530: " page %p; item ordinal %d;"
1531: " addr %p (p %p)\n",
1532: pp->pr_wchan, pi->pi_magic, ph->ph_page,
1533: n, pi, page);
1534: panic("pool");
1535: }
1536: #endif
1537: page =
1538: (caddr_t)((u_long)pi & pp->pr_alloc->pa_pagemask);
1539: if (page == ph->ph_page)
1540: continue;
1541:
1542: if (label != NULL)
1543: printf("%s: ", label);
1544: printf("pool(%p:%s): page inconsistency: page %p;"
1545: " item ordinal %d; addr %p (p %p)\n", pp,
1546: pp->pr_wchan, ph->ph_page,
1547: n, pi, page);
1548: return 1;
1549: }
1550: return 0;
1551: }
1552:
1553: int
1554: pool_chk(struct pool *pp, const char *label)
1555: {
1556: struct pool_item_header *ph;
1557: int r = 0;
1558:
1559: simple_lock(&pp->pr_slock);
1560: LIST_FOREACH(ph, &pp->pr_emptypages, ph_pagelist) {
1561: r = pool_chk_page(pp, label, ph);
1562: if (r) {
1563: goto out;
1564: }
1565: }
1566: LIST_FOREACH(ph, &pp->pr_fullpages, ph_pagelist) {
1567: r = pool_chk_page(pp, label, ph);
1568: if (r) {
1569: goto out;
1570: }
1571: }
1572: LIST_FOREACH(ph, &pp->pr_partpages, ph_pagelist) {
1573: r = pool_chk_page(pp, label, ph);
1574: if (r) {
1575: goto out;
1576: }
1577: }
1578:
1579: out:
1580: simple_unlock(&pp->pr_slock);
1581: return (r);
1582: }
1583: #endif
1584:
1585: /*
1586: * pool_cache_init:
1587: *
1588: * Initialize a pool cache.
1589: *
1590: * NOTE: If the pool must be protected from interrupts, we expect
1591: * to be called at the appropriate interrupt priority level.
1592: */
1593: void
1594: pool_cache_init(struct pool_cache *pc, struct pool *pp,
1595: int (*ctor)(void *, void *, int),
1596: void (*dtor)(void *, void *),
1597: void *arg)
1598: {
1599:
1600: TAILQ_INIT(&pc->pc_grouplist);
1601: simple_lock_init(&pc->pc_slock);
1602:
1603: pc->pc_allocfrom = NULL;
1604: pc->pc_freeto = NULL;
1605: pc->pc_pool = pp;
1606:
1607: pc->pc_ctor = ctor;
1608: pc->pc_dtor = dtor;
1609: pc->pc_arg = arg;
1610:
1611: pc->pc_hits = 0;
1612: pc->pc_misses = 0;
1613:
1614: pc->pc_ngroups = 0;
1615:
1616: pc->pc_nitems = 0;
1617:
1618: simple_lock(&pp->pr_slock);
1619: TAILQ_INSERT_TAIL(&pp->pr_cachelist, pc, pc_poollist);
1620: simple_unlock(&pp->pr_slock);
1621: }
1622:
1623: /*
1624: * pool_cache_destroy:
1625: *
1626: * Destroy a pool cache.
1627: */
1628: void
1629: pool_cache_destroy(struct pool_cache *pc)
1630: {
1631: struct pool *pp = pc->pc_pool;
1632:
1633: /* First, invalidate the entire cache. */
1634: pool_cache_invalidate(pc);
1635:
1636: /* ...and remove it from the pool's cache list. */
1637: simple_lock(&pp->pr_slock);
1638: TAILQ_REMOVE(&pp->pr_cachelist, pc, pc_poollist);
1639: simple_unlock(&pp->pr_slock);
1640: }
1641:
1642: static __inline void *
1643: pcg_get(struct pool_cache_group *pcg)
1644: {
1645: void *object;
1646: u_int idx;
1647:
1648: KASSERT(pcg->pcg_avail <= PCG_NOBJECTS);
1649: KASSERT(pcg->pcg_avail != 0);
1650: idx = --pcg->pcg_avail;
1651:
1652: KASSERT(pcg->pcg_objects[idx] != NULL);
1653: object = pcg->pcg_objects[idx];
1654: pcg->pcg_objects[idx] = NULL;
1655:
1656: return (object);
1657: }
1658:
1659: static __inline void
1660: pcg_put(struct pool_cache_group *pcg, void *object)
1661: {
1662: u_int idx;
1663:
1664: KASSERT(pcg->pcg_avail < PCG_NOBJECTS);
1665: idx = pcg->pcg_avail++;
1666:
1667: KASSERT(pcg->pcg_objects[idx] == NULL);
1668: pcg->pcg_objects[idx] = object;
1669: }
1670:
1671: /*
1672: * pool_cache_get:
1673: *
1674: * Get an object from a pool cache.
1675: */
1676: void *
1677: pool_cache_get(struct pool_cache *pc, int flags)
1678: {
1679: struct pool_cache_group *pcg;
1680: void *object;
1681:
1682: #ifdef LOCKDEBUG
1683: if (flags & PR_WAITOK)
1684: simple_lock_only_held(NULL, "pool_cache_get(PR_WAITOK)");
1685: #endif
1686:
1687: simple_lock(&pc->pc_slock);
1688:
1689: if ((pcg = pc->pc_allocfrom) == NULL) {
1690: TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1691: if (pcg->pcg_avail != 0) {
1692: pc->pc_allocfrom = pcg;
1693: goto have_group;
1694: }
1695: }
1696:
1697: /*
1698: * No groups with any available objects. Allocate
1699: * a new object, construct it, and return it to
1700: * the caller. We will allocate a group, if necessary,
1701: * when the object is freed back to the cache.
1702: */
1703: pc->pc_misses++;
1704: simple_unlock(&pc->pc_slock);
1705: object = pool_get(pc->pc_pool, flags);
1706: if (object != NULL && pc->pc_ctor != NULL) {
1707: if ((*pc->pc_ctor)(pc->pc_arg, object, flags) != 0) {
1708: pool_put(pc->pc_pool, object);
1709: return (NULL);
1710: }
1711: }
1712: return (object);
1713: }
1714:
1715: have_group:
1716: pc->pc_hits++;
1717: pc->pc_nitems--;
1718: object = pcg_get(pcg);
1719:
1720: if (pcg->pcg_avail == 0)
1721: pc->pc_allocfrom = NULL;
1722:
1723: simple_unlock(&pc->pc_slock);
1724:
1725: return (object);
1726: }
1727:
1728: /*
1729: * pool_cache_put:
1730: *
1731: * Put an object back to the pool cache.
1732: */
1733: void
1734: pool_cache_put(struct pool_cache *pc, void *object)
1735: {
1736: struct pool_cache_group *pcg;
1737: int s;
1738:
1739: simple_lock(&pc->pc_slock);
1740:
1741: if ((pcg = pc->pc_freeto) == NULL) {
1742: TAILQ_FOREACH(pcg, &pc->pc_grouplist, pcg_list) {
1743: if (pcg->pcg_avail != PCG_NOBJECTS) {
1744: pc->pc_freeto = pcg;
1745: goto have_group;
1746: }
1747: }
1748:
1749: /*
1750: * No empty groups to free the object to. Attempt to
1751: * allocate one.
1752: */
1753: simple_unlock(&pc->pc_slock);
1754: s = splvm();
1755: pcg = pool_get(&pcgpool, PR_NOWAIT);
1756: splx(s);
1757: if (pcg != NULL) {
1758: memset(pcg, 0, sizeof(*pcg));
1759: simple_lock(&pc->pc_slock);
1760: pc->pc_ngroups++;
1761: TAILQ_INSERT_TAIL(&pc->pc_grouplist, pcg, pcg_list);
1762: if (pc->pc_freeto == NULL)
1763: pc->pc_freeto = pcg;
1764: goto have_group;
1765: }
1766:
1767: /*
1768: * Unable to allocate a cache group; destruct the object
1769: * and free it back to the pool.
1770: */
1771: pool_cache_destruct_object(pc, object);
1772: return;
1773: }
1774:
1775: have_group:
1776: pc->pc_nitems++;
1777: pcg_put(pcg, object);
1778:
1779: if (pcg->pcg_avail == PCG_NOBJECTS)
1780: pc->pc_freeto = NULL;
1781:
1782: simple_unlock(&pc->pc_slock);
1783: }
1784:
1785: /*
1786: * pool_cache_destruct_object:
1787: *
1788: * Force destruction of an object and its release back into
1789: * the pool.
1790: */
1791: void
1792: pool_cache_destruct_object(struct pool_cache *pc, void *object)
1793: {
1794:
1795: if (pc->pc_dtor != NULL)
1796: (*pc->pc_dtor)(pc->pc_arg, object);
1797: pool_put(pc->pc_pool, object);
1798: }
1799:
1800: /*
1801: * pool_cache_do_invalidate:
1802: *
1803: * This internal function implements pool_cache_invalidate() and
1804: * pool_cache_reclaim().
1805: */
1806: void
1807: pool_cache_do_invalidate(struct pool_cache *pc, int free_groups,
1808: void (*putit)(struct pool *, void *))
1809: {
1810: struct pool_cache_group *pcg, *npcg;
1811: void *object;
1812: int s;
1813:
1814: for (pcg = TAILQ_FIRST(&pc->pc_grouplist); pcg != NULL;
1815: pcg = npcg) {
1816: npcg = TAILQ_NEXT(pcg, pcg_list);
1817: while (pcg->pcg_avail != 0) {
1818: pc->pc_nitems--;
1819: object = pcg_get(pcg);
1820: if (pcg->pcg_avail == 0 && pc->pc_allocfrom == pcg)
1821: pc->pc_allocfrom = NULL;
1822: if (pc->pc_dtor != NULL)
1823: (*pc->pc_dtor)(pc->pc_arg, object);
1824: (*putit)(pc->pc_pool, object);
1825: }
1826: if (free_groups) {
1827: pc->pc_ngroups--;
1828: TAILQ_REMOVE(&pc->pc_grouplist, pcg, pcg_list);
1829: if (pc->pc_freeto == pcg)
1830: pc->pc_freeto = NULL;
1831: s = splvm();
1832: pool_put(&pcgpool, pcg);
1833: splx(s);
1834: }
1835: }
1836: }
1837:
1838: /*
1839: * pool_cache_invalidate:
1840: *
1841: * Invalidate a pool cache (destruct and release all of the
1842: * cached objects).
1843: */
1844: void
1845: pool_cache_invalidate(struct pool_cache *pc)
1846: {
1847:
1848: simple_lock(&pc->pc_slock);
1849: pool_cache_do_invalidate(pc, 0, pool_put);
1850: simple_unlock(&pc->pc_slock);
1851: }
1852:
1853: /*
1854: * pool_cache_reclaim:
1855: *
1856: * Reclaim a pool cache for pool_reclaim().
1857: */
1858: void
1859: pool_cache_reclaim(struct pool_cache *pc)
1860: {
1861:
1862: simple_lock(&pc->pc_slock);
1863: pool_cache_do_invalidate(pc, 1, pool_do_put);
1864: simple_unlock(&pc->pc_slock);
1865: }
1866:
1867: /*
1868: * We have three different sysctls.
1869: * kern.pool.npools - the number of pools.
1870: * kern.pool.pool.<pool#> - the pool struct for the pool#.
1871: * kern.pool.name.<pool#> - the name for pool#.
1872: */
1873: int
1874: sysctl_dopool(int *name, u_int namelen, char *where, size_t *sizep)
1875: {
1876: struct pool *pp, *foundpool = NULL;
1877: size_t buflen = where != NULL ? *sizep : 0;
1878: int npools = 0, s;
1879: unsigned int lookfor;
1880: size_t len;
1881:
1882: switch (*name) {
1883: case KERN_POOL_NPOOLS:
1884: if (namelen != 1 || buflen != sizeof(int))
1885: return (EINVAL);
1886: lookfor = 0;
1887: break;
1888: case KERN_POOL_NAME:
1889: if (namelen != 2 || buflen < 1)
1890: return (EINVAL);
1891: lookfor = name[1];
1892: break;
1893: case KERN_POOL_POOL:
1894: if (namelen != 2 || buflen != sizeof(struct pool))
1895: return (EINVAL);
1896: lookfor = name[1];
1897: break;
1898: default:
1899: return (EINVAL);
1900: }
1901:
1902: s = splvm();
1903: simple_lock(&pool_head_slock);
1904:
1905: TAILQ_FOREACH(pp, &pool_head, pr_poollist) {
1906: npools++;
1907: if (lookfor == pp->pr_serial) {
1908: foundpool = pp;
1909: break;
1910: }
1911: }
1912:
1913: simple_unlock(&pool_head_slock);
1914: splx(s);
1915:
1916: if (*name != KERN_POOL_NPOOLS && foundpool == NULL)
1917: return (ENOENT);
1918:
1919: switch (*name) {
1920: case KERN_POOL_NPOOLS:
1921: return copyout(&npools, where, buflen);
1922: case KERN_POOL_NAME:
1923: len = strlen(foundpool->pr_wchan) + 1;
1924: if (*sizep < len)
1925: return (ENOMEM);
1926: *sizep = len;
1927: return copyout(foundpool->pr_wchan, where, len);
1928: case KERN_POOL_POOL:
1929: return copyout(foundpool, where, buflen);
1930: }
1931: /* NOTREACHED */
1932: return (0); /* XXX - Stupid gcc */
1933: }
1934:
1935: /*
1936: * Pool backend allocators.
1937: *
1938: * Each pool has a backend allocator that handles allocation, deallocation
1939: */
1940: void *pool_page_alloc_oldnointr(struct pool *, int);
1941: void pool_page_free_oldnointr(struct pool *, void *);
1942: void *pool_page_alloc(struct pool *, int);
1943: void pool_page_free(struct pool *, void *);
1944:
1945: /* previous nointr. handles large allocations safely */
1946: struct pool_allocator pool_allocator_oldnointr = {
1947: pool_page_alloc_oldnointr, pool_page_free_oldnointr, 0,
1948: };
1949: /* safe for interrupts, name preserved for compat
1950: * this is the default allocator */
1951: struct pool_allocator pool_allocator_nointr = {
1952: pool_page_alloc, pool_page_free, 0,
1953: };
1954:
1955: /*
1956: * XXX - we have at least three different resources for the same allocation
1957: * and each resource can be depleted. First we have the ready elements in
1958: * the pool. Then we have the resource (typically a vm_map) for this
1959: * allocator, then we have physical memory. Waiting for any of these can
1960: * be unnecessary when any other is freed, but the kernel doesn't support
1961: * sleeping on multiple addresses, so we have to fake. The caller sleeps on
1962: * the pool (so that we can be awakened when an item is returned to the pool),
1963: * but we set PA_WANT on the allocator. When a page is returned to
1964: * the allocator and PA_WANT is set pool_allocator_free will wakeup all
1965: * sleeping pools belonging to this allocator. (XXX - thundering herd).
1966: * We also wake up the allocator in case someone without a pool (malloc)
1967: * is sleeping waiting for this allocator.
1968: */
1969:
1970: void *
1971: pool_allocator_alloc(struct pool *pp, int flags)
1972: {
1973:
1974: return (pp->pr_alloc->pa_alloc(pp, flags));
1975: }
1976:
1977: void
1978: pool_allocator_free(struct pool *pp, void *v)
1979: {
1980: struct pool_allocator *pa = pp->pr_alloc;
1981: int s;
1982:
1983: (*pa->pa_free)(pp, v);
1984:
1985: s = splvm();
1986: simple_lock(&pa->pa_slock);
1987: if ((pa->pa_flags & PA_WANT) == 0) {
1988: simple_unlock(&pa->pa_slock);
1989: splx(s);
1990: return;
1991: }
1992:
1993: TAILQ_FOREACH(pp, &pa->pa_list, pr_alloc_list) {
1994: simple_lock(&pp->pr_slock);
1995: if ((pp->pr_flags & PR_WANTED) != 0) {
1996: pp->pr_flags &= ~PR_WANTED;
1997: wakeup(pp);
1998: }
1999: simple_unlock(&pp->pr_slock);
2000: }
2001: pa->pa_flags &= ~PA_WANT;
2002: simple_unlock(&pa->pa_slock);
2003: splx(s);
2004: }
2005:
2006: void *
2007: pool_page_alloc(struct pool *pp, int flags)
2008: {
2009: boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
2010:
2011: return (uvm_km_getpage(waitok));
2012: }
2013:
2014: void
2015: pool_page_free(struct pool *pp, void *v)
2016: {
2017:
2018: uvm_km_putpage(v);
2019: }
2020:
2021: void *
2022: pool_page_alloc_oldnointr(struct pool *pp, int flags)
2023: {
2024: boolean_t waitok = (flags & PR_WAITOK) ? TRUE : FALSE;
2025:
2026: splassert(IPL_NONE);
2027:
2028: return ((void *)uvm_km_alloc_poolpage1(kernel_map, uvm.kernel_object,
2029: waitok));
2030: }
2031:
2032: void
2033: pool_page_free_oldnointr(struct pool *pp, void *v)
2034: {
2035: splassert(IPL_NONE);
2036:
2037: uvm_km_free_poolpage1(kernel_map, (vaddr_t)v);
2038: }
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to subr_pool.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / kern