kernel26-2.6.12-alx-r9/mm/mempool.c

/*
 *  linux/mm/mempool.c
 *
 *  memory buffer pool support. Such pools are mostly used
 *  for guaranteed, deadlock-free memory allocations during
 *  extreme VM load.
 *
 *  started by Ingo Molnar, Copyright (C) 2001
 */

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/mempool.h>
#include <linux/blkdev.h>
#include <linux/writeback.h>

static void add_element(mempool_t *pool, void *element)
{
	BUG_ON(pool->curr_nr >= pool->min_nr);
	pool->elements[pool->curr_nr++] = element;
}

static void *remove_element(mempool_t *pool)
{
	BUG_ON(pool->curr_nr <= 0);
	return pool->elements[--pool->curr_nr];
}

static void free_pool(mempool_t *pool)
{
	while (pool->curr_nr) {
		void *element = remove_element(pool);
		pool->free(element, pool->pool_data);
	}
	kfree(pool->elements);
	kfree(pool);
}

/**
 * mempool_create - create a memory pool
 * @min_nr:    the minimum number of elements guaranteed to be
 *             allocated for this pool.
 * @alloc_fn:  user-defined element-allocation function.
 * @free_fn:   user-defined element-freeing function.
 * @pool_data: optional private data available to the user-defined functions.
 *
 * this function creates and allocates a guaranteed size, preallocated
 * memory pool. The pool can be used from the mempool_alloc and mempool_free
 * functions. This function might sleep. Both the alloc_fn() and the free_fn()
 * functions might sleep - as long as the mempool_alloc function is not called
 * from IRQ contexts.
 */
mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
				mempool_free_t *free_fn, void *pool_data)
{
	mempool_t *pool;

	pool = kmalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool)
		return NULL;
	memset(pool, 0, sizeof(*pool));
	pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL);
	if (!pool->elements) {
		kfree(pool);
		return NULL;
	}
	spin_lock_init(&pool->lock);
	pool->min_nr = min_nr;
	pool->pool_data = pool_data;
	init_waitqueue_head(&pool->wait);
	pool->alloc = alloc_fn;
	pool->free = free_fn;

	/*
	 * First pre-allocate the guaranteed number of buffers.
	 */
	while (pool->curr_nr < pool->min_nr) {
		void *element;

		element = pool->alloc(GFP_KERNEL, pool->pool_data);
		if (unlikely(!element)) {
			free_pool(pool);
			return NULL;
		}
		add_element(pool, element);
	}
	return pool;
}
EXPORT_SYMBOL(mempool_create);

/**
 * mempool_resize - resize an existing memory pool
 * @pool:       pointer to the memory pool which was allocated via
 *              mempool_create().
 * @new_min_nr: the new minimum number of elements guaranteed to be
 *              allocated for this pool.
 * @gfp_mask:   the usual allocation bitmask.
 *
 * This function shrinks/grows the pool. In the case of growing,
 * it cannot be guaranteed that the pool will be grown to the new
 * size immediately, but new mempool_free() calls will refill it.
 *
 * Note, the caller must guarantee that no mempool_destroy is called
 * while this function is running. mempool_alloc() & mempool_free()
 * might be called (eg. from IRQ contexts) while this function executes.
 */
int mempool_resize(mempool_t *pool, int new_min_nr, unsigned int __nocast gfp_mask)
{
	void *element;
	void **new_elements;
	unsigned long flags;

	BUG_ON(new_min_nr <= 0);

	spin_lock_irqsave(&pool->lock, flags);
	if (new_min_nr <= pool->min_nr) {
		while (new_min_nr < pool->curr_nr) {
			element = remove_element(pool);
			spin_unlock_irqrestore(&pool->lock, flags);
			pool->free(element, pool->pool_data);
			spin_lock_irqsave(&pool->lock, flags);
		}
		pool->min_nr = new_min_nr;
		goto out_unlock;
	}
	spin_unlock_irqrestore(&pool->lock, flags);

	/* Grow the pool */
	new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask);
	if (!new_elements)
		return -ENOMEM;

	spin_lock_irqsave(&pool->lock, flags);
	if (unlikely(new_min_nr <= pool->min_nr)) {
		/* Raced, other resize will do our work */
		spin_unlock_irqrestore(&pool->lock, flags);
		kfree(new_elements);
		goto out;
	}
	memcpy(new_elements, pool->elements,
			pool->curr_nr * sizeof(*new_elements));
	kfree(pool->elements);
	pool->elements = new_elements;
	pool->min_nr = new_min_nr;

	while (pool->curr_nr < pool->min_nr) {
		spin_unlock_irqrestore(&pool->lock, flags);
		element = pool->alloc(gfp_mask, pool->pool_data);
		if (!element)
			goto out;
		spin_lock_irqsave(&pool->lock, flags);
		if (pool->curr_nr < pool->min_nr) {
			add_element(pool, element);
		} else {
			spin_unlock_irqrestore(&pool->lock, flags);
			pool->free(element, pool->pool_data);	/* Raced */
			goto out;
		}
	}
out_unlock:
	spin_unlock_irqrestore(&pool->lock, flags);
out:
	return 0;
}
EXPORT_SYMBOL(mempool_resize);

/**
 * mempool_destroy - deallocate a memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * this function only sleeps if the free_fn() function sleeps. The caller
 * has to guarantee that all elements have been returned to the pool (ie:
 * freed) prior to calling mempool_destroy().
 */
void mempool_destroy(mempool_t *pool)
{
	if (pool->curr_nr != pool->min_nr)
		BUG();		/* There were outstanding elements */
	free_pool(pool);
}
EXPORT_SYMBOL(mempool_destroy);

/**
 * mempool_alloc - allocate an element from a specific memory pool
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 * @gfp_mask:  the usual allocation bitmask.
 *
 * this function only sleeps if the alloc_fn function sleeps or
 * returns NULL. Note that due to preallocation, this function
 * *never* fails when called from process contexts. (it might
 * fail if called from an IRQ context.)
 */
void * mempool_alloc(mempool_t *pool, unsigned int __nocast gfp_mask)
{
	void *element;
	unsigned long flags;
	DEFINE_WAIT(wait);
	int gfp_temp;

	might_sleep_if(gfp_mask & __GFP_WAIT);

	gfp_mask |= __GFP_NOMEMALLOC;	/* don't allocate emergency reserves */
	gfp_mask |= __GFP_NORETRY;	/* don't loop in __alloc_pages */
	gfp_mask |= __GFP_NOWARN;	/* failures are OK */

	gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO);

repeat_alloc:

	element = pool->alloc(gfp_temp, pool->pool_data);
	if (likely(element != NULL))
		return element;

	spin_lock_irqsave(&pool->lock, flags);
	if (likely(pool->curr_nr)) {
		element = remove_element(pool);
		spin_unlock_irqrestore(&pool->lock, flags);
		return element;
	}
	spin_unlock_irqrestore(&pool->lock, flags);

	/* We must not sleep in the GFP_ATOMIC case */
	if (!(gfp_mask & __GFP_WAIT))
		return NULL;

	/* Now start performing page reclaim */
	gfp_temp = gfp_mask;
	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
	smp_mb();
	if (!pool->curr_nr)
		io_schedule();
	finish_wait(&pool->wait, &wait);

	goto repeat_alloc;
}
EXPORT_SYMBOL(mempool_alloc);

/**
 * mempool_free - return an element to the pool.
 * @element:   pool element pointer.
 * @pool:      pointer to the memory pool which was allocated via
 *             mempool_create().
 *
 * this function only sleeps if the free_fn() function sleeps.
 */
void mempool_free(void *element, mempool_t *pool)
{
	unsigned long flags;

	smp_mb();
	if (pool->curr_nr < pool->min_nr) {
		spin_lock_irqsave(&pool->lock, flags);
		if (pool->curr_nr < pool->min_nr) {
			add_element(pool, element);
			spin_unlock_irqrestore(&pool->lock, flags);
			wake_up(&pool->wait);
			return;
		}
		spin_unlock_irqrestore(&pool->lock, flags);
	}
	pool->free(element, pool->pool_data);
}
EXPORT_SYMBOL(mempool_free);

/*
 * A commonly used alloc and free fn.
 */
void *mempool_alloc_slab(unsigned int __nocast gfp_mask, void *pool_data)
{
	kmem_cache_t *mem = (kmem_cache_t *) pool_data;
	return kmem_cache_alloc(mem, gfp_mask);
}
EXPORT_SYMBOL(mempool_alloc_slab);

void mempool_free_slab(void *element, void *pool_data)
{
	kmem_cache_t *mem = (kmem_cache_t *) pool_data;
	kmem_cache_free(mem, element);
}
EXPORT_SYMBOL(mempool_free_slab);
1	niro	628	/*
2			* linux/mm/mempool.c
3			*
4			* memory buffer pool support. Such pools are mostly used
5			* for guaranteed, deadlock-free memory allocations during
6			* extreme VM load.
7			*
8			* started by Ingo Molnar, Copyright (C) 2001
9			*/
10
11			#include <linux/mm.h>
12			#include <linux/slab.h>
13			#include <linux/module.h>
14			#include <linux/mempool.h>
15			#include <linux/blkdev.h>
16			#include <linux/writeback.h>
17
18			static void add_element(mempool_t pool, void element)
19			{
20			BUG_ON(pool->curr_nr >= pool->min_nr);
21			pool->elements[pool->curr_nr++] = element;
22			}
23
24			static void remove_element(mempool_t pool)
25			{
26			BUG_ON(pool->curr_nr <= 0);
27			return pool->elements[--pool->curr_nr];
28			}
29
30			static void free_pool(mempool_t *pool)
31			{
32			while (pool->curr_nr) {
33			void *element = remove_element(pool);
34			pool->free(element, pool->pool_data);
35			}
36			kfree(pool->elements);
37			kfree(pool);
38			}
39
40			/**
41			* mempool_create - create a memory pool
42			* @min_nr: the minimum number of elements guaranteed to be
43			* allocated for this pool.
44			* @alloc_fn: user-defined element-allocation function.
45			* @free_fn: user-defined element-freeing function.
46			* @pool_data: optional private data available to the user-defined functions.
47			*
48			* this function creates and allocates a guaranteed size, preallocated
49			* memory pool. The pool can be used from the mempool_alloc and mempool_free
50			* functions. This function might sleep. Both the alloc_fn() and the free_fn()
51			* functions might sleep - as long as the mempool_alloc function is not called
52			* from IRQ contexts.
53			*/
54			mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
55			mempool_free_t free_fn, void pool_data)
56			{
57			mempool_t *pool;
58
59			pool = kmalloc(sizeof(*pool), GFP_KERNEL);
60			if (!pool)
61			return NULL;
62			memset(pool, 0, sizeof(*pool));
63			pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL);
64			if (!pool->elements) {
65			kfree(pool);
66			return NULL;
67			}
68			spin_lock_init(&pool->lock);
69			pool->min_nr = min_nr;
70			pool->pool_data = pool_data;
71			init_waitqueue_head(&pool->wait);
72			pool->alloc = alloc_fn;
73			pool->free = free_fn;
74
75			/*
76			* First pre-allocate the guaranteed number of buffers.
77			*/
78			while (pool->curr_nr < pool->min_nr) {
79			void *element;
80
81			element = pool->alloc(GFP_KERNEL, pool->pool_data);
82			if (unlikely(!element)) {
83			free_pool(pool);
84			return NULL;
85			}
86			add_element(pool, element);
87			}
88			return pool;
89			}
90			EXPORT_SYMBOL(mempool_create);
91
92			/**
93			* mempool_resize - resize an existing memory pool
94			* @pool: pointer to the memory pool which was allocated via
95			* mempool_create().
96			* @new_min_nr: the new minimum number of elements guaranteed to be
97			* allocated for this pool.
98			* @gfp_mask: the usual allocation bitmask.
99			*
100			* This function shrinks/grows the pool. In the case of growing,
101			* it cannot be guaranteed that the pool will be grown to the new
102			* size immediately, but new mempool_free() calls will refill it.
103			*
104			* Note, the caller must guarantee that no mempool_destroy is called
105			* while this function is running. mempool_alloc() & mempool_free()
106			* might be called (eg. from IRQ contexts) while this function executes.
107			*/
108			int mempool_resize(mempool_t *pool, int new_min_nr, unsigned int __nocast gfp_mask)
109			{
110			void *element;
111			void **new_elements;
112			unsigned long flags;
113
114			BUG_ON(new_min_nr <= 0);
115
116			spin_lock_irqsave(&pool->lock, flags);
117			if (new_min_nr <= pool->min_nr) {
118			while (new_min_nr < pool->curr_nr) {
119			element = remove_element(pool);
120			spin_unlock_irqrestore(&pool->lock, flags);
121			pool->free(element, pool->pool_data);
122			spin_lock_irqsave(&pool->lock, flags);
123			}
124			pool->min_nr = new_min_nr;
125			goto out_unlock;
126			}
127			spin_unlock_irqrestore(&pool->lock, flags);
128
129			/* Grow the pool */
130			new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask);
131			if (!new_elements)
132			return -ENOMEM;
133
134			spin_lock_irqsave(&pool->lock, flags);
135			if (unlikely(new_min_nr <= pool->min_nr)) {
136			/* Raced, other resize will do our work */
137			spin_unlock_irqrestore(&pool->lock, flags);
138			kfree(new_elements);
139			goto out;
140			}
141			memcpy(new_elements, pool->elements,
142			pool->curr_nr * sizeof(*new_elements));
143			kfree(pool->elements);
144			pool->elements = new_elements;
145			pool->min_nr = new_min_nr;
146
147			while (pool->curr_nr < pool->min_nr) {
148			spin_unlock_irqrestore(&pool->lock, flags);
149			element = pool->alloc(gfp_mask, pool->pool_data);
150			if (!element)
151			goto out;
152			spin_lock_irqsave(&pool->lock, flags);
153			if (pool->curr_nr < pool->min_nr) {
154			add_element(pool, element);
155			} else {
156			spin_unlock_irqrestore(&pool->lock, flags);
157			pool->free(element, pool->pool_data); /* Raced */
158			goto out;
159			}
160			}
161			out_unlock:
162			spin_unlock_irqrestore(&pool->lock, flags);
163			out:
164			return 0;
165			}
166			EXPORT_SYMBOL(mempool_resize);
167
168			/**
169			* mempool_destroy - deallocate a memory pool
170			* @pool: pointer to the memory pool which was allocated via
171			* mempool_create().
172			*
173			* this function only sleeps if the free_fn() function sleeps. The caller
174			* has to guarantee that all elements have been returned to the pool (ie:
175			* freed) prior to calling mempool_destroy().
176			*/
177			void mempool_destroy(mempool_t *pool)
178			{
179			if (pool->curr_nr != pool->min_nr)
180			BUG(); /* There were outstanding elements */
181			free_pool(pool);
182			}
183			EXPORT_SYMBOL(mempool_destroy);
184
185			/**
186			* mempool_alloc - allocate an element from a specific memory pool
187			* @pool: pointer to the memory pool which was allocated via
188			* mempool_create().
189			* @gfp_mask: the usual allocation bitmask.
190			*
191			* this function only sleeps if the alloc_fn function sleeps or
192			* returns NULL. Note that due to preallocation, this function
193			* never fails when called from process contexts. (it might
194			* fail if called from an IRQ context.)
195			*/
196			void * mempool_alloc(mempool_t *pool, unsigned int __nocast gfp_mask)
197			{
198			void *element;
199			unsigned long flags;
200			DEFINE_WAIT(wait);
201			int gfp_temp;
202
203			might_sleep_if(gfp_mask & __GFP_WAIT);
204
205			gfp_mask \|= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */
206			gfp_mask \|= __GFP_NORETRY; /* don't loop in __alloc_pages */
207			gfp_mask \|= __GFP_NOWARN; /* failures are OK */
208
209			gfp_temp = gfp_mask & ~(__GFP_WAIT\|__GFP_IO);
210
211			repeat_alloc:
212
213			element = pool->alloc(gfp_temp, pool->pool_data);
214			if (likely(element != NULL))
215			return element;
216
217			spin_lock_irqsave(&pool->lock, flags);
218			if (likely(pool->curr_nr)) {
219			element = remove_element(pool);
220			spin_unlock_irqrestore(&pool->lock, flags);
221			return element;
222			}
223			spin_unlock_irqrestore(&pool->lock, flags);
224
225			/* We must not sleep in the GFP_ATOMIC case */
226			if (!(gfp_mask & __GFP_WAIT))
227			return NULL;
228
229			/* Now start performing page reclaim */
230			gfp_temp = gfp_mask;
231			prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
232			smp_mb();
233			if (!pool->curr_nr)
234			io_schedule();
235			finish_wait(&pool->wait, &wait);
236
237			goto repeat_alloc;
238			}
239			EXPORT_SYMBOL(mempool_alloc);
240
241			/**
242			* mempool_free - return an element to the pool.
243			* @element: pool element pointer.
244			* @pool: pointer to the memory pool which was allocated via
245			* mempool_create().
246			*
247			* this function only sleeps if the free_fn() function sleeps.
248			*/
249			void mempool_free(void element, mempool_t pool)
250			{
251			unsigned long flags;
252
253			smp_mb();
254			if (pool->curr_nr < pool->min_nr) {
255			spin_lock_irqsave(&pool->lock, flags);
256			if (pool->curr_nr < pool->min_nr) {
257			add_element(pool, element);
258			spin_unlock_irqrestore(&pool->lock, flags);
259			wake_up(&pool->wait);
260			return;
261			}
262			spin_unlock_irqrestore(&pool->lock, flags);
263			}
264			pool->free(element, pool->pool_data);
265			}
266			EXPORT_SYMBOL(mempool_free);
267
268			/*
269			* A commonly used alloc and free fn.
270			*/
271			void mempool_alloc_slab(unsigned int __nocast gfp_mask, void pool_data)
272			{
273			kmem_cache_t mem = (kmem_cache_t ) pool_data;
274			return kmem_cache_alloc(mem, gfp_mask);
275			}
276			EXPORT_SYMBOL(mempool_alloc_slab);
277
278			void mempool_free_slab(void element, void pool_data)
279			{
280			kmem_cache_t mem = (kmem_cache_t ) pool_data;
281			kmem_cache_free(mem, element);
282			}
283			EXPORT_SYMBOL(mempool_free_slab);