path: root/mm/slab_common.c

/*
 * Slab allocator functions that are independent of the allocator strategy
 *
 * (C) 2012 Christoph Lameter <cl@linux.com>
 */
#include <linux/slab.h>

#include <linux/mm.h>
#include <linux/poison.h>
#include <linux/interrupt.h>
#include <linux/memory.h>
#include <linux/compiler.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/uaccess.h>
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/page.h>
#include <linux/memcontrol.h>

#define CREATE_TRACE_POINTS
#include <trace/events/kmem.h>

#include "slab.h"

enum slab_state slab_state;
LIST_HEAD(slab_caches);
DEFINE_MUTEX(slab_mutex);
struct kmem_cache *kmem_cache;

/*
 * Set of flags that will prevent slab merging
 */
#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
		SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
		SLAB_FAILSLAB)

#define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
		SLAB_CACHE_DMA | SLAB_NOTRACK)

/*
 * Merge control. If this is set then no merging of slab caches will occur.
 * (Could be removed. This was introduced to pacify the merge skeptics.)
 */
static int slab_nomerge;

static int __init setup_slab_nomerge(char *str)
{
	slab_nomerge = 1;
	return 1;
}

#ifdef CONFIG_SLUB
__setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
#endif

__setup("slab_nomerge", setup_slab_nomerge);

/*
 * Determine the size of a slab object
 */
unsigned int kmem_cache_size(struct kmem_cache *s)
{
	return s->object_size;
}
EXPORT_SYMBOL(kmem_cache_size);

#ifdef CONFIG_DEBUG_VM
static int kmem_cache_sanity_check(const char *name, size_t size)
{
	struct kmem_cache *s = NULL;

	if (!name || in_interrupt() || size < sizeof(void *) ||
		size > KMALLOC_MAX_SIZE) {
		pr_err("kmem_cache_create(%s) integrity check failed\n", name);
		return -EINVAL;
	}

	list_for_each_entry(s, &slab_caches, list) {
		char tmp;
		int res;

		/*
		 * This happens when the module gets unloaded and doesn't
		 * destroy its slab cache and no-one else reuses the vmalloc
		 * area of the module.  Print a warning.
		 */
		res = probe_kernel_address(s->name, tmp);
		if (res) {
			pr_err("Slab cache with size %d has lost its name\n",
			       s->object_size);
			continue;
		}

#if !defined(CONFIG_SLUB)
		if (!strcmp(s->name, name)) {
			pr_err("%s (%s): Cache name already exists.\n",
			       __func__, name);
			dump_stack();
			s = NULL;
			return -EINVAL;
		}
#endif
	}

	WARN_ON(strchr(name, ' '));	/* It confuses parsers */
	return 0;
}
#else
static inline int kmem_cache_sanity_check(const char *name, size_t size)
{
	return 0;
}
#endif

#ifdef CONFIG_MEMCG_KMEM
static int memcg_alloc_cache_params(struct mem_cgroup *memcg,
		struct kmem_cache *s, struct kmem_cache *root_cache)
{
	size_t size;

	if (!memcg_kmem_enabled())
		return 0;

	if (!memcg) {
		size = offsetof(struct memcg_cache_params, memcg_caches);
		size += memcg_limited_groups_array_size * sizeof(void *);
	} else
		size = sizeof(struct memcg_cache_params);

	s->memcg_params = kzalloc(size, GFP_KERNEL);
	if (!s->memcg_params)
		return -ENOMEM;

	if (memcg) {
		s->memcg_params->memcg = memcg;
		s->memcg_params->root_cache = root_cache;
	} else
		s->memcg_params->is_root_cache = true;

	return 0;
}

static void memcg_free_cache_params(struct kmem_cache *s)
{
	kfree(s->memcg_params);
}

static int memcg_update_cache_params(struct kmem_cache *s, int num_memcgs)
{
	int size;
	struct memcg_cache_params *new_params, *cur_params;

	BUG_ON(!is_root_cache(s));

	size = offsetof(struct memcg_cache_params, memcg_caches);
	size += num_memcgs * sizeof(void *);

	new_params = kzalloc(size, GFP_KERNEL);
	if (!new_params)
		return -ENOMEM;

	cur_params = s->memcg_params;
	memcpy(new_params->memcg_caches, cur_params->memcg_caches,
	       memcg_limited_groups_array_size * sizeof(void *));

	new_params->is_root_cache = true;

	rcu_assign_pointer(s->memcg_params, new_params);
	if (cur_params)
		kfree_rcu(cur_params, rcu_head);

	return 0;
}

int memcg_update_all_caches(int num_memcgs)
{
	struct kmem_cache *s;
	int ret = 0;
	mutex_lock(&slab_mutex);

	list_for_each_entry(s, &slab_caches, list) {
		if (!is_root_cache(s))
			continue;

		ret = memcg_update_cache_params(s, num_memcgs);
		/*
		 * Instead of freeing the memory, we'll just leave the caches
		 * up to this point in an updated state.
		 */
		if (ret)
			goto out;
	}

	memcg_update_array_size(num_memcgs);
out:
	mutex_unlock(&slab_mutex);
	return ret;
}
#else
static inline int memcg_alloc_cache_params(struct mem_cgroup *memcg,
		struct kmem_cache *s, struct kmem_cache *root_cache)
{
	return 0;
}

static