From ed70f3a264e9f746eaf17c96ccc4c9b7eda742dc Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Wed, 4 Jun 2014 10:11:06 +0100 Subject: ASoC: arizona: Implement TDM support for Arizona devices Signed-off-by: Charles Keepax Signed-off-by: Mark Brown --- include/linux/mfd/arizona/core.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/arizona/core.h b/include/linux/mfd/arizona/core.h index 5cf8b91ce996..11783b511b9a 100644 --- a/include/linux/mfd/arizona/core.h +++ b/include/linux/mfd/arizona/core.h @@ -110,6 +110,9 @@ struct arizona { int clk32k_ref; struct snd_soc_dapm_context *dapm; + + int tdm_width[ARIZONA_MAX_AIF]; + int tdm_slots[ARIZONA_MAX_AIF]; }; int arizona_clk32k_enable(struct arizona *arizona); -- cgit v1.2.3 From cc9e92431ee9c7fe974266e0e6533a1a68e45539 Mon Sep 17 00:00:00 2001 From: Charles Keepax Date: Fri, 6 Jun 2014 14:14:05 +0100 Subject: ASoC: wm5102: Add controls to allow shaping of ultrasonic response Add controls to allow custom shaping of the ultrasonic response. This custom shaping can be turned on/off at runtime, although, it should be noted that settings will not affect a currently open audio stream, they will be applied when the next audio stream is started. Signed-off-by: Charles Keepax Signed-off-by: Mark Brown --- include/linux/mfd/arizona/core.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/arizona/core.h b/include/linux/mfd/arizona/core.h index 11783b511b9a..55926517d50b 100644 --- a/include/linux/mfd/arizona/core.h +++ b/include/linux/mfd/arizona/core.h @@ -113,6 +113,9 @@ struct arizona { int tdm_width[ARIZONA_MAX_AIF]; int tdm_slots[ARIZONA_MAX_AIF]; + + uint16_t dac_comp_coeff; + uint8_t dac_comp_enabled; }; int arizona_clk32k_enable(struct arizona *arizona); -- cgit v1.2.3 From 962bd40bc30e412828e091bfda041b7547e779c8 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 10 Jun 2014 12:24:40 -0400 Subject: locks: add missing memory barrier in break_deleg break_deleg is subject to the same potential race as break_lease. Add a memory barrier to prevent it. Signed-off-by: Jeff Layton --- include/linux/fs.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include/linux') diff --git a/include/linux/fs.h b/include/linux/fs.h index c3f46e499dd0..22ae79650b82 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1914,6 +1914,12 @@ static inline int break_lease(struct inode *inode, unsigned int mode) static inline int break_deleg(struct inode *inode, unsigned int mode) { + /* + * Since this check is lockless, we must ensure that any refcounts + * taken are done before checking inode->i_flock. Otherwise, we could + * end up racing with tasks trying to set a new lease on this file. + */ + smp_mb(); if (inode->i_flock) return __break_lease(inode, mode, FL_DELEG); return 0; -- cgit v1.2.3 From 398fd22b6b94cb15c1c299bceecd63644a1b17b4 Mon Sep 17 00:00:00 2001 From: Peter Meerwald Date: Sat, 6 Dec 2014 06:46:00 +0000 Subject: iio: Remove timestamp argument from iio_trigger_poll() and iio_trigger_poll_chained() argument has been ignored; adjust drivers accordingly Signed-off-by: Peter Meerwald Signed-off-by: Jonathan Cameron --- include/linux/iio/trigger.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/iio/trigger.h b/include/linux/iio/trigger.h index 369cf2cd5144..4b79ffe7b188 100644 --- a/include/linux/iio/trigger.h +++ b/include/linux/iio/trigger.h @@ -129,12 +129,11 @@ void iio_trigger_unregister(struct iio_trigger *trig_info); /** * iio_trigger_poll() - called on a trigger occurring * @trig: trigger which occurred - * @time: timestamp when trigger occurred * * Typically called in relevant hardware interrupt handler. **/ -void iio_trigger_poll(struct iio_trigger *trig, s64 time); -void iio_trigger_poll_chained(struct iio_trigger *trig, s64 time); +void iio_trigger_poll(struct iio_trigger *trig); +void iio_trigger_poll_chained(struct iio_trigger *trig); irqreturn_t iio_trigger_generic_data_rdy_poll(int irq, void *private); -- cgit v1.2.3 From 1a4fbf6a9286a6e3db497bc7bbae2024f0f1ad90 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Thu, 6 Nov 2014 23:07:00 +0000 Subject: iio: accel: kxcjk1013 3-axis accelerometer driver This patch adds IIO driver for KXCJK 1013 triaxis accelerometer sensor. The specifications for this driver is downloaded from: http://www.kionix.com/sites/default/files/KXCJK-1013%20Specifications%20Rev%202.pdf Signed-off-by: Srinivas Pandruvada Signed-off-by: Jonathan Cameron --- include/linux/iio/accel/kxcjk_1013.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 include/linux/iio/accel/kxcjk_1013.h (limited to 'include/linux') diff --git a/include/linux/iio/accel/kxcjk_1013.h b/include/linux/iio/accel/kxcjk_1013.h new file mode 100644 index 000000000000..fd1d540ea62d --- /dev/null +++ b/include/linux/iio/accel/kxcjk_1013.h @@ -0,0 +1,22 @@ +/* + * KXCJK-1013 3-axis accelerometer Interface + * Copyright (c) 2014, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + */ + +#ifndef __IIO_KXCJK_1013_H__ +#define __IIO_KXCJK_1013_H__ + +struct kxcjk_1013_platform_data { + bool active_high_intr; +}; + +#endif -- cgit v1.2.3 From a288d648715bab6a1ab2b72cb1c1cc79cdc8cb43 Mon Sep 17 00:00:00 2001 From: Richard Fitzgerald Date: Fri, 23 May 2014 12:54:57 +0100 Subject: extcon: arizona: support inverted jack detect switch Add config option for inverted jack detect switch that opens when jack is inserted. Signed-off-by: Richard Fitzgerald [Acked by Lee Jones for MFD part] Acked-by: Lee Jones Signed-off-by: Chanwoo Choi --- include/linux/mfd/arizona/pdata.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/mfd/arizona/pdata.h b/include/linux/mfd/arizona/pdata.h index 12a5c135c746..4578c72c9b86 100644 --- a/include/linux/mfd/arizona/pdata.h +++ b/include/linux/mfd/arizona/pdata.h @@ -127,6 +127,9 @@ struct arizona_pdata { /** Internal pull on GPIO5 is disabled when used for jack detection */ bool jd_gpio5_nopull; + /** set to true if jackdet contact opens on insert */ + bool jd_invert; + /** Use the headphone detect circuit to identify the accessory */ bool hpdet_acc_id; -- cgit v1.2.3 From 478850160636c4f0b2558451df0e42f8c5a10939 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 8 May 2014 01:37:48 +0200 Subject: irq_work: Implement remote queueing irq work currently only supports local callbacks. However its code is mostly ready to run remote callbacks and we have some potential user. The full nohz subsystem currently open codes its own remote irq work on top of the scheduler ipi when it wants a CPU to reevaluate its next tick. However this ad hoc solution bloats the scheduler IPI. Lets just extend the irq work subsystem to support remote queuing on top of the generic SMP IPI to handle this kind of user. This shouldn't add noticeable overhead. Suggested-by: Peter Zijlstra Acked-by: Peter Zijlstra Cc: Andrew Morton Cc: Eric Dumazet Cc: Ingo Molnar Cc: Kevin Hilman Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker --- include/linux/irq_work.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index 19ae05d4b8ec..bf9422c3aefe 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -33,6 +33,11 @@ void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *)) #define DEFINE_IRQ_WORK(name, _f) struct irq_work name = { .func = (_f), } bool irq_work_queue(struct irq_work *work); + +#ifdef CONFIG_SMP +bool irq_work_queue_on(struct irq_work *work, int cpu); +#endif + void irq_work_run(void); void irq_work_sync(struct irq_work *work); -- cgit v1.2.3 From 3d36aebc2e78923095575df954f3f3b430ac0a30 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 4 Jun 2014 16:17:33 +0200 Subject: nohz: Support nohz full remote kick Remotely kicking a full nohz CPU in order to make it re-evaluate its next tick is currently implemented using the scheduler IPI. However this bloats a scheduler fast path with an off-topic feature. The scheduler tick was abused here for its cool "callable anywhere/anytime" properties. But now that the irq work subsystem can queue remote callbacks, it's a perfect fit to safely queue IPIs when interrupts are disabled without worrying about concurrent callers. So lets implement remote kick on top of irq work. This is going to be used when a new event requires the next tick to be recalculated: more than 1 task competing on the CPU, timer armed, ... Acked-by: Peter Zijlstra Cc: Andrew Morton Cc: Ingo Molnar Cc: Kevin Hilman Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker --- include/linux/tick.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/tick.h b/include/linux/tick.h index b84773cb9f4c..8a4987f2294a 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -181,7 +181,13 @@ static inline bool tick_nohz_full_cpu(int cpu) extern void tick_nohz_init(void); extern void __tick_nohz_full_check(void); -extern void tick_nohz_full_kick(void); +extern void tick_nohz_full_kick_cpu(int cpu); + +static inline void tick_nohz_full_kick(void) +{ + tick_nohz_full_kick_cpu(smp_processor_id()); +} + extern void tick_nohz_full_kick_all(void); extern void __tick_nohz_task_switch(struct task_struct *tsk); #else @@ -189,6 +195,7 @@ static inline void tick_nohz_init(void) { } static inline bool tick_nohz_full_enabled(void) { return false; } static inline bool tick_nohz_full_cpu(int cpu) { return false; } static inline void __tick_nohz_full_check(void) { } +static inline void tick_nohz_full_kick_cpu(int cpu) { } static inline void tick_nohz_full_kick(void) { } static inline void tick_nohz_full_kick_all(void) { } static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } -- cgit v1.2.3 From d0540f91cf74fab90e1143d8d40da8a5b5fabc8a Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 21 May 2014 08:52:47 +0200 Subject: mfd: max77693: Remove unnecessary wrapper functions This patch removes wrapper functions used to access regmap, and make driver using regmap_*() functions instead. Signed-off-by: Robert Baldyga Reviewed-by: Krzysztof Kozlowski Acked-by: Mark Brown Acked-by: Chanwoo Choi Signed-off-by: Lee Jones --- include/linux/mfd/max77693-private.h | 8 -------- 1 file changed, 8 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/max77693-private.h b/include/linux/mfd/max77693-private.h index 3e050b933dd0..80ec31d561c4 100644 --- a/include/linux/mfd/max77693-private.h +++ b/include/linux/mfd/max77693-private.h @@ -332,14 +332,6 @@ enum max77693_types { TYPE_MAX77693, }; -extern int max77693_read_reg(struct regmap *map, u8 reg, u8 *dest); -extern int max77693_bulk_read(struct regmap *map, u8 reg, int count, - u8 *buf); -extern int max77693_write_reg(struct regmap *map, u8 reg, u8 value); -extern int max77693_bulk_write(struct regmap *map, u8 reg, int count, - u8 *buf); -extern int max77693_update_reg(struct regmap *map, u8 reg, u8 val, u8 mask); - extern int max77693_irq_init(struct max77693_dev *max77686); extern void max77693_irq_exit(struct max77693_dev *max77686); extern int max77693_irq_resume(struct max77693_dev *max77686); -- cgit v1.2.3 From 342d669c1ee421323f552a62729d3a3d0065093c Mon Sep 17 00:00:00 2001 From: Robert Baldyga Date: Wed, 21 May 2014 08:52:48 +0200 Subject: mfd: max77693: Handle IRQs using regmap This patch modifies mfd driver to use regmap for handling interrupts. It allows to simplify irq handling process. This modifications needed to make small changes in function drivers, which use interrupts. Signed-off-by: Robert Baldyga Reviewed-by: Krzysztof Kozlowski Acked-by: Chanwoo Choi Signed-off-by: Lee Jones --- include/linux/mfd/max77693-private.h | 46 ++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mfd/max77693-private.h b/include/linux/mfd/max77693-private.h index 80ec31d561c4..c466ff3e16b8 100644 --- a/include/linux/mfd/max77693-private.h +++ b/include/linux/mfd/max77693-private.h @@ -262,6 +262,41 @@ enum max77693_irq_source { MAX77693_IRQ_GROUP_NR, }; +#define LED_IRQ_FLED2_OPEN BIT(0) +#define LED_IRQ_FLED2_SHORT BIT(1) +#define LED_IRQ_FLED1_OPEN BIT(2) +#define LED_IRQ_FLED1_SHORT BIT(3) +#define LED_IRQ_MAX_FLASH BIT(4) + +#define TOPSYS_IRQ_T120C_INT BIT(0) +#define TOPSYS_IRQ_T140C_INT BIT(1) +#define TOPSYS_IRQ_LOWSYS_INT BIT(3) + +#define CHG_IRQ_BYP_I BIT(0) +#define CHG_IRQ_THM_I BIT(2) +#define CHG_IRQ_BAT_I BIT(3) +#define CHG_IRQ_CHG_I BIT(4) +#define CHG_IRQ_CHGIN_I BIT(6) + +#define MUIC_IRQ_INT1_ADC BIT(0) +#define MUIC_IRQ_INT1_ADC_LOW BIT(1) +#define MUIC_IRQ_INT1_ADC_ERR BIT(2) +#define MUIC_IRQ_INT1_ADC1K BIT(3) + +#define MUIC_IRQ_INT2_CHGTYP BIT(0) +#define MUIC_IRQ_INT2_CHGDETREUN BIT(1) +#define MUIC_IRQ_INT2_DCDTMR BIT(2) +#define MUIC_IRQ_INT2_DXOVP BIT(3) +#define MUIC_IRQ_INT2_VBVOLT BIT(4) +#define MUIC_IRQ_INT2_VIDRM BIT(5) + +#define MUIC_IRQ_INT3_EOC BIT(0) +#define MUIC_IRQ_INT3_CGMBC BIT(1) +#define MUIC_IRQ_INT3_OVP BIT(2) +#define MUIC_IRQ_INT3_MBCCHG_ERR BIT(3) +#define MUIC_IRQ_INT3_CHG_ENABLED BIT(4) +#define MUIC_IRQ_INT3_BAT_DET BIT(5) + enum max77693_irq { /* PMIC - FLASH */ MAX77693_LED_IRQ_FLED2_OPEN, @@ -282,6 +317,10 @@ enum max77693_irq { MAX77693_CHG_IRQ_CHG_I, MAX77693_CHG_IRQ_CHGIN_I, + MAX77693_IRQ_NR, +}; + +enum max77693_irq_muic { /* MUIC INT1 */ MAX77693_MUIC_IRQ_INT1_ADC, MAX77693_MUIC_IRQ_INT1_ADC_LOW, @@ -304,7 +343,7 @@ enum max77693_irq { MAX77693_MUIC_IRQ_INT3_CHG_ENABLED, MAX77693_MUIC_IRQ_INT3_BAT_DET, - MAX77693_IRQ_NR, + MAX77693_MUIC_IRQ_NR, }; struct max77693_dev { @@ -319,7 +358,10 @@ struct max77693_dev { struct regmap *regmap_muic; struct regmap *regmap_haptic; - struct irq_domain *irq_domain; + struct regmap_irq_chip_data *irq_data_led; + struct regmap_irq_chip_data *irq_data_topsys; + struct regmap_irq_chip_data *irq_data_charger; + struct regmap_irq_chip_data *irq_data_muic; int irq; int irq_gpio; -- cgit v1.2.3 From 5165238460068e53c740eaa621ebb6623dc4a50d Mon Sep 17 00:00:00 2001 From: "Zhu, Lejun" Date: Tue, 3 Jun 2014 13:26:02 +0800 Subject: mfd: intel_soc_pmic: Core driver This patch provides the common I2C driver code for Intel SoC PMICs. Signed-off-by: Yang, Bin Signed-off-by: Zhu, Lejun Signed-off-by: Lee Jones --- include/linux/mfd/intel_soc_pmic.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 include/linux/mfd/intel_soc_pmic.h (limited to 'include/linux') diff --git a/include/linux/mfd/intel_soc_pmic.h b/include/linux/mfd/intel_soc_pmic.h new file mode 100644 index 000000000000..abcbfcf32d10 --- /dev/null +++ b/include/linux/mfd/intel_soc_pmic.h @@ -0,0 +1,30 @@ +/* + * intel_soc_pmic.h - Intel SoC PMIC Driver + * + * Copyright (C) 2012-2014 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License version + * 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Author: Yang, Bin + * Author: Zhu, Lejun + */ + +#ifndef __INTEL_SOC_PMIC_H__ +#define __INTEL_SOC_PMIC_H__ + +#include + +struct intel_soc_pmic { + int irq; + struct regmap *regmap; + struct regmap_irq_chip_data *irq_chip_data; +}; + +#endif /* __INTEL_SOC_PMIC_H__ */ -- cgit v1.2.3 From 62fde54123fb64879326c8b71c3f92cc5db1c452 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:34 -0400 Subject: percpu: include/asm-generic/percpu.h should contain only arch-overridable parts The roles of the various percpu header files has become unclear. There are four header files involved. include/linux/percpu-defs.h include/linux/percpu.h include/asm-generic/percpu.h arch/*/include/asm/percpu.h The original intention for include/asm-generic/percpu.h is providing generic definitions for arch-overridable parts; however, it now hosts various stuff which can't be overridden by archs. Also, include/linux/percpu-defs.h was initially added to contain section and percpu variable definition macros so that arch header files can make use of them without worrying about introducing cyclic inclusion dependency by including include/linux/percpu.h; however, arch headers sometimes need to access percpu variables too and this is one of the reasons why some accessors were implemented in include/linux/asm-generic/percpu.h. Let's clear up the situation by making include/asm-generic/percpu.h contain only arch-overridable parts and moving accessors and operations into include/linux/percpu-defs. Note that this patch only moves things from include/asm-generic/percpu.h. include/linux/percpu.h will be taken care of by later patches. This patch moves the followings. * SHIFT_PERCPU_PTR() / VERIFY_PERCPU_PTR() * per_cpu() * raw_cpu_ptr() * this_cpu_ptr() * __get_cpu_var() * __raw_get_cpu_var() * __this_cpu_ptr() * PER_CPU_[SHARED_]ALIGNED_SECTION * PER_CPU_[SHARED_]ALIGNED_SECTION * PER_CPU_FIRST_SECTION This patch is pure reorganization. Signed-off-by: Tejun Heo Acked-by: Christoph Lameter --- include/linux/percpu-defs.h | 89 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) (limited to 'include/linux') diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index a5fc7d01aad6..1a1af3e06a71 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -1,6 +1,40 @@ +/* + * linux/percpu-defs.h - basic definitions for percpu areas + * + * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER. + * + * This file is separate from linux/percpu.h to avoid cyclic inclusion + * dependency from arch header files. Only to be included from + * asm/percpu.h. + * + * This file includes macros necessary to declare percpu sections and + * variables, and definitions of percpu accessors and operations. It + * should provide enough percpu features to arch header files even when + * they can only include asm/percpu.h to avoid cyclic inclusion dependency. + */ + #ifndef _LINUX_PERCPU_DEFS_H #define _LINUX_PERCPU_DEFS_H +#ifdef CONFIG_SMP + +#ifdef MODULE +#define PER_CPU_SHARED_ALIGNED_SECTION "" +#define PER_CPU_ALIGNED_SECTION "" +#else +#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned" +#define PER_CPU_ALIGNED_SECTION "..shared_aligned" +#endif +#define PER_CPU_FIRST_SECTION "..first" + +#else + +#define PER_CPU_SHARED_ALIGNED_SECTION "" +#define PER_CPU_ALIGNED_SECTION "..shared_aligned" +#define PER_CPU_FIRST_SECTION "" + +#endif + /* * Base implementations of per-CPU variable declarations and definitions, where * the section in which the variable is to be placed is provided by the @@ -164,4 +198,59 @@ #define EXPORT_PER_CPU_SYMBOL_GPL(var) #endif +/* + * Accessors and operations. + */ +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_SMP + +/* + * Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE() + * to prevent the compiler from making incorrect assumptions about the + * pointer value. The weird cast keeps both GCC and sparse happy. + */ +#define SHIFT_PERCPU_PTR(__p, __offset) ({ \ + __verify_pcpu_ptr((__p)); \ + RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \ +}) + +/* + * A percpu variable may point to a discarded regions. The following are + * established ways to produce a usable pointer from the percpu variable + * offset. + */ +#define per_cpu(var, cpu) \ + (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) + +#define raw_cpu_ptr(ptr) arch_raw_cpu_ptr(ptr) + +#ifdef CONFIG_DEBUG_PREEMPT +#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#else +#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) +#endif + +#define __get_cpu_var(var) (*this_cpu_ptr(&(var))) +#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var))) + +#else /* CONFIG_SMP */ + +#define VERIFY_PERCPU_PTR(__p) ({ \ + __verify_pcpu_ptr((__p)); \ + (typeof(*(__p)) __kernel __force *)(__p); \ +}) + +#define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var)))) +#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) +#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) +#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) +#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr) + +#endif /* CONFIG_SMP */ + +/* keep until we have removed all uses of __this_cpu_ptr */ +#define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr) + +#endif /* __ASSEMBLY__ */ #endif /* _LINUX_PERCPU_DEFS_H */ -- cgit v1.2.3 From 9defda18f913181debfe7cdc8c0a752f707ac861 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:34 -0400 Subject: percpu: move accessors from include/linux/percpu.h to percpu-defs.h include/linux/percpu-defs.h is gonna host all accessors and operations so that arch headers can make use of them too without worrying about circular dependency through include/linux/percpu.h. This patch moves the following accessors from include/linux/percpu.h to include/linux/percpu-defs.h. * get/put_cpu_var() * get/put_cpu_ptr() * per_cpu_ptr() This is pure reorgniazation. Signed-off-by: Tejun Heo Acked-by: Christoph Lameter --- include/linux/percpu-defs.h | 32 ++++++++++++++++++++++++++++++++ include/linux/percpu.h | 37 ------------------------------------- 2 files changed, 32 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 1a1af3e06a71..f782f98004db 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -252,5 +252,37 @@ /* keep until we have removed all uses of __this_cpu_ptr */ #define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr) +/* + * Must be an lvalue. Since @var must be a simple identifier, + * we force a syntax error here if it isn't. + */ +#define get_cpu_var(var) (*({ \ + preempt_disable(); \ + this_cpu_ptr(&var); })) + +/* + * The weird & is necessary because sparse considers (void)(var) to be + * a direct dereference of percpu variable (var). + */ +#define put_cpu_var(var) do { \ + (void)&(var); \ + preempt_enable(); \ +} while (0) + +#define get_cpu_ptr(var) ({ \ + preempt_disable(); \ + this_cpu_ptr(var); }) + +#define put_cpu_ptr(var) do { \ + (void)(var); \ + preempt_enable(); \ +} while (0) + +#ifdef CONFIG_SMP +#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) +#else +#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) +#endif + #endif /* __ASSEMBLY__ */ #endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 8419053d0f2e..97b207990c45 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -23,32 +23,6 @@ PERCPU_MODULE_RESERVE) #endif -/* - * Must be an lvalue. Since @var must be a simple identifier, - * we force a syntax error here if it isn't. - */ -#define get_cpu_var(var) (*({ \ - preempt_disable(); \ - this_cpu_ptr(&var); })) - -/* - * The weird & is necessary because sparse considers (void)(var) to be - * a direct dereference of percpu variable (var). - */ -#define put_cpu_var(var) do { \ - (void)&(var); \ - preempt_enable(); \ -} while (0) - -#define get_cpu_ptr(var) ({ \ - preempt_disable(); \ - this_cpu_ptr(var); }) - -#define put_cpu_ptr(var) do { \ - (void)(var); \ - preempt_enable(); \ -} while (0) - /* minimum unit size, also is the maximum supported allocation size */ #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) @@ -140,17 +114,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_populate_pte_fn_t populate_pte_fn); #endif -/* - * Use this to get to a cpu's version of the per-cpu object - * dynamically allocated. Non-atomic access to the current CPU's - * version should probably be combined with get_cpu()/put_cpu(). - */ -#ifdef CONFIG_SMP -#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) -#else -#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) -#endif - extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); extern bool is_kernel_percpu_address(unsigned long addr); -- cgit v1.2.3 From 3b8ed91d6463f48ab180f5ebedc9663eddfa0587 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:37 -0400 Subject: percpu: reorganize include/linux/percpu-defs.h Reorganize for better readability. * Accessor definitions are collected into one place and SMP and UP now define them in the same order. * Definitions are layered when possible - e.g. per_cpu() is now defined in terms of this_cpu_ptr(). * Rather pointless comment dropped. * per_cpu(), __raw_get_cpu_var() and __get_cpu_var() are defined in a way which can be shared between SMP and UP and moved out of CONFIG_SMP blocks. This patch doesn't introduce any functional difference. Signed-off-by: Tejun Heo Cc: Christoph Lameter --- include/linux/percpu-defs.h | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index f782f98004db..94cd90afadac 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -215,15 +215,8 @@ RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \ }) -/* - * A percpu variable may point to a discarded regions. The following are - * established ways to produce a usable pointer from the percpu variable - * offset. - */ -#define per_cpu(var, cpu) \ - (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) - -#define raw_cpu_ptr(ptr) arch_raw_cpu_ptr(ptr) +#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) +#define raw_cpu_ptr(ptr) arch_raw_cpu_ptr(ptr) #ifdef CONFIG_DEBUG_PREEMPT #define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) @@ -231,9 +224,6 @@ #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) #endif -#define __get_cpu_var(var) (*this_cpu_ptr(&(var))) -#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var))) - #else /* CONFIG_SMP */ #define VERIFY_PERCPU_PTR(__p) ({ \ @@ -241,14 +231,16 @@ (typeof(*(__p)) __kernel __force *)(__p); \ }) -#define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var)))) -#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) -#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) -#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) -#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr) +#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) +#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) +#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) #endif /* CONFIG_SMP */ +#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu)) +#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var))) +#define __get_cpu_var(var) (*this_cpu_ptr(&(var))) + /* keep until we have removed all uses of __this_cpu_ptr */ #define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr) @@ -278,11 +270,5 @@ preempt_enable(); \ } while (0) -#ifdef CONFIG_SMP -#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) -#else -#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); }) -#endif - #endif /* __ASSEMBLY__ */ #endif /* _LINUX_PERCPU_DEFS_H */ -- cgit v1.2.3 From dcba4333683c3a0642fd575e475c6c740122a037 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:39 -0400 Subject: percpu: only allow sized arch overrides for {raw|this}_cpu_*() ops Currently, percpu allows two separate methods for overriding {raw|this}_cpu_*() ops - for a given operation, an arch can provide whole replacement or sized sub operations to override specific parts of it. e.g. arch either can provide this_cpu_add() or this_cpu_add_4() to override only the 4 byte operation. While quite flexible on a glance, the dual-overriding scheme complicates the code path for no actual gain. It compilcates the already complex operation definitions and if an arch wants to override all sizes, it can easily provide all variants anyway. In fact, no arch is actually making use of whole operation override. Another oddity is that __this_cpu_*() operations are defined in the same way as raw_cpu_*() but ignores full overrides of the raw_cpu_*() and doesn't allow full operation override, so if an arch provides whole overrides for raw_cpu_*() operations __this_cpu_*() ends up using the generic implementations. More importantly, it takes away the layering between arch-specific and generic parts making it impossible for the generic part to implement arch-independent features on top of arch-specific overrides. This patch removes the support for whole operation overrides. As no arch is using it, this doesn't cause any actual difference. Signed-off-by: Tejun Heo Acked-by: Christoph Lameter --- include/linux/percpu.h | 94 +++----------------------------------------------- 1 file changed, 5 insertions(+), 89 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 97b207990c45..95d380e5d246 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -226,17 +226,11 @@ do { \ * safe. Interrupts may occur. If the interrupt modifies the variable * too then RMW actions will not be reliable. * - * The arch code can provide optimized functions in two ways: - * - * 1. Override the function completely. F.e. define this_cpu_add(). - * The arch must then ensure that the various scalar format passed - * are handled correctly. - * - * 2. Provide functions for certain scalar sizes. F.e. provide - * this_cpu_add_2() to provide per cpu atomic operations for 2 byte - * sized RMW actions. If arch code does not provide operations for - * a scalar size then the fallback in the generic code will be - * used. + * The arch code can provide optimized implementation by defining macros + * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per + * cpu atomic operations for 2 byte sized RMW actions. If arch code does + * not provide operations for a scalar size then the fallback in the + * generic code will be used. */ #define _this_cpu_generic_read(pcp) \ @@ -247,7 +241,6 @@ do { \ ret__; \ }) -#ifndef this_cpu_read # ifndef this_cpu_read_1 # define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) # endif @@ -261,7 +254,6 @@ do { \ # define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) # endif # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp)) -#endif #define _this_cpu_generic_to_op(pcp, val, op) \ do { \ @@ -271,7 +263,6 @@ do { \ raw_local_irq_restore(flags); \ } while (0) -#ifndef this_cpu_write # ifndef this_cpu_write_1 # define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif @@ -285,9 +276,7 @@ do { \ # define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) # endif # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val)) -#endif -#ifndef this_cpu_add # ifndef this_cpu_add_1 # define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif @@ -301,21 +290,11 @@ do { \ # define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) # endif # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val)) -#endif -#ifndef this_cpu_sub # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val)) -#endif - -#ifndef this_cpu_inc # define this_cpu_inc(pcp) this_cpu_add((pcp), 1) -#endif - -#ifndef this_cpu_dec # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1) -#endif -#ifndef this_cpu_and # ifndef this_cpu_and_1 # define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif @@ -329,9 +308,7 @@ do { \ # define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) # endif # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val)) -#endif -#ifndef this_cpu_or # ifndef this_cpu_or_1 # define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif @@ -345,7 +322,6 @@ do { \ # define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) # endif # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) -#endif #define _this_cpu_generic_add_return(pcp, val) \ ({ \ @@ -358,7 +334,6 @@ do { \ ret__; \ }) -#ifndef this_cpu_add_return # ifndef this_cpu_add_return_1 # define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif @@ -372,7 +347,6 @@ do { \ # define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val) # endif # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) -#endif #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val)) #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) @@ -388,7 +362,6 @@ do { \ ret__; \ }) -#ifndef this_cpu_xchg # ifndef this_cpu_xchg_1 # define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval) # endif @@ -403,7 +376,6 @@ do { \ # endif # define this_cpu_xchg(pcp, nval) \ __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval) -#endif #define _this_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ @@ -417,7 +389,6 @@ do { \ ret__; \ }) -#ifndef this_cpu_cmpxchg # ifndef this_cpu_cmpxchg_1 # define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) # endif @@ -432,7 +403,6 @@ do { \ # endif # define this_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) -#endif /* * cmpxchg_double replaces two adjacent scalars at once. The first @@ -453,7 +423,6 @@ do { \ ret__; \ }) -#ifndef this_cpu_cmpxchg_double # ifndef this_cpu_cmpxchg_double_1 # define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) @@ -472,7 +441,6 @@ do { \ # endif # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) -#endif /* * Generic percpu operations for contexts where we do not want to do @@ -484,7 +452,6 @@ do { \ * or an interrupt occurred and the same percpu variable was modified from * the interrupt context. */ -#ifndef raw_cpu_read # ifndef raw_cpu_read_1 # define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp))) # endif @@ -498,15 +465,12 @@ do { \ # define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp))) # endif # define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp)) -#endif #define raw_cpu_generic_to_op(pcp, val, op) \ do { \ *raw_cpu_ptr(&(pcp)) op val; \ } while (0) - -#ifndef raw_cpu_write # ifndef raw_cpu_write_1 # define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif @@ -520,9 +484,7 @@ do { \ # define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) # endif # define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val)) -#endif -#ifndef raw_cpu_add # ifndef raw_cpu_add_1 # define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif @@ -536,21 +498,13 @@ do { \ # define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) # endif # define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val)) -#endif -#ifndef raw_cpu_sub # define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val)) -#endif -#ifndef raw_cpu_inc # define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1) -#endif -#ifndef raw_cpu_dec # define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1) -#endif -#ifndef raw_cpu_and # ifndef raw_cpu_and_1 # define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif @@ -564,9 +518,7 @@ do { \ # define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) # endif # define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val)) -#endif -#ifndef raw_cpu_or # ifndef raw_cpu_or_1 # define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif @@ -580,7 +532,6 @@ do { \ # define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) # endif # define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val)) -#endif #define raw_cpu_generic_add_return(pcp, val) \ ({ \ @@ -588,7 +539,6 @@ do { \ raw_cpu_read(pcp); \ }) -#ifndef raw_cpu_add_return # ifndef raw_cpu_add_return_1 # define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) # endif @@ -603,7 +553,6 @@ do { \ # endif # define raw_cpu_add_return(pcp, val) \ __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val) -#endif #define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val)) #define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1) @@ -616,7 +565,6 @@ do { \ ret__; \ }) -#ifndef raw_cpu_xchg # ifndef raw_cpu_xchg_1 # define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) # endif @@ -631,7 +579,6 @@ do { \ # endif # define raw_cpu_xchg(pcp, nval) \ __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval) -#endif #define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ ({ \ @@ -642,7 +589,6 @@ do { \ ret__; \ }) -#ifndef raw_cpu_cmpxchg # ifndef raw_cpu_cmpxchg_1 # define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) # endif @@ -657,7 +603,6 @@ do { \ # endif # define raw_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) -#endif #define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ ({ \ @@ -671,7 +616,6 @@ do { \ (__ret); \ }) -#ifndef raw_cpu_cmpxchg_double # ifndef raw_cpu_cmpxchg_double_1 # define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) @@ -690,79 +634,51 @@ do { \ # endif # define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) -#endif /* * Generic percpu operations for context that are safe from preemption/interrupts. */ -#ifndef __this_cpu_read # define __this_cpu_read(pcp) \ (__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp))) -#endif -#ifndef __this_cpu_write # define __this_cpu_write(pcp, val) \ do { __this_cpu_preempt_check("write"); \ __pcpu_size_call(raw_cpu_write_, (pcp), (val)); \ } while (0) -#endif -#ifndef __this_cpu_add # define __this_cpu_add(pcp, val) \ do { __this_cpu_preempt_check("add"); \ __pcpu_size_call(raw_cpu_add_, (pcp), (val)); \ } while (0) -#endif -#ifndef __this_cpu_sub # define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val)) -#endif - -#ifndef __this_cpu_inc # define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1) -#endif - -#ifndef __this_cpu_dec # define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1) -#endif -#ifndef __this_cpu_and # define __this_cpu_and(pcp, val) \ do { __this_cpu_preempt_check("and"); \ __pcpu_size_call(raw_cpu_and_, (pcp), (val)); \ } while (0) -#endif - -#ifndef __this_cpu_or # define __this_cpu_or(pcp, val) \ do { __this_cpu_preempt_check("or"); \ __pcpu_size_call(raw_cpu_or_, (pcp), (val)); \ } while (0) -#endif -#ifndef __this_cpu_add_return # define __this_cpu_add_return(pcp, val) \ (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)) -#endif #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val)) #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1) #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1) -#ifndef __this_cpu_xchg # define __this_cpu_xchg(pcp, nval) \ (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)) -#endif -#ifndef __this_cpu_cmpxchg # define __this_cpu_cmpxchg(pcp, oval, nval) \ (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)) -#endif -#ifndef __this_cpu_cmpxchg_double # define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))) -#endif #endif /* __LINUX_PERCPU_H */ -- cgit v1.2.3 From 47b69ad673d9aa53c1d6032a6a522fc0ce8d6fc1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:39 -0400 Subject: percpu: move generic {raw|this}_cpu_*_N() definitions to include/asm-generic/percpu.h {raw|this}_cpu_*_N() operations are expected to be provided by archs and the generic definitions are provided as fallbacks. As such, these firmly belong to include/asm-generic/percpu.h. Move the generic definitions to include/asm-generic/percpu.h. The code is moved mostly verbatim; however, raw_cpu_*_N() are placed above this_cpu_*_N() which is more conventional as the raw operations may be used to defined other variants. This is pure reorganization. Signed-off-by: Tejun Heo Acked-by: Christoph Lameter --- include/linux/percpu.h | 344 ------------------------------------------------- 1 file changed, 344 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 95d380e5d246..20b953532596 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -233,174 +233,20 @@ do { \ * generic code will be used. */ -#define _this_cpu_generic_read(pcp) \ -({ typeof(pcp) ret__; \ - preempt_disable(); \ - ret__ = *this_cpu_ptr(&(pcp)); \ - preempt_enable(); \ - ret__; \ -}) - -# ifndef this_cpu_read_1 -# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp) -# endif -# ifndef this_cpu_read_2 -# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp) -# endif -# ifndef this_cpu_read_4 -# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp) -# endif -# ifndef this_cpu_read_8 -# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) -# endif # define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp)) - -#define _this_cpu_generic_to_op(pcp, val, op) \ -do { \ - unsigned long flags; \ - raw_local_irq_save(flags); \ - *raw_cpu_ptr(&(pcp)) op val; \ - raw_local_irq_restore(flags); \ -} while (0) - -# ifndef this_cpu_write_1 -# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef this_cpu_write_2 -# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef this_cpu_write_4 -# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef this_cpu_write_8 -# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) -# endif # define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val)) - -# ifndef this_cpu_add_1 -# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef this_cpu_add_2 -# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef this_cpu_add_4 -# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef this_cpu_add_8 -# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) -# endif # define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val)) - # define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val)) # define this_cpu_inc(pcp) this_cpu_add((pcp), 1) # define this_cpu_dec(pcp) this_cpu_sub((pcp), 1) - -# ifndef this_cpu_and_1 -# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef this_cpu_and_2 -# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef this_cpu_and_4 -# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef this_cpu_and_8 -# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) -# endif # define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val)) - -# ifndef this_cpu_or_1 -# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef this_cpu_or_2 -# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef this_cpu_or_4 -# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef this_cpu_or_8 -# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) -# endif # define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val)) - -#define _this_cpu_generic_add_return(pcp, val) \ -({ \ - typeof(pcp) ret__; \ - unsigned long flags; \ - raw_local_irq_save(flags); \ - raw_cpu_add(pcp, val); \ - ret__ = raw_cpu_read(pcp); \ - raw_local_irq_restore(flags); \ - ret__; \ -}) - -# ifndef this_cpu_add_return_1 -# define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val) -# endif -# ifndef this_cpu_add_return_2 -# define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val) -# endif -# ifndef this_cpu_add_return_4 -# define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val) -# endif -# ifndef this_cpu_add_return_8 -# define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val) -# endif # define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val) - #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val)) #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) - -#define _this_cpu_generic_xchg(pcp, nval) \ -({ typeof(pcp) ret__; \ - unsigned long flags; \ - raw_local_irq_save(flags); \ - ret__ = raw_cpu_read(pcp); \ - raw_cpu_write(pcp, nval); \ - raw_local_irq_restore(flags); \ - ret__; \ -}) - -# ifndef this_cpu_xchg_1 -# define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval) -# endif -# ifndef this_cpu_xchg_2 -# define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval) -# endif -# ifndef this_cpu_xchg_4 -# define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval) -# endif -# ifndef this_cpu_xchg_8 -# define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval) -# endif # define this_cpu_xchg(pcp, nval) \ __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval) - -#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \ -({ \ - typeof(pcp) ret__; \ - unsigned long flags; \ - raw_local_irq_save(flags); \ - ret__ = raw_cpu_read(pcp); \ - if (ret__ == (oval)) \ - raw_cpu_write(pcp, nval); \ - raw_local_irq_restore(flags); \ - ret__; \ -}) - -# ifndef this_cpu_cmpxchg_1 -# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef this_cpu_cmpxchg_2 -# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef this_cpu_cmpxchg_4 -# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef this_cpu_cmpxchg_8 -# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval) -# endif # define this_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) @@ -412,33 +258,6 @@ do { \ * very limited hardware support for these operations, so only certain * sizes may work. */ -#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ -({ \ - int ret__; \ - unsigned long flags; \ - raw_local_irq_save(flags); \ - ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ - oval1, oval2, nval1, nval2); \ - raw_local_irq_restore(flags); \ - ret__; \ -}) - -# ifndef this_cpu_cmpxchg_double_1 -# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef this_cpu_cmpxchg_double_2 -# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef this_cpu_cmpxchg_double_4 -# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef this_cpu_cmpxchg_double_8 -# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif # define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) @@ -452,186 +271,23 @@ do { \ * or an interrupt occurred and the same percpu variable was modified from * the interrupt context. */ -# ifndef raw_cpu_read_1 -# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp))) -# endif -# ifndef raw_cpu_read_2 -# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp))) -# endif -# ifndef raw_cpu_read_4 -# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp))) -# endif -# ifndef raw_cpu_read_8 -# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp))) -# endif # define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp)) - -#define raw_cpu_generic_to_op(pcp, val, op) \ -do { \ - *raw_cpu_ptr(&(pcp)) op val; \ -} while (0) - -# ifndef raw_cpu_write_1 -# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef raw_cpu_write_2 -# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef raw_cpu_write_4 -# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) -# endif -# ifndef raw_cpu_write_8 -# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =) -# endif # define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val)) - -# ifndef raw_cpu_add_1 -# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef raw_cpu_add_2 -# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef raw_cpu_add_4 -# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) -# endif -# ifndef raw_cpu_add_8 -# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=) -# endif # define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val)) - # define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val)) - # define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1) - # define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1) - -# ifndef raw_cpu_and_1 -# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef raw_cpu_and_2 -# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef raw_cpu_and_4 -# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) -# endif -# ifndef raw_cpu_and_8 -# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=) -# endif # define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val)) - -# ifndef raw_cpu_or_1 -# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef raw_cpu_or_2 -# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef raw_cpu_or_4 -# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) -# endif -# ifndef raw_cpu_or_8 -# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=) -# endif # define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val)) - -#define raw_cpu_generic_add_return(pcp, val) \ -({ \ - raw_cpu_add(pcp, val); \ - raw_cpu_read(pcp); \ -}) - -# ifndef raw_cpu_add_return_1 -# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val) -# endif -# ifndef raw_cpu_add_return_2 -# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val) -# endif -# ifndef raw_cpu_add_return_4 -# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val) -# endif -# ifndef raw_cpu_add_return_8 -# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val) -# endif # define raw_cpu_add_return(pcp, val) \ __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val) - #define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val)) #define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1) #define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1) - -#define raw_cpu_generic_xchg(pcp, nval) \ -({ typeof(pcp) ret__; \ - ret__ = raw_cpu_read(pcp); \ - raw_cpu_write(pcp, nval); \ - ret__; \ -}) - -# ifndef raw_cpu_xchg_1 -# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval) -# endif -# ifndef raw_cpu_xchg_2 -# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval) -# endif -# ifndef raw_cpu_xchg_4 -# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval) -# endif -# ifndef raw_cpu_xchg_8 -# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval) -# endif # define raw_cpu_xchg(pcp, nval) \ __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval) - -#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \ -({ \ - typeof(pcp) ret__; \ - ret__ = raw_cpu_read(pcp); \ - if (ret__ == (oval)) \ - raw_cpu_write(pcp, nval); \ - ret__; \ -}) - -# ifndef raw_cpu_cmpxchg_1 -# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef raw_cpu_cmpxchg_2 -# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef raw_cpu_cmpxchg_4 -# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) -# endif -# ifndef raw_cpu_cmpxchg_8 -# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval) -# endif # define raw_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) - -#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ -({ \ - int __ret = 0; \ - if (raw_cpu_read(pcp1) == (oval1) && \ - raw_cpu_read(pcp2) == (oval2)) { \ - raw_cpu_write(pcp1, (nval1)); \ - raw_cpu_write(pcp2, (nval2)); \ - __ret = 1; \ - } \ - (__ret); \ -}) - -# ifndef raw_cpu_cmpxchg_double_1 -# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef raw_cpu_cmpxchg_double_2 -# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef raw_cpu_cmpxchg_double_4 -# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif -# ifndef raw_cpu_cmpxchg_double_8 -# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \ - raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) -# endif # define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) -- cgit v1.2.3 From a32f8d8eda8bd49017ac5f88e2b859f1f582557f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 17 Jun 2014 19:12:39 -0400 Subject: percpu: move {raw|this}_cpu_*() definitions to include/linux/percpu-defs.h We're in the process of moving all percpu accessors and operations to include/linux/percpu-defs.h so that they're available to arch headers without having to include full include/linux/percpu.h which may cause cyclic inclusion dependency. This patch moves {raw|this}_cpu_*() definitions from include/linux/percpu.h to include/linux/percpu-defs.h. The code is moved mostly verbatim; however, raw_cpu_*() are placed above this_cpu_*() which is more conventional as the raw operations may be used to defined other variants. This is pure reorganization. Signed-off-by: Tejun Heo Acked-by: Christoph Lameter --- include/linux/percpu-defs.h | 209 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/percpu.h | 208 ------------------------------------------- 2 files changed, 209 insertions(+), 208 deletions(-) (limited to 'include/linux') diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 94cd90afadac..6710eb9555fa 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -270,5 +270,214 @@ preempt_enable(); \ } while (0) +/* + * Branching function to split up a function into a set of functions that + * are called for different scalar sizes of the objects handled. + */ + +extern void __bad_size_call_parameter(void); + +#ifdef CONFIG_DEBUG_PREEMPT +extern void __this_cpu_preempt_check(const char *op); +#else +static inline void __this_cpu_preempt_check(const char *op) { } +#endif + +#define __pcpu_size_call_return(stem, variable) \ +({ typeof(variable) pscr_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr_ret__ = stem##1(variable);break; \ + case 2: pscr_ret__ = stem##2(variable);break; \ + case 4: pscr_ret__ = stem##4(variable);break; \ + case 8: pscr_ret__ = stem##8(variable);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ + pscr_ret__; \ +}) + +#define __pcpu_size_call_return2(stem, variable, ...) \ +({ \ + typeof(variable) pscr2_ret__; \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ + case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ + case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ + case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pscr2_ret__; \ +}) + +/* + * Special handling for cmpxchg_double. cmpxchg_double is passed two + * percpu variables. The first has to be aligned to a double word + * boundary and the second has to follow directly thereafter. + * We enforce this on all architectures even if they don't support + * a double cmpxchg instruction, since it's a cheap requirement, and it + * avoids breaking the requirement for architectures with the instruction. + */ +#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \ +({ \ + bool pdcrb_ret__; \ + __verify_pcpu_ptr(&pcp1); \ + BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ + VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \ + VM_BUG_ON((unsigned long)(&pcp2) != \ + (unsigned long)(&pcp1) + sizeof(pcp1)); \ + switch(sizeof(pcp1)) { \ + case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ + case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ + case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ + case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ + default: \ + __bad_size_call_parameter(); break; \ + } \ + pdcrb_ret__; \ +}) + +#define __pcpu_size_call(stem, variable, ...) \ +do { \ + __verify_pcpu_ptr(&(variable)); \ + switch(sizeof(variable)) { \ + case 1: stem##1(variable, __VA_ARGS__);break; \ + case 2: stem##2(variable, __VA_ARGS__);break; \ + case 4: stem##4(variable, __VA_ARGS__);break; \ + case 8: stem##8(variable, __VA_ARGS__);break; \ + default: \ + __bad_size_call_parameter();break; \ + } \ +} while (0) + +/* + * this_cpu operations (C) 2008-2013 Christoph Lameter + * + * Optimized manipulation for memory allocated through the per cpu + * allocator or for addresses of per cpu variables. + * + * These operation guarantee exclusivity of access for other operations + * on the *same* processor. The assumption is that per cpu data is only + * accessed by a single processor instance (the current one). + * + * The arch code can provide optimized implementation by defining macros + * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per + * cpu atomic operations for 2 byte sized RMW actions. If arch code does + * not provide operations for a scalar size then the fallback in the + * generic code will be used. + */ + +/* + * Generic percpu operations for contexts where we do not want to do + * any checks for preemptiosn. + * + * If there is no other protection through preempt disable and/or + * disabling interupts then one of these RMW operations can show unexpected + * behavior because the execution thread was rescheduled on another processor + * or an interrupt occurred and the same percpu variable was modified from + * the interrupt context. + */ +# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp)) +# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val)) +# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val)) +# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val)) +# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1) +# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1) +# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val)) +# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val)) +# define raw_cpu_add_return(pcp, val) \ +