include/linux/cache.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __LINUX_CACHE_H
   3 #define __LINUX_CACHE_H
   4
   5 #include <uapi/linux/kernel.h>
   6 #include <vdso/cache.h>
   7 #include <asm/cache.h>
   8
   9 #ifndef L1_CACHE_ALIGN
  10 #define L1_CACHE_ALIGN(x) __ALIGN_KERNEL(x, L1_CACHE_BYTES)
  11 #endif
  12
  13 /**
  14  * SMP_CACHE_ALIGN - align a value to the L2 cacheline size
  15  * @x: value to align
  16  *
  17  * On some architectures, L2 ("SMP") CL size is bigger than L1, and sometimes,
  18  * this needs to be accounted.
  19  *
  20  * Return: aligned value.
  21  */
  22 #ifndef SMP_CACHE_ALIGN
  23 #define SMP_CACHE_ALIGN(x)      ALIGN(x, SMP_CACHE_BYTES)
  24 #endif
  25
  26 /*
  27  * ``__aligned_largest`` aligns a field to the value most optimal for the
  28  * target architecture to perform memory operations. Get the actual value
  29  * to be able to use it anywhere else.
  30  */
  31 #ifndef __LARGEST_ALIGN
  32 #define __LARGEST_ALIGN         sizeof(struct { long x; } __aligned_largest)
  33 #endif
  34
  35 #ifndef LARGEST_ALIGN
  36 #define LARGEST_ALIGN(x)        ALIGN(x, __LARGEST_ALIGN)
  37 #endif
  38
  39 /*
  40  * __read_mostly is used to keep rarely changing variables out of frequently
  41  * updated cachelines. Its use should be reserved for data that is used
  42  * frequently in hot paths. Performance traces can help decide when to use
  43  * this. You want __read_mostly data to be tightly packed, so that in the
  44  * best case multiple frequently read variables for a hot path will be next
  45  * to each other in order to reduce the number of cachelines needed to
  46  * execute a critical path. We should be mindful and selective of its use.
  47  * ie: if you're going to use it please supply a *good* justification in your
  48  * commit log
  49  */
  50 #ifndef __read_mostly
  51 #define __read_mostly
  52 #endif
  53
  54 /*
  55  * __ro_after_init is used to mark things that are read-only after init (i.e.
  56  * after mark_rodata_ro() has been called). These are effectively read-only,
  57  * but may get written to during init, so can't live in .rodata (via "const").
  58  */
  59 #ifndef __ro_after_init
  60 #define __ro_after_init __section(".data..ro_after_init")
  61 #endif
  62
  63 #ifndef ____cacheline_aligned_in_smp
  64 #ifdef CONFIG_SMP
  65 #define ____cacheline_aligned_in_smp ____cacheline_aligned
  66 #else
  67 #define ____cacheline_aligned_in_smp
  68 #endif /* CONFIG_SMP */
  69 #endif
  70
  71 #ifndef __cacheline_aligned
  72 #define __cacheline_aligned                                     \
  73   __attribute__((__aligned__(SMP_CACHE_BYTES),                  \
  74                  __section__(".data..cacheline_aligned")))
  75 #endif /* __cacheline_aligned */
  76
  77 #ifndef __cacheline_aligned_in_smp
  78 #ifdef CONFIG_SMP
  79 #define __cacheline_aligned_in_smp __cacheline_aligned
  80 #else
  81 #define __cacheline_aligned_in_smp
  82 #endif /* CONFIG_SMP */
  83 #endif
  84
  85 /*
  86  * The maximum alignment needed for some critical structures
  87  * These could be inter-node cacheline sizes/L3 cacheline
  88  * size etc.  Define this in asm/cache.h for your arch
  89  */
  90 #ifndef INTERNODE_CACHE_SHIFT
  91 #define INTERNODE_CACHE_SHIFT L1_CACHE_SHIFT
  92 #endif
  93
  94 #if !defined(____cacheline_internodealigned_in_smp)
  95 #if defined(CONFIG_SMP)
  96 #define ____cacheline_internodealigned_in_smp \
  97         __attribute__((__aligned__(1 << (INTERNODE_CACHE_SHIFT))))
  98 #else
  99 #define ____cacheline_internodealigned_in_smp
 100 #endif
 101 #endif
 102
 103 #ifndef CONFIG_ARCH_HAS_CACHE_LINE_SIZE
 104 #define cache_line_size()       L1_CACHE_BYTES
 105 #endif
 106
 107 #ifndef __cacheline_group_begin
 108 #define __cacheline_group_begin(GROUP) \
 109         __u8 __cacheline_group_begin__##GROUP[0]
 110 #endif
 111
 112 #ifndef __cacheline_group_end
 113 #define __cacheline_group_end(GROUP) \
 114         __u8 __cacheline_group_end__##GROUP[0]
 115 #endif
 116
 117 /**
 118  * __cacheline_group_begin_aligned - declare an aligned group start
 119  * @GROUP: name of the group
 120  * @...: optional group alignment
 121  *
 122  * The following block inside a struct:
 123  *
 124  *      __cacheline_group_begin_aligned(grp);
 125  *      field a;
 126  *      field b;
 127  *      __cacheline_group_end_aligned(grp);
 128  *
 129  * will always be aligned to either the specified alignment or
 130  * ``SMP_CACHE_BYTES``.
 131  */
 132 #define __cacheline_group_begin_aligned(GROUP, ...)             \
 133         __cacheline_group_begin(GROUP)                          \
 134         __aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)
 135
 136 /**
 137  * __cacheline_group_end_aligned - declare an aligned group end
 138  * @GROUP: name of the group
 139  * @...: optional alignment (same as was in __cacheline_group_begin_aligned())
 140  *
 141  * Note that the end marker is aligned to sizeof(long) to allow more precise
 142  * size assertion. It also declares a padding at the end to avoid next field
 143  * falling into this cacheline.
 144  */
 145 #define __cacheline_group_end_aligned(GROUP, ...)               \
 146         __cacheline_group_end(GROUP) __aligned(sizeof(long));   \
 147         struct { } __cacheline_group_pad__##GROUP               \
 148         __aligned((__VA_ARGS__ + 0) ? : SMP_CACHE_BYTES)
 149
 150 #ifndef CACHELINE_ASSERT_GROUP_MEMBER
 151 #define CACHELINE_ASSERT_GROUP_MEMBER(TYPE, GROUP, MEMBER) \
 152         BUILD_BUG_ON(!(offsetof(TYPE, MEMBER) >= \
 153                        offsetofend(TYPE, __cacheline_group_begin__##GROUP) && \
 154                        offsetofend(TYPE, MEMBER) <= \
 155                        offsetof(TYPE, __cacheline_group_end__##GROUP)))
 156 #endif
 157
 158 #ifndef CACHELINE_ASSERT_GROUP_SIZE
 159 #define CACHELINE_ASSERT_GROUP_SIZE(TYPE, GROUP, SIZE) \
 160         BUILD_BUG_ON(offsetof(TYPE, __cacheline_group_end__##GROUP) - \
 161                      offsetofend(TYPE, __cacheline_group_begin__##GROUP) > \
 162                      SIZE)
 163 #endif
 164
 165 /*
 166  * Helper to add padding within a struct to ensure data fall into separate
 167  * cachelines.
 168  */
 169 #if defined(CONFIG_SMP)
 170 struct cacheline_padding {
 171         char x[0];
 172 } ____cacheline_internodealigned_in_smp;
 173 #define CACHELINE_PADDING(name)         struct cacheline_padding name
 174 #else
 175 #define CACHELINE_PADDING(name)
 176 #endif
 177
 178 #ifdef ARCH_DMA_MINALIGN
 179 #define ARCH_HAS_DMA_MINALIGN
 180 #else
 181 #define ARCH_DMA_MINALIGN __alignof__(unsigned long long)
 182 #endif
 183
 184 #endif /* __LINUX_CACHE_H */