optimization.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. //
  2. // Copyright 2017 The Abseil Authors.
  3. //
  4. // Licensed under the Apache License, Version 2.0 (the "License");
  5. // you may not use this file except in compliance with the License.
  6. // You may obtain a copy of the License at
  7. //
  8. // https://www.apache.org/licenses/LICENSE-2.0
  9. //
  10. // Unless required by applicable law or agreed to in writing, software
  11. // distributed under the License is distributed on an "AS IS" BASIS,
  12. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. // See the License for the specific language governing permissions and
  14. // limitations under the License.
  15. //
  16. // -----------------------------------------------------------------------------
  17. // File: optimization.h
  18. // -----------------------------------------------------------------------------
  19. //
  20. // This header file defines portable macros for performance optimization.
  21. #ifndef ABSL_BASE_OPTIMIZATION_H_
  22. #define ABSL_BASE_OPTIMIZATION_H_
  23. #include <assert.h>
  24. #include "absl/base/config.h"
  25. #include "absl/base/options.h"
  26. // ABSL_BLOCK_TAIL_CALL_OPTIMIZATION
  27. //
  28. // Instructs the compiler to avoid optimizing tail-call recursion. This macro is
  29. // useful when you wish to preserve the existing function order within a stack
  30. // trace for logging, debugging, or profiling purposes.
  31. //
  32. // Example:
  33. //
  34. // int f() {
  35. // int result = g();
  36. // ABSL_BLOCK_TAIL_CALL_OPTIMIZATION();
  37. // return result;
  38. // }
  39. #if defined(__pnacl__)
  40. #define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
  41. #elif defined(__clang__)
  42. // Clang will not tail call given inline volatile assembly.
  43. #define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
  44. #elif defined(__GNUC__)
  45. // GCC will not tail call given inline volatile assembly.
  46. #define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
  47. #elif defined(_MSC_VER)
  48. #include <intrin.h>
  49. // The __nop() intrinsic blocks the optimisation.
  50. #define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __nop()
  51. #else
  52. #define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
  53. #endif
  54. // ABSL_CACHELINE_SIZE
  55. //
  56. // Explicitly defines the size of the L1 cache for purposes of alignment.
  57. // Setting the cacheline size allows you to specify that certain objects be
  58. // aligned on a cacheline boundary with `ABSL_CACHELINE_ALIGNED` declarations.
  59. // (See below.)
  60. //
  61. // NOTE: this macro should be replaced with the following C++17 features, when
  62. // those are generally available:
  63. //
  64. // * `std::hardware_constructive_interference_size`
  65. // * `std::hardware_destructive_interference_size`
  66. //
  67. // See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
  68. // for more information.
  69. #if defined(__GNUC__)
  70. // Cache line alignment
  71. #if defined(__i386__) || defined(__x86_64__)
  72. #define ABSL_CACHELINE_SIZE 64
  73. #elif defined(__powerpc64__)
  74. #define ABSL_CACHELINE_SIZE 128
  75. #elif defined(__aarch64__)
  76. // We would need to read special register ctr_el0 to find out L1 dcache size.
  77. // This value is a good estimate based on a real aarch64 machine.
  78. #define ABSL_CACHELINE_SIZE 64
  79. #elif defined(__arm__)
  80. // Cache line sizes for ARM: These values are not strictly correct since
  81. // cache line sizes depend on implementations, not architectures. There
  82. // are even implementations with cache line sizes configurable at boot
  83. // time.
  84. #if defined(__ARM_ARCH_5T__)
  85. #define ABSL_CACHELINE_SIZE 32
  86. #elif defined(__ARM_ARCH_7A__)
  87. #define ABSL_CACHELINE_SIZE 64
  88. #endif
  89. #endif
  90. #endif
  91. #ifndef ABSL_CACHELINE_SIZE
  92. // A reasonable default guess. Note that overestimates tend to waste more
  93. // space, while underestimates tend to waste more time.
  94. #define ABSL_CACHELINE_SIZE 64
  95. #endif
  96. // ABSL_CACHELINE_ALIGNED
  97. //
  98. // Indicates that the declared object be cache aligned using
  99. // `ABSL_CACHELINE_SIZE` (see above). Cacheline aligning objects allows you to
  100. // load a set of related objects in the L1 cache for performance improvements.
  101. // Cacheline aligning objects properly allows constructive memory sharing and
  102. // prevents destructive (or "false") memory sharing.
  103. //
  104. // NOTE: callers should replace uses of this macro with `alignas()` using
  105. // `std::hardware_constructive_interference_size` and/or
  106. // `std::hardware_destructive_interference_size` when C++17 becomes available to
  107. // them.
  108. //
  109. // See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
  110. // for more information.
  111. //
  112. // On some compilers, `ABSL_CACHELINE_ALIGNED` expands to an `__attribute__`
  113. // or `__declspec` attribute. For compilers where this is not known to work,
  114. // the macro expands to nothing.
  115. //
  116. // No further guarantees are made here. The result of applying the macro
  117. // to variables and types is always implementation-defined.
  118. //
  119. // WARNING: It is easy to use this attribute incorrectly, even to the point
  120. // of causing bugs that are difficult to diagnose, crash, etc. It does not
  121. // of itself guarantee that objects are aligned to a cache line.
  122. //
  123. // NOTE: Some compilers are picky about the locations of annotations such as
  124. // this attribute, so prefer to put it at the beginning of your declaration.
  125. // For example,
  126. //
  127. // ABSL_CACHELINE_ALIGNED static Foo* foo = ...
  128. //
  129. // class ABSL_CACHELINE_ALIGNED Bar { ...
  130. //
  131. // Recommendations:
  132. //
  133. // 1) Consult compiler documentation; this comment is not kept in sync as
  134. // toolchains evolve.
  135. // 2) Verify your use has the intended effect. This often requires inspecting
  136. // the generated machine code.
  137. // 3) Prefer applying this attribute to individual variables. Avoid
  138. // applying it to types. This tends to localize the effect.
  139. #if defined(__clang__) || defined(__GNUC__)
  140. #define ABSL_CACHELINE_ALIGNED __attribute__((aligned(ABSL_CACHELINE_SIZE)))
  141. #elif defined(_MSC_VER)
  142. #define ABSL_CACHELINE_ALIGNED __declspec(align(ABSL_CACHELINE_SIZE))
  143. #else
  144. #define ABSL_CACHELINE_ALIGNED
  145. #endif
  146. // ABSL_PREDICT_TRUE, ABSL_PREDICT_FALSE
  147. //
  148. // Enables the compiler to prioritize compilation using static analysis for
  149. // likely paths within a boolean branch.
  150. //
  151. // Example:
  152. //
  153. // if (ABSL_PREDICT_TRUE(expression)) {
  154. // return result; // Faster if more likely
  155. // } else {
  156. // return 0;
  157. // }
  158. //
  159. // Compilers can use the information that a certain branch is not likely to be
  160. // taken (for instance, a CHECK failure) to optimize for the common case in
  161. // the absence of better information (ie. compiling gcc with `-fprofile-arcs`).
  162. //
  163. // Recommendation: Modern CPUs dynamically predict branch execution paths,
  164. // typically with accuracy greater than 97%. As a result, annotating every
  165. // branch in a codebase is likely counterproductive; however, annotating
  166. // specific branches that are both hot and consistently mispredicted is likely
  167. // to yield performance improvements.
  168. #if ABSL_HAVE_BUILTIN(__builtin_expect) || \
  169. (defined(__GNUC__) && !defined(__clang__))
  170. #define ABSL_PREDICT_FALSE(x) (__builtin_expect(false || (x), false))
  171. #define ABSL_PREDICT_TRUE(x) (__builtin_expect(false || (x), true))
  172. #else
  173. #define ABSL_PREDICT_FALSE(x) (x)
  174. #define ABSL_PREDICT_TRUE(x) (x)
  175. #endif
  176. // `ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL()` aborts the program in the fastest
  177. // possible way, with no attempt at logging. One use is to implement hardening
  178. // aborts with ABSL_OPTION_HARDENED. Since this is an internal symbol, it
  179. // should not be used directly outside of Abseil.
  180. #if ABSL_HAVE_BUILTIN(__builtin_trap) || \
  181. (defined(__GNUC__) && !defined(__clang__))
  182. #define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() __builtin_trap()
  183. #else
  184. #define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() abort()
  185. #endif
  186. // `ABSL_INTERNAL_UNREACHABLE_IMPL()` is the platform specific directive to
  187. // indicate that a statement is unreachable, and to allow the compiler to
  188. // optimize accordingly. Clients should use `ABSL_UNREACHABLE()`, which is
  189. // defined below.
  190. #if defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
  191. #define ABSL_INTERNAL_UNREACHABLE_IMPL() std::unreachable()
  192. #elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
  193. #define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_unreachable()
  194. #elif ABSL_HAVE_BUILTIN(__builtin_assume)
  195. #define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_assume(false)
  196. #elif defined(_MSC_VER)
  197. #define ABSL_INTERNAL_UNREACHABLE_IMPL() __assume(false)
  198. #else
  199. #define ABSL_INTERNAL_UNREACHABLE_IMPL()
  200. #endif
  201. // `ABSL_UNREACHABLE()` is an unreachable statement. A program which reaches
  202. // one has undefined behavior, and the compiler may optimize accordingly.
  203. #if ABSL_OPTION_HARDENED == 1 && defined(NDEBUG)
  204. // Abort in hardened mode to avoid dangerous undefined behavior.
  205. #define ABSL_UNREACHABLE() \
  206. do { \
  207. ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL(); \
  208. ABSL_INTERNAL_UNREACHABLE_IMPL(); \
  209. } while (false)
  210. #else
  211. // The assert only fires in debug mode to aid in debugging.
  212. // When NDEBUG is defined, reaching ABSL_UNREACHABLE() is undefined behavior.
  213. #define ABSL_UNREACHABLE() \
  214. do { \
  215. /* NOLINTNEXTLINE: misc-static-assert */ \
  216. assert(false && "ABSL_UNREACHABLE reached"); \
  217. ABSL_INTERNAL_UNREACHABLE_IMPL(); \
  218. } while (false)
  219. #endif
  220. // ABSL_ASSUME(cond)
  221. //
  222. // Informs the compiler that a condition is always true and that it can assume
  223. // it to be true for optimization purposes.
  224. //
  225. // WARNING: If the condition is false, the program can produce undefined and
  226. // potentially dangerous behavior.
  227. //
  228. // In !NDEBUG mode, the condition is checked with an assert().
  229. //
  230. // NOTE: The expression must not have side effects, as it may only be evaluated
  231. // in some compilation modes and not others. Some compilers may issue a warning
  232. // if the compiler cannot prove the expression has no side effects. For example,
  233. // the expression should not use a function call since the compiler cannot prove
  234. // that a function call does not have side effects.
  235. //
  236. // Example:
  237. //
  238. // int x = ...;
  239. // ABSL_ASSUME(x >= 0);
  240. // // The compiler can optimize the division to a simple right shift using the
  241. // // assumption specified above.
  242. // int y = x / 16;
  243. //
  244. #if !defined(NDEBUG)
  245. #define ABSL_ASSUME(cond) assert(cond)
  246. #elif ABSL_HAVE_BUILTIN(__builtin_assume)
  247. #define ABSL_ASSUME(cond) __builtin_assume(cond)
  248. #elif defined(_MSC_VER)
  249. #define ABSL_ASSUME(cond) __assume(cond)
  250. #elif defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
  251. #define ABSL_ASSUME(cond) \
  252. do { \
  253. if (!(cond)) std::unreachable(); \
  254. } while (false)
  255. #elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
  256. #define ABSL_ASSUME(cond) \
  257. do { \
  258. if (!(cond)) __builtin_unreachable(); \
  259. } while (false)
  260. #else
  261. #define ABSL_ASSUME(cond) \
  262. do { \
  263. static_cast<void>(false && (cond)); \
  264. } while (false)
  265. #endif
  266. // ABSL_INTERNAL_UNIQUE_SMALL_NAME(cond)
  267. // This macro forces small unique name on a static file level symbols like
  268. // static local variables or static functions. This is intended to be used in
  269. // macro definitions to optimize the cost of generated code. Do NOT use it on
  270. // symbols exported from translation unit since it may cause a link time
  271. // conflict.
  272. //
  273. // Example:
  274. //
  275. // #define MY_MACRO(txt)
  276. // namespace {
  277. // char VeryVeryLongVarName[] ABSL_INTERNAL_UNIQUE_SMALL_NAME() = txt;
  278. // const char* VeryVeryLongFuncName() ABSL_INTERNAL_UNIQUE_SMALL_NAME();
  279. // const char* VeryVeryLongFuncName() { return txt; }
  280. // }
  281. //
  282. #if defined(__GNUC__)
  283. #define ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x) #x
  284. #define ABSL_INTERNAL_UNIQUE_SMALL_NAME1(x) ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x)
  285. #define ABSL_INTERNAL_UNIQUE_SMALL_NAME() \
  286. asm(ABSL_INTERNAL_UNIQUE_SMALL_NAME1(.absl.__COUNTER__))
  287. #else
  288. #define ABSL_INTERNAL_UNIQUE_SMALL_NAME()
  289. #endif
  290. #endif // ABSL_BASE_OPTIMIZATION_H_