base_intrinsics.h (8702B)
1 /* See LICENSE for license details. */ 2 #ifndef BASE_INTRINSICS_H 3 #define BASE_INTRINSICS_H 4 5 #include "base_compiler.h" 6 #include "base_types.h" 7 8 #define function static 9 #define global static 10 #define local_persist static 11 12 #ifndef asm 13 #define asm __asm__ 14 #endif 15 16 #ifndef typeof 17 #define typeof __typeof__ 18 #endif 19 20 #if COMPILER_CLANG || COMPILER_GCC 21 #define force_inline inline __attribute__((always_inline)) 22 #elif COMPILER_MSVC 23 #define force_inline __forceinline 24 #endif 25 26 // NOTE(rnp): const is all that is needed for C and is compatible 27 // with all major compilers. Only C++ needs extra care. 28 #if 0 29 #if COMPILER_MSVC || (COMPILER_CLANG && OS_WINDOWS) 30 #pragma section(".rdata$", read) 31 #define read_only __declspec(allocate(".rdata$")) 32 #elif COMPILER_CLANG 33 #define read_only __attribute__((section(".rodata"))) 34 #elif COMPILER_GCC 35 /* TODO(rnp): how do we do this with gcc, putting it in rodata causes warnings and writing to 36 * it doesn't cause a fault */ 37 #define read_only 38 #endif 39 #endif 40 #define read_only static const 41 42 #if !defined(countof) 43 #define countof(a) (i64)(sizeof(a) / sizeof(*a)) 44 #endif 45 46 #if COMPILER_MSVC 47 #define alignas(n) __declspec(align(n)) 48 #define pack_struct(s) __pragma(pack(push, 1)) s __pragma(pack(pop)) 49 #define no_return __declspec(noreturn) 50 51 #define likely(x) (x) 52 #define unlikely(x) (x) 53 54 #define print_format(f, va) 55 56 #define assume(x) __assume(x) 57 #define debugbreak() __debugbreak() 58 #define unreachable() __assume(0) 59 60 #if ARCH_ARM64 61 #define cpu_yield() __yield() 62 #define store_fence() __dmb(0x0A) // 0x0A: ishst 63 #endif 64 65 #define atomic_add_u32(ptr, n) _InterlockedExchangeAdd((volatile u32 *)(ptr), (n)) 66 #define atomic_add_u64(ptr, n) _InterlockedExchangeAdd64((volatile u64 *)(ptr), (n)) 67 #define atomic_and_u32(ptr, n) _InterlockedAnd((volatile u32 *)(ptr), (n)) 68 #define atomic_and_u64(ptr, n) _InterlockedAnd64((volatile u64 *)(ptr), (n)) 69 #define atomic_cas_u32(ptr, cptr, n) (_InterlockedCompareExchange((volatile u32 *)(ptr), *(cptr), (n)) == *(cptr)) 70 #define atomic_cas_u64(ptr, cptr, n) (_InterlockedCompareExchange64((volatile u64 *)(ptr), *(cptr), (n)) == *(cptr)) 71 #define atomic_load_u32(ptr) *((volatile u32 *)(ptr)) 72 #define atomic_load_u64(ptr) *((volatile u64 *)(ptr)) 73 #define atomic_or_u32(ptr, n) _InterlockedOr((volatile u32 *)(ptr), (n)) 74 #define atomic_store_u32(ptr, n) *((volatile u32 *)(ptr)) = (u32)(n) 75 #define atomic_store_u64(ptr, n) *((volatile u64 *)(ptr)) = (u64)(n) 76 #define atomic_swap_u32(ptr, n) _InterlockedExchange((volatile u32 *)(ptr), n) 77 #define atomic_swap_u64(ptr, n) _InterlockedExchange64((volatile u64 *)(ptr), n) 78 79 #define atan2_f32(y, x) atan2f(y, x) 80 #define cos_f32(a) cosf(a) 81 #define sin_f32(a) sinf(a) 82 #define tan_f32(a) tanf(a) 83 #define ceil_f32(a) ceilf(a) 84 #define sqrt_f32(a) sqrtf(a) 85 86 #define exp_f64(a) exp(a) 87 #define sqrt_f64(a) sqrt(a) 88 89 #else 90 #define alignas(n) __attribute__((aligned(n))) 91 #define pack_struct(s) s __attribute__((packed)) 92 #define no_return __attribute__((noreturn)) 93 94 #define likely(x) (__builtin_expect(!!(x), 1)) 95 #define unlikely(x) (__builtin_expect(!!(x), 0)) 96 97 #define print_format(f, va) __attribute__((format(printf, f, va))) 98 99 #if COMPILER_CLANG 100 #define assume(x) __builtin_assume(x) 101 #else 102 #if defined(__has_attribute) 103 #if __has_attribute(assume) 104 #define assume(x) __attribute__((assume(x))) 105 #endif 106 #endif 107 #endif 108 #if !defined(assume) 109 #define assume(x) if (!(x)) unreachable() 110 #endif 111 #define unreachable() __builtin_unreachable() 112 #if ARCH_ARM64 113 /* TODO? debuggers just loop here forever and need a manual PC increment (step over) */ 114 #define debugbreak() asm volatile ("brk 0xf000") 115 #define cpu_yield() asm volatile ("isb") 116 #define store_fence() asm volatile ("dmb ishst" ::: "memory") 117 #else 118 #define debugbreak() asm volatile ("int3; nop") 119 #endif 120 121 #define atomic_add_u64(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST) 122 #define atomic_and_u64(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST) 123 #define atomic_cas_u64(ptr, cptr, n) __atomic_compare_exchange_n(ptr, cptr, n, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) 124 #define atomic_load_u64(ptr) __atomic_load_n(ptr, __ATOMIC_SEQ_CST) 125 #define atomic_or_u32(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST) 126 #define atomic_store_u64(ptr, n) __atomic_store_n(ptr, n, __ATOMIC_SEQ_CST) 127 #define atomic_swap_u64(ptr, n) __atomic_exchange_n(ptr, n, __ATOMIC_SEQ_CST) 128 #define atomic_add_u32 atomic_add_u64 129 #define atomic_and_u32 atomic_and_u64 130 #define atomic_cas_u32 atomic_cas_u64 131 #define atomic_load_u32 atomic_load_u64 132 #define atomic_store_u32 atomic_store_u64 133 #define atomic_swap_u32 atomic_swap_u64 134 135 #define atan2_f32(y, x) __builtin_atan2f(y, x) 136 #define cos_f32(a) __builtin_cosf(a) 137 #define sin_f32(a) __builtin_sinf(a) 138 #define tan_f32(a) __builtin_tanf(a) 139 #define ceil_f32(a) __builtin_ceilf(a) 140 #define sqrt_f32(a) __builtin_sqrtf(a) 141 142 #define exp_f64(a) __builtin_exp(a) 143 #define sqrt_f64(a) __builtin_sqrt(a) 144 145 #define popcount_u64(a) (u64)__builtin_popcountll(a) 146 #endif 147 148 #define KB(a) ((u64)(a) << 10ULL) 149 #define MB(a) ((u64)(a) << 20ULL) 150 #define GB(a) ((u64)(a) << 30ULL) 151 152 #if COMPILER_MSVC 153 154 function force_inline u64 155 clz_u64(u64 a) 156 { 157 u64 result = 64, index; 158 if (a) { 159 _BitScanReverse64(&index, a); 160 result = index; 161 } 162 return result; 163 } 164 165 function force_inline u64 166 ctz_u64(u64 a) 167 { 168 u64 result = 64, index; 169 if (a) { 170 _BitScanForward64(&index, a); 171 result = index; 172 } 173 return result; 174 } 175 176 #else /* !COMPILER_MSVC */ 177 178 function force_inline u64 179 clz_u64(u32 a) 180 { 181 u64 result = 64; 182 if (a) result = (u64)__builtin_clzll(a); 183 return result; 184 } 185 186 function force_inline u64 187 ctz_u64(u64 a) 188 { 189 u64 result = 64; 190 if (a) result = (u64)__builtin_ctzll(a); 191 return result; 192 } 193 194 #endif 195 196 #if ARCH_ARM64 197 /* NOTE(rnp): we are only doing a handful of f32x4 operations so we will just use NEON and do 198 * the macro renaming thing. If you are implementing a serious wide vector operation you should 199 * use SVE(2) instead. The semantics are different however and the code will be written for an 200 * arbitrary vector bit width. In that case you will also need x86_64 code for determining 201 * the supported vector width (ideally at runtime though that may not be possible). 202 */ 203 #include <arm_neon.h> 204 typedef float32x4_t f32x4; 205 typedef int32x4_t i32x4; 206 typedef uint32x4_t u32x4; 207 208 #define add_f32x4(a, b) vaddq_f32(a, b) 209 #define cvt_i32x4_f32x4(a) vcvtq_f32_s32(a) 210 #define cvt_f32x4_i32x4(a) vcvtq_s32_f32(a) 211 #define div_f32x4(a, b) vdivq_f32(a, b) 212 #define dup_f32x4(f) vdupq_n_f32(f) 213 #define floor_f32x4(a) vrndmq_f32(a) 214 #define load_f32x4(a) vld1q_f32(a) 215 #define load_i32x4(a) vld1q_s32(a) 216 #define max_f32x4(a, b) vmaxq_f32(a, b) 217 #define min_f32x4(a, b) vminq_f32(a, b) 218 #define mul_f32x4(a, b) vmulq_f32(a, b) 219 #define set_f32x4(a, b, c, d) vld1q_f32((f32 []){d, c, b, a}) 220 #define sqrt_f32x4(a) vsqrtq_f32(a) 221 #define store_f32x4(o, a) vst1q_f32(o, a) 222 #define store_i32x4(o, a) vst1q_s32(o, a) 223 #define sub_f32x4(a, b) vsubq_f32(a, b) 224 225 #elif ARCH_X64 226 #include <immintrin.h> 227 typedef __m128 f32x4; 228 typedef __m128i i32x4; 229 typedef __m128i u32x4; 230 231 #define add_f32x4(a, b) _mm_add_ps(a, b) 232 #define cvt_i32x4_f32x4(a) _mm_cvtepi32_ps(a) 233 #define cvt_f32x4_i32x4(a) _mm_cvtps_epi32(a) 234 #define div_f32x4(a, b) _mm_div_ps(a, b) 235 #define dup_f32x4(f) _mm_set1_ps(f) 236 #define floor_f32x4(a) _mm_floor_ps(a) 237 #define load_f32x4(a) _mm_loadu_ps(a) 238 #define load_i32x4(a) _mm_loadu_si128((i32x4 *)a) 239 #define max_f32x4(a, b) _mm_max_ps(a, b) 240 #define min_f32x4(a, b) _mm_min_ps(a, b) 241 #define mul_f32x4(a, b) _mm_mul_ps(a, b) 242 #define set_f32x4(a, b, c, d) _mm_set_ps(a, b, c, d) 243 #define sqrt_f32x4(a) _mm_sqrt_ps(a) 244 #define store_f32x4(o, a) _mm_storeu_ps(o, a) 245 #define store_i32x4(o, a) _mm_storeu_si128((i32x4 *)o, a) 246 #define sub_f32x4(a, b) _mm_sub_ps(a, b) 247 248 #define cpu_yield _mm_pause 249 #define store_fence _mm_sfence 250 251 #endif 252 253 function force_inline f32 254 inf32(void) 255 { 256 union {u32 u; f32 f;} result; 257 result.u = 0x7F800000u; 258 return result.f; 259 } 260 261 #endif /* BASE_INTRINSICS_H */