base_intrinsics.h (8538B)
1 /* See LICENSE for license details. */ 2 #ifndef BASE_INTRINSICS_H 3 #define BASE_INTRINSICS_H 4 5 #include "base_compiler.h" 6 #include "base_types.h" 7 8 #define function static 9 #define global static 10 #define local_persist static 11 12 #ifndef asm 13 #define asm __asm__ 14 #endif 15 16 #ifndef typeof 17 #define typeof __typeof__ 18 #endif 19 20 #if COMPILER_CLANG || COMPILER_GCC 21 #define force_inline inline __attribute__((always_inline)) 22 #elif COMPILER_MSVC 23 #define force_inline __forceinline 24 #endif 25 26 #if COMPILER_MSVC || (COMPILER_CLANG && OS_WINDOWS) 27 #pragma section(".rdata$", read) 28 #define read_only __declspec(allocate(".rdata$")) 29 #elif COMPILER_CLANG 30 #define read_only __attribute__((section(".rodata"))) 31 #elif COMPILER_GCC 32 /* TODO(rnp): how do we do this with gcc, putting it in rodata causes warnings and writing to 33 * it doesn't cause a fault */ 34 #define read_only 35 #endif 36 37 #if !defined(countof) 38 #define countof(a) (i64)(sizeof(a) / sizeof(*a)) 39 #endif 40 41 #if COMPILER_MSVC 42 #define alignas(n) __declspec(align(n)) 43 #define pack_struct(s) __pragma(pack(push, 1)) s __pragma(pack(pop)) 44 #define no_return __declspec(noreturn) 45 46 #define likely(x) (x) 47 #define unlikely(x) (x) 48 49 #define print_format(f, va) 50 51 #define assume(x) __assume(x) 52 #define debugbreak() __debugbreak() 53 #define unreachable() __assume(0) 54 55 #if ARCH_ARM64 56 #define cpu_yield() __yield() 57 #define store_fence() __dmb(0x0A) // 0x0A: ishst 58 #endif 59 60 #define atomic_add_u32(ptr, n) _InterlockedExchangeAdd((volatile u32 *)(ptr), (n)) 61 #define atomic_add_u64(ptr, n) _InterlockedExchangeAdd64((volatile u64 *)(ptr), (n)) 62 #define atomic_and_u32(ptr, n) _InterlockedAnd((volatile u32 *)(ptr), (n)) 63 #define atomic_and_u64(ptr, n) _InterlockedAnd64((volatile u64 *)(ptr), (n)) 64 #define atomic_cas_u32(ptr, cptr, n) (_InterlockedCompareExchange((volatile u32 *)(ptr), *(cptr), (n)) == *(cptr)) 65 #define atomic_cas_u64(ptr, cptr, n) (_InterlockedCompareExchange64((volatile u64 *)(ptr), *(cptr), (n)) == *(cptr)) 66 #define atomic_load_u32(ptr) *((volatile u32 *)(ptr)) 67 #define atomic_load_u64(ptr) *((volatile u64 *)(ptr)) 68 #define atomic_or_u32(ptr, n) _InterlockedOr((volatile u32 *)(ptr), (n)) 69 #define atomic_store_u32(ptr, n) *((volatile u32 *)(ptr)) = (u32)(n) 70 #define atomic_store_u64(ptr, n) *((volatile u64 *)(ptr)) = (u64)(n) 71 #define atomic_swap_u32(ptr, n) _InterlockedExchange((volatile u32 *)(ptr), n) 72 #define atomic_swap_u64(ptr, n) _InterlockedExchange64((volatile u64 *)(ptr), n) 73 74 #define atan2_f32(y, x) atan2f(y, x) 75 #define cos_f32(a) cosf(a) 76 #define sin_f32(a) sinf(a) 77 #define tan_f32(a) tanf(a) 78 #define ceil_f32(a) ceilf(a) 79 #define sqrt_f32(a) sqrtf(a) 80 81 #define exp_f64(a) exp(a) 82 #define sqrt_f64(a) sqrt(a) 83 84 #else 85 #define alignas(n) __attribute__((aligned(n))) 86 #define pack_struct(s) s __attribute__((packed)) 87 #define no_return __attribute__((noreturn)) 88 89 #define likely(x) (__builtin_expect(!!(x), 1)) 90 #define unlikely(x) (__builtin_expect(!!(x), 0)) 91 92 #define print_format(f, va) __attribute__((format(printf, f, va))) 93 94 #if COMPILER_CLANG 95 #define assume(x) __builtin_assume(x) 96 #else 97 #if defined(__has_attribute) 98 #if __has_attribute(assume) 99 #define assume(x) __attribute__((assume(x))) 100 #endif 101 #endif 102 #endif 103 #if !defined(assume) 104 #define assume(x) if (!(x)) unreachable() 105 #endif 106 #define unreachable() __builtin_unreachable() 107 #if ARCH_ARM64 108 /* TODO? debuggers just loop here forever and need a manual PC increment (step over) */ 109 #define debugbreak() asm volatile ("brk 0xf000") 110 #define cpu_yield() asm volatile ("yield") 111 #define store_fence() asm volatile ("dmb ishst" ::: "memory") 112 #else 113 #define debugbreak() asm volatile ("int3; nop") 114 #endif 115 116 #define atomic_add_u64(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST) 117 #define atomic_and_u64(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST) 118 #define atomic_cas_u64(ptr, cptr, n) __atomic_compare_exchange_n(ptr, cptr, n, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) 119 #define atomic_load_u64(ptr) __atomic_load_n(ptr, __ATOMIC_SEQ_CST) 120 #define atomic_or_u32(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST) 121 #define atomic_store_u64(ptr, n) __atomic_store_n(ptr, n, __ATOMIC_SEQ_CST) 122 #define atomic_swap_u64(ptr, n) __atomic_exchange_n(ptr, n, __ATOMIC_SEQ_CST) 123 #define atomic_add_u32 atomic_add_u64 124 #define atomic_and_u32 atomic_and_u64 125 #define atomic_cas_u32 atomic_cas_u64 126 #define atomic_load_u32 atomic_load_u64 127 #define atomic_store_u32 atomic_store_u64 128 #define atomic_swap_u32 atomic_swap_u64 129 130 #define atan2_f32(y, x) __builtin_atan2f(y, x) 131 #define cos_f32(a) __builtin_cosf(a) 132 #define sin_f32(a) __builtin_sinf(a) 133 #define tan_f32(a) __builtin_tanf(a) 134 #define ceil_f32(a) __builtin_ceilf(a) 135 #define sqrt_f32(a) __builtin_sqrtf(a) 136 137 #define exp_f64(a) __builtin_exp(a) 138 #define sqrt_f64(a) __builtin_sqrt(a) 139 140 #define popcount_u64(a) (u64)__builtin_popcountll(a) 141 #endif 142 143 #define KB(a) ((u64)(a) << 10ULL) 144 #define MB(a) ((u64)(a) << 20ULL) 145 #define GB(a) ((u64)(a) << 30ULL) 146 147 #if COMPILER_MSVC 148 149 function force_inline u64 150 clz_u64(u64 a) 151 { 152 u64 result = 64, index; 153 if (a) { 154 _BitScanReverse64(&index, a); 155 result = index; 156 } 157 return result; 158 } 159 160 function force_inline u64 161 ctz_u64(u64 a) 162 { 163 u64 result = 64, index; 164 if (a) { 165 _BitScanForward64(&index, a); 166 result = index; 167 } 168 return result; 169 } 170 171 #else /* !COMPILER_MSVC */ 172 173 function force_inline u64 174 clz_u64(u32 a) 175 { 176 u64 result = 64; 177 if (a) result = (u64)__builtin_clzll(a); 178 return result; 179 } 180 181 function force_inline u64 182 ctz_u64(u64 a) 183 { 184 u64 result = 64; 185 if (a) result = (u64)__builtin_ctzll(a); 186 return result; 187 } 188 189 #endif 190 191 #if ARCH_ARM64 192 /* NOTE(rnp): we are only doing a handful of f32x4 operations so we will just use NEON and do 193 * the macro renaming thing. If you are implementing a serious wide vector operation you should 194 * use SVE(2) instead. The semantics are different however and the code will be written for an 195 * arbitrary vector bit width. In that case you will also need x86_64 code for determining 196 * the supported vector width (ideally at runtime though that may not be possible). 197 */ 198 #include <arm_neon.h> 199 typedef float32x4_t f32x4; 200 typedef int32x4_t i32x4; 201 typedef uint32x4_t u32x4; 202 203 #define add_f32x4(a, b) vaddq_f32(a, b) 204 #define cvt_i32x4_f32x4(a) vcvtq_f32_s32(a) 205 #define cvt_f32x4_i32x4(a) vcvtq_s32_f32(a) 206 #define div_f32x4(a, b) vdivq_f32(a, b) 207 #define dup_f32x4(f) vdupq_n_f32(f) 208 #define floor_f32x4(a) vrndmq_f32(a) 209 #define load_f32x4(a) vld1q_f32(a) 210 #define load_i32x4(a) vld1q_s32(a) 211 #define max_f32x4(a, b) vmaxq_f32(a, b) 212 #define min_f32x4(a, b) vminq_f32(a, b) 213 #define mul_f32x4(a, b) vmulq_f32(a, b) 214 #define set_f32x4(a, b, c, d) vld1q_f32((f32 []){d, c, b, a}) 215 #define sqrt_f32x4(a) vsqrtq_f32(a) 216 #define store_f32x4(o, a) vst1q_f32(o, a) 217 #define store_i32x4(o, a) vst1q_s32(o, a) 218 #define sub_f32x4(a, b) vsubq_f32(a, b) 219 220 #elif ARCH_X64 221 #include <immintrin.h> 222 typedef __m128 f32x4; 223 typedef __m128i i32x4; 224 typedef __m128i u32x4; 225 226 #define add_f32x4(a, b) _mm_add_ps(a, b) 227 #define cvt_i32x4_f32x4(a) _mm_cvtepi32_ps(a) 228 #define cvt_f32x4_i32x4(a) _mm_cvtps_epi32(a) 229 #define div_f32x4(a, b) _mm_div_ps(a, b) 230 #define dup_f32x4(f) _mm_set1_ps(f) 231 #define floor_f32x4(a) _mm_floor_ps(a) 232 #define load_f32x4(a) _mm_loadu_ps(a) 233 #define load_i32x4(a) _mm_loadu_si128((i32x4 *)a) 234 #define max_f32x4(a, b) _mm_max_ps(a, b) 235 #define min_f32x4(a, b) _mm_min_ps(a, b) 236 #define mul_f32x4(a, b) _mm_mul_ps(a, b) 237 #define set_f32x4(a, b, c, d) _mm_set_ps(a, b, c, d) 238 #define sqrt_f32x4(a) _mm_sqrt_ps(a) 239 #define store_f32x4(o, a) _mm_storeu_ps(o, a) 240 #define store_i32x4(o, a) _mm_storeu_si128((i32x4 *)o, a) 241 #define sub_f32x4(a, b) _mm_sub_ps(a, b) 242 243 #define cpu_yield _mm_pause 244 #define store_fence _mm_sfence 245 246 #endif 247 248 function force_inline f32 249 inf32(void) 250 { 251 union {u32 u; f32 f;} result; 252 result.u = 0x7F800000u; 253 return result.f; 254 } 255 256 #endif /* BASE_INTRINSICS_H */