blake3_impl.h (9906B)
1 #ifndef BLAKE3_IMPL_H 2 #define BLAKE3_IMPL_H 3 4 #include <assert.h> 5 #include <stdbool.h> 6 #include <stddef.h> 7 #include <stdint.h> 8 #include <string.h> 9 10 #include "blake3.h" 11 12 // internal flags 13 enum blake3_flags { 14 CHUNK_START = 1 << 0, 15 CHUNK_END = 1 << 1, 16 PARENT = 1 << 2, 17 ROOT = 1 << 3, 18 KEYED_HASH = 1 << 4, 19 DERIVE_KEY_CONTEXT = 1 << 5, 20 DERIVE_KEY_MATERIAL = 1 << 6, 21 }; 22 23 // This C implementation tries to support recent versions of GCC, Clang, and 24 // MSVC. 25 #if defined(_MSC_VER) 26 #define INLINE static __forceinline 27 #else 28 #define INLINE static inline __attribute__((always_inline)) 29 #endif 30 31 #if defined(__x86_64__) || defined(_M_X64) 32 #define IS_X86 33 #define IS_X86_64 34 #endif 35 36 #if defined(__i386__) || defined(_M_IX86) 37 #define IS_X86 38 #define IS_X86_32 39 #endif 40 41 #if defined(IS_X86) 42 #if defined(_MSC_VER) 43 #include <intrin.h> 44 #endif 45 #include <immintrin.h> 46 #endif 47 48 #if defined(IS_X86) 49 #define MAX_SIMD_DEGREE 16 50 #elif defined(BLAKE3_USE_NEON) 51 #define MAX_SIMD_DEGREE 4 52 #else 53 #define MAX_SIMD_DEGREE 1 54 #endif 55 56 // There are some places where we want a static size that's equal to the 57 // MAX_SIMD_DEGREE, but also at least 2. 58 #define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2) 59 60 static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 61 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL, 62 0x1F83D9ABUL, 0x5BE0CD19UL}; 63 64 static const uint8_t MSG_SCHEDULE[7][16] = { 65 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, 66 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, 67 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, 68 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, 69 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, 70 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, 71 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, 72 }; 73 74 /* Find index of the highest set bit */ 75 /* x is assumed to be nonzero. */ 76 static unsigned int highest_one(uint64_t x) { 77 #if defined(__GNUC__) || defined(__clang__) 78 return 63 ^ __builtin_clzll(x); 79 #elif defined(_MSC_VER) && defined(IS_X86_64) 80 unsigned long index; 81 _BitScanReverse64(&index, x); 82 return index; 83 #elif defined(_MSC_VER) && defined(IS_X86_32) 84 if(x >> 32) { 85 unsigned long index; 86 _BitScanReverse(&index, x >> 32); 87 return 32 + index; 88 } else { 89 unsigned long index; 90 _BitScanReverse(&index, x); 91 return index; 92 } 93 #else 94 unsigned int c = 0; 95 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; } 96 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; } 97 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; } 98 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; } 99 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; } 100 if(x & 0x0000000000000002ULL) { c += 1; } 101 return c; 102 #endif 103 } 104 105 // Count the number of 1 bits. 106 INLINE unsigned int popcnt(uint64_t x) { 107 #if defined(__GNUC__) || defined(__clang__) 108 return __builtin_popcountll(x); 109 #else 110 unsigned int count = 0; 111 while (x != 0) { 112 count += 1; 113 x &= x - 1; 114 } 115 return count; 116 #endif 117 } 118 119 // Largest power of two less than or equal to x. As a special case, returns 1 120 // when x is 0. 121 INLINE uint64_t round_down_to_power_of_2(uint64_t x) { 122 return 1ULL << highest_one(x | 1); 123 } 124 125 INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } 126 127 INLINE uint32_t counter_high(uint64_t counter) { 128 return (uint32_t)(counter >> 32); 129 } 130 131 INLINE uint32_t load32(const void *src) { 132 const uint8_t *p = (const uint8_t *)src; 133 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | 134 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); 135 } 136 137 INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], 138 uint32_t key_words[8]) { 139 key_words[0] = load32(&key[0 * 4]); 140 key_words[1] = load32(&key[1 * 4]); 141 key_words[2] = load32(&key[2 * 4]); 142 key_words[3] = load32(&key[3 * 4]); 143 key_words[4] = load32(&key[4 * 4]); 144 key_words[5] = load32(&key[5 * 4]); 145 key_words[6] = load32(&key[6 * 4]); 146 key_words[7] = load32(&key[7 * 4]); 147 } 148 149 INLINE void store32(void *dst, uint32_t w) { 150 uint8_t *p = (uint8_t *)dst; 151 p[0] = (uint8_t)(w >> 0); 152 p[1] = (uint8_t)(w >> 8); 153 p[2] = (uint8_t)(w >> 16); 154 p[3] = (uint8_t)(w >> 24); 155 } 156 157 INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) { 158 store32(&bytes_out[0 * 4], cv_words[0]); 159 store32(&bytes_out[1 * 4], cv_words[1]); 160 store32(&bytes_out[2 * 4], cv_words[2]); 161 store32(&bytes_out[3 * 4], cv_words[3]); 162 store32(&bytes_out[4 * 4], cv_words[4]); 163 store32(&bytes_out[5 * 4], cv_words[5]); 164 store32(&bytes_out[6 * 4], cv_words[6]); 165 store32(&bytes_out[7 * 4], cv_words[7]); 166 } 167 168 void blake3_compress_in_place(uint32_t cv[8], 169 const uint8_t block[BLAKE3_BLOCK_LEN], 170 uint8_t block_len, uint64_t counter, 171 uint8_t flags); 172 173 void blake3_compress_xof(const uint32_t cv[8], 174 const uint8_t block[BLAKE3_BLOCK_LEN], 175 uint8_t block_len, uint64_t counter, uint8_t flags, 176 uint8_t out[64]); 177 178 void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, 179 size_t blocks, const uint32_t key[8], uint64_t counter, 180 bool increment_counter, uint8_t flags, 181 uint8_t flags_start, uint8_t flags_end, uint8_t *out); 182 183 size_t blake3_simd_degree(void); 184 185 186 // Declarations for implementation-specific functions. 187 void blake3_compress_in_place_portable(uint32_t cv[8], 188 const uint8_t block[BLAKE3_BLOCK_LEN], 189 uint8_t block_len, uint64_t counter, 190 uint8_t flags); 191 192 void blake3_compress_xof_portable(const uint32_t cv[8], 193 const uint8_t block[BLAKE3_BLOCK_LEN], 194 uint8_t block_len, uint64_t counter, 195 uint8_t flags, uint8_t out[64]); 196 197 void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, 198 size_t blocks, const uint32_t key[8], 199 uint64_t counter, bool increment_counter, 200 uint8_t flags, uint8_t flags_start, 201 uint8_t flags_end, uint8_t *out); 202 203 #if defined(IS_X86) 204 #if !defined(BLAKE3_NO_SSE2) 205 void blake3_compress_in_place_sse2(uint32_t cv[8], 206 const uint8_t block[BLAKE3_BLOCK_LEN], 207 uint8_t block_len, uint64_t counter, 208 uint8_t flags); 209 void blake3_compress_xof_sse2(const uint32_t cv[8], 210 const uint8_t block[BLAKE3_BLOCK_LEN], 211 uint8_t block_len, uint64_t counter, 212 uint8_t flags, uint8_t out[64]); 213 void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs, 214 size_t blocks, const uint32_t key[8], 215 uint64_t counter, bool increment_counter, 216 uint8_t flags, uint8_t flags_start, 217 uint8_t flags_end, uint8_t *out); 218 #endif 219 #if !defined(BLAKE3_NO_SSE41) 220 void blake3_compress_in_place_sse41(uint32_t cv[8], 221 const uint8_t block[BLAKE3_BLOCK_LEN], 222 uint8_t block_len, uint64_t counter, 223 uint8_t flags); 224 void blake3_compress_xof_sse41(const uint32_t cv[8], 225 const uint8_t block[BLAKE3_BLOCK_LEN], 226 uint8_t block_len, uint64_t counter, 227 uint8_t flags, uint8_t out[64]); 228 void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs, 229 size_t blocks, const uint32_t key[8], 230 uint64_t counter, bool increment_counter, 231 uint8_t flags, uint8_t flags_start, 232 uint8_t flags_end, uint8_t *out); 233 #endif 234 #if !defined(BLAKE3_NO_AVX2) 235 void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs, 236 size_t blocks, const uint32_t key[8], 237 uint64_t counter, bool increment_counter, 238 uint8_t flags, uint8_t flags_start, 239 uint8_t flags_end, uint8_t *out); 240 #endif 241 #if !defined(BLAKE3_NO_AVX512) 242 void blake3_compress_in_place_avx512(uint32_t cv[8], 243 const uint8_t block[BLAKE3_BLOCK_LEN], 244 uint8_t block_len, uint64_t counter, 245 uint8_t flags); 246 247 void blake3_compress_xof_avx512(const uint32_t cv[8], 248 const uint8_t block[BLAKE3_BLOCK_LEN], 249 uint8_t block_len, uint64_t counter, 250 uint8_t flags, uint8_t out[64]); 251 252 void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs, 253 size_t blocks, const uint32_t key[8], 254 uint64_t counter, bool increment_counter, 255 uint8_t flags, uint8_t flags_start, 256 uint8_t flags_end, uint8_t *out); 257 #endif 258 #endif 259 260 #if defined(BLAKE3_USE_NEON) 261 void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs, 262 size_t blocks, const uint32_t key[8], 263 uint64_t counter, bool increment_counter, 264 uint8_t flags, uint8_t flags_start, 265 uint8_t flags_end, uint8_t *out); 266 #endif 267 268 269 #endif /* BLAKE3_IMPL_H */