bits.h 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. /*
  2. * Copyright (c) Meta Platforms, Inc. and affiliates.
  3. * All rights reserved.
  4. *
  5. * This source code is licensed under both the BSD-style license (found in the
  6. * LICENSE file in the root directory of this source tree) and the GPLv2 (found
  7. * in the COPYING file in the root directory of this source tree).
  8. * You may select, at your option, one of the above-listed licenses.
  9. */
  10. #ifndef ZSTD_BITS_H
  11. #define ZSTD_BITS_H
  12. #include "mem.h"
  13. MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
  14. {
  15. assert(val != 0);
  16. {
  17. static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
  18. 30, 22, 20, 15, 25, 17, 4, 8,
  19. 31, 27, 13, 23, 21, 19, 16, 7,
  20. 26, 12, 18, 6, 11, 5, 10, 9};
  21. return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
  22. }
  23. }
  24. MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
  25. {
  26. assert(val != 0);
  27. # if defined(_MSC_VER)
  28. # if STATIC_BMI2 == 1
  29. return (unsigned)_tzcnt_u32(val);
  30. # else
  31. if (val != 0) {
  32. unsigned long r;
  33. _BitScanForward(&r, val);
  34. return (unsigned)r;
  35. } else {
  36. /* Should not reach this code path */
  37. __assume(0);
  38. }
  39. # endif
  40. # elif defined(__GNUC__) && (__GNUC__ >= 4)
  41. return (unsigned)__builtin_ctz(val);
  42. # else
  43. return ZSTD_countTrailingZeros32_fallback(val);
  44. # endif
  45. }
  46. MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
  47. assert(val != 0);
  48. {
  49. static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
  50. 11, 14, 16, 18, 22, 25, 3, 30,
  51. 8, 12, 20, 28, 15, 17, 24, 7,
  52. 19, 27, 23, 6, 26, 5, 4, 31};
  53. val |= val >> 1;
  54. val |= val >> 2;
  55. val |= val >> 4;
  56. val |= val >> 8;
  57. val |= val >> 16;
  58. return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
  59. }
  60. }
  61. MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
  62. {
  63. assert(val != 0);
  64. # if defined(_MSC_VER)
  65. # if STATIC_BMI2 == 1
  66. return (unsigned)_lzcnt_u32(val);
  67. # else
  68. if (val != 0) {
  69. unsigned long r;
  70. _BitScanReverse(&r, val);
  71. return (unsigned)(31 - r);
  72. } else {
  73. /* Should not reach this code path */
  74. __assume(0);
  75. }
  76. # endif
  77. # elif defined(__GNUC__) && (__GNUC__ >= 4)
  78. return (unsigned)__builtin_clz(val);
  79. # else
  80. return ZSTD_countLeadingZeros32_fallback(val);
  81. # endif
  82. }
  83. MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
  84. {
  85. assert(val != 0);
  86. # if defined(_MSC_VER) && defined(_WIN64)
  87. # if STATIC_BMI2 == 1
  88. return (unsigned)_tzcnt_u64(val);
  89. # else
  90. if (val != 0) {
  91. unsigned long r;
  92. _BitScanForward64(&r, val);
  93. return (unsigned)r;
  94. } else {
  95. /* Should not reach this code path */
  96. __assume(0);
  97. }
  98. # endif
  99. # elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__)
  100. return (unsigned)__builtin_ctzll(val);
  101. # else
  102. {
  103. U32 mostSignificantWord = (U32)(val >> 32);
  104. U32 leastSignificantWord = (U32)val;
  105. if (leastSignificantWord == 0) {
  106. return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
  107. } else {
  108. return ZSTD_countTrailingZeros32(leastSignificantWord);
  109. }
  110. }
  111. # endif
  112. }
  113. MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
  114. {
  115. assert(val != 0);
  116. # if defined(_MSC_VER) && defined(_WIN64)
  117. # if STATIC_BMI2 == 1
  118. return (unsigned)_lzcnt_u64(val);
  119. # else
  120. if (val != 0) {
  121. unsigned long r;
  122. _BitScanReverse64(&r, val);
  123. return (unsigned)(63 - r);
  124. } else {
  125. /* Should not reach this code path */
  126. __assume(0);
  127. }
  128. # endif
  129. # elif defined(__GNUC__) && (__GNUC__ >= 4)
  130. return (unsigned)(__builtin_clzll(val));
  131. # else
  132. {
  133. U32 mostSignificantWord = (U32)(val >> 32);
  134. U32 leastSignificantWord = (U32)val;
  135. if (mostSignificantWord == 0) {
  136. return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
  137. } else {
  138. return ZSTD_countLeadingZeros32(mostSignificantWord);
  139. }
  140. }
  141. # endif
  142. }
  143. MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
  144. {
  145. if (MEM_isLittleEndian()) {
  146. if (MEM_64bits()) {
  147. return ZSTD_countTrailingZeros64((U64)val) >> 3;
  148. } else {
  149. return ZSTD_countTrailingZeros32((U32)val) >> 3;
  150. }
  151. } else { /* Big Endian CPU */
  152. if (MEM_64bits()) {
  153. return ZSTD_countLeadingZeros64((U64)val) >> 3;
  154. } else {
  155. return ZSTD_countLeadingZeros32((U32)val) >> 3;
  156. }
  157. }
  158. }
  159. MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */
  160. {
  161. assert(val != 0);
  162. return 31 - ZSTD_countLeadingZeros32(val);
  163. }
  164. /* ZSTD_rotateRight_*():
  165. * Rotates a bitfield to the right by "count" bits.
  166. * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
  167. */
  168. MEM_STATIC
  169. U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
  170. assert(count < 64);
  171. count &= 0x3F; /* for fickle pattern recognition */
  172. return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
  173. }
  174. MEM_STATIC
  175. U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
  176. assert(count < 32);
  177. count &= 0x1F; /* for fickle pattern recognition */
  178. return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
  179. }
  180. MEM_STATIC
  181. U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
  182. assert(count < 16);
  183. count &= 0x0F; /* for fickle pattern recognition */
  184. return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
  185. }
  186. #endif /* ZSTD_BITS_H */