3 #undef _bitset_inside_clear
4 #undef _bitset_inside_set
5 #undef _bitset_inside_flip
6 #undef _bitset_inside_is_set
8 #undef _bitset_inside_nlz
9 #undef _bitset_inside_ntz
10 #undef _bitset_inside_ntz_value
12 #define _bitset_inside_set(unit,bit) \
13 __asm__( "btsl %1,%0" :"=m" (unit) :"Ir" (bit))
15 #define _bitset_inside_clear(unit,bit) \
16 __asm__( "btrl %1,%0" :"=m" (unit) :"Ir" (bit))
18 #define _bitset_inside_flip(unit,bit) \
19 __asm__( "btcl %1,%0" :"=m" (unit) :"Ir" (bit))
21 #define _bitset_inside_is_set(unit,bit) _bitset_ia32_inside_is_set(unit, bit)
22 #define _bitset_inside_nlz(unit) _bitset_ia32_inside_nlz(unit)
23 #define _bitset_inside_ntz(unit) _bitset_ia32_inside_ntz(unit)
24 #define _bitset_inside_ntz_value(unit) _bitset_ia32_inside_ntz_value(unit)
26 static INLINE int _bitset_ia32_inside_is_set(unsigned long *unit, unsigned bit)
29 __asm__("mov $0,%0\n\tbtl %1,%2\n\tadc $0,%0"
31 : "Ir" (bit), "m" (unit)
36 static INLINE unsigned _bitset_ia32_inside_nlz(unsigned long *unit)
39 __asm__("bsrl %1,%0" :"=r" (res) :"m" (unit));
40 return *unit == 0 ? 32 : res;
43 static INLINE unsigned _bitset_ia32_inside_ntz(unsigned long *unit) {
45 __asm__("bsfl %1,%0" :"=r" (res) :"m" (unit));
46 return *unit == 0 ? 32 : res;
49 static INLINE unsigned _bitset_ia32_inside_ntz_value(unsigned long unit) {
51 __asm__("bsfl %1,%0" :"=r" (res) :"rm" (unit));
52 return unit == 0 ? 32 : res;
55 #if defined(__GNUC__) && defined(__SSE2__)
58 #include <xmmintrin.h>
61 #undef _bitset_overall_size
62 #undef _bitset_data_ptr
64 #undef _BITSET_BINOP_UNITS_INC
66 #undef _bitset_inside_binop_and
67 #undef _bitset_inside_binop_andnot
68 #undef _bitset_inside_binop_or
69 #undef _bitset_inside_binop_xor
71 #undef _bitset_inside_binop_with_zero_and
72 #undef _bitset_inside_binop_with_zero_andnot
73 #undef _bitset_inside_binop_with_zero_or
74 #undef _bitset_inside_binop_with_zero_xor
76 #define _bitset_units(highest_bit) (round_up2(highest_bit, 128) / BS_UNIT_SIZE_BITS)
78 #define _bitset_overall_size(bitset_base_size,highest_bit) \
79 ((bitset_base_size) + 16 + _bitset_units(highest_bit) * BS_UNIT_SIZE)
81 #define _bitset_data_ptr(data,bitset_base_size,highest_bit) \
82 _bitset_sse_data_ptr(data, bitset_base_size, highest_bit)
84 static INLINE unsigned long *_bitset_sse_data_ptr(void *data, size_t bitset_base_size,
85 unsigned long highest_bit)
90 diff = (units - (char *) 0) + bitset_base_size;
91 diff = round_up2(diff, 16);
92 units = (char *) 0 + diff;
93 return (unsigned long *) units;
96 #define _BITSET_BINOP_UNITS_INC 4
97 #define _bitset_inside_binop_and(tgt,src) _bitset_sse_inside_binop_and(tgt,src)
98 #define _bitset_inside_binop_andnot(tgt,src) _bitset_sse_inside_binop_andnot(tgt,src)
99 #define _bitset_inside_binop_or(tgt,src) _bitset_sse_inside_binop_or(tgt,src)
100 #define _bitset_inside_binop_xor(tgt,src) _bitset_sse_inside_binop_xor(tgt,src)
102 /* and with zero sets everything to zero. */
103 #define _bitset_inside_binop_with_zero_and(tgt) _bitset_sse_inside_binop_with_zero_and(tgt)
105 /* And Not with 0 is the identity (its like and with 1) */
106 #define _bitset_inside_binop_with_zero_andnot(tgt)
108 /* Or with zero is also the identity */
109 #define _bitset_inside_binop_with_zero_or(tgt)
111 /* Xor with 0 is like negation, we have to do it. */
112 #define _bitset_inside_binop_with_zero_xor(tgt) _bitset_sse_inside_binop_with_zero_xor(tgt)
114 #define _BITSET_SSE_BINOP(name) \
115 static INLINE void _bitset_sse_inside_binop_ ## name(unsigned long *tgt, unsigned long *src) \
117 __m128i src_op = _mm_load_si128((__m128i *) src); \
118 __m128i tgt_op = _mm_load_si128((__m128i *) tgt); \
119 __m128i res = _mm_ ## name ## _si128(tgt_op, src_op); \
120 _mm_store_si128((void *) tgt, res); \
124 static INLINE void _bitset_sse_inside_binop_with_zero_and(unsigned long *tgt)
132 static INLINE void _bitset_sse_inside_binop_with_zero_xor(unsigned long *tgt)
134 __m128i src_op = _mm_setzero_si128();
135 __m128i tgt_op = _mm_load_si128((void *) tgt);
136 __m128i res = _mm_xor_si128(tgt_op, src_op);
137 _mm_store_si128((__m128i *) tgt, res);
140 static INLINE void _bitset_sse_inside_binop_andnot(unsigned long *tgt, unsigned long *src)
142 __m128i src_op = _mm_load_si128((void *) src);
143 __m128i tgt_op = _mm_load_si128((void *) tgt);
144 __m128i res = _mm_andnot_si128(src_op, tgt_op);
145 _mm_store_si128((__m128i *) tgt, res);
148 _BITSET_SSE_BINOP(and)
149 _BITSET_SSE_BINOP(or)
150 _BITSET_SSE_BINOP(xor)