/xmlbench/trunk

/*  Idealized SIMD Operations with SSE versions
    Copyright (C) 2006, 2007, 2008, Robert D. Cameron and Dan Lin
    Licensed to the public under the Open Software License 3.0.
    Licensed to International Characters Inc. 
       under the Academic Free License version 3.0.
*/
#ifndef SSE_SIMD_H
#define SSE_SIMD_H

/*------------------------------------------------------------*/
#ifndef _MSC_VER
#include <stdint.h>
#endif
#ifdef _MSC_VER
#include "stdint.h"
#define LITTLE_ENDIAN 1234
#define BIG_ENDIAN 4321
#define BYTE_ORDER LITTLE_ENDIAN
#endif
#include <limits.h>
#ifndef LONG_BIT
#define LONG_BIT (8* sizeof(unsigned long))
#endif 
#include <emmintrin.h>
#ifdef USE_LDDQU
#include <pmmintrin.h>
#endif
typedef __m128i SIMD_type;
/*------------------------------------------------------------*/
/* I. SIMD bitwise logical operations */

static inline SIMD_type simd_and(SIMD_type b1, SIMD_type b2) {
	return _mm_and_si128(b1, b2);
}
static inline SIMD_type simd_andc(SIMD_type b1, SIMD_type b2) {
	return _mm_andnot_si128(b2, b1);
}
static inline SIMD_type simd_or(SIMD_type b1, SIMD_type b2) {
	return  _mm_or_si128(b1, b2);
}
static inline SIMD_type simd_xor(SIMD_type b1, SIMD_type b2) {
	return  _mm_xor_si128(b1, b2);
}
static inline SIMD_type simd_not(SIMD_type b) {
	return  simd_xor(b, _mm_set1_epi32(0xFFFFFFFF));
}
static inline SIMD_type simd_nor(SIMD_type b1, SIMD_type b2) {
	return  simd_not(simd_or(b1,b2));
}
static inline SIMD_type simd_if(SIMD_type cond, SIMD_type then_val, SIMD_type else_val) {	
	return  simd_or(simd_and(then_val, cond), simd_andc(else_val, cond));
}


/*------------------------------------------------------------*/
/* II. Declarations of field-width based operations. */

/*  Half-operand modifier specifications use "x", "h" or "l",
 *  "x" - no modification of the corresponding operand value
 *  "h" - each n-bit field is modified by taking the high n/2 bits.
 *  "l" - each n-bit field is modified by taking the low n/2 bits. */
 
enum HOM_t {x,h,l};

/* simd<fw> is a template struct providing all the simd operations
 * for a given field width.  */
template <int fw>
struct simd {
	/* The himask selector in which each field is fw/2 1 bits,
	 * followed by fw/2 0 bits. */
	static inline SIMD_type himask();
	
	/* Splat constant generator with compile-time constant. */
	template <int v> static inline SIMD_type constant();
	/* Splat generator using the first field of a register. */
	static inline SIMD_type splat(SIMD_type r);
	
	/* Shift immediate with the shift constant as a template parameter. */
	template <int shft> static inline SIMD_type srli(SIMD_type r);
	template <int shft> static inline SIMD_type slli(SIMD_type r);
	template <int shft> static inline SIMD_type srai(SIMD_type r);
	
	/* Shift operations with register-specified shift values. */
	static inline SIMD_type srl(SIMD_type r, SIMD_type shft);
	static inline SIMD_type sll(SIMD_type r, SIMD_type shft);
	
	/* Binary operations. */
	static inline SIMD_type add(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type sub(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type mult(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type max(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type eq(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type gt(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type pack(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type mergeh(SIMD_type r1, SIMD_type r2);
	static inline SIMD_type mergel(SIMD_type r1, SIMD_type r2);

//	/* Functions for half-operand modification. */
//	
//	template <HOM_t m> static inline SIMD_type hom(SIMD_type r);
//	template <HOM_t m> static inline SIMD_type hx(SIMD_type r);
	
	/* Binary operations with half-operand modifiers */
	
	template <HOM_t m1, HOM_t m2> static inline SIMD_type add(SIMD_type r1, SIMD_type r2);
	template <HOM_t m1, HOM_t m2> static inline SIMD_type sub(SIMD_type r1, SIMD_type r2);
	template <HOM_t m1, HOM_t m2> static inline SIMD_type mult(SIMD_type r1, SIMD_type r2);
	template <HOM_t m1, HOM_t m2> static inline SIMD_type pack(SIMD_type r1, SIMD_type r2);
	template <HOM_t m1, HOM_t m2> static inline SIMD_type mergeh(SIMD_type r1, SIMD_type r2);
	template <HOM_t m1, HOM_t m2> static inline SIMD_type mergel(SIMD_type r1, SIMD_type r2);
};

#define sisd_to_int(x) _mm_cvtsi128_si32(x)

#define sisd_from_int(n) _mm_cvtsi32_si128(n)




/* III.  Implementations of simd<fw> operations. */

/* Constant generator functions for various field widths. */

template<> inline SIMD_type simd<2>::himask() {return _mm_set1_epi8(0xAA);}

template<> inline SIMD_type simd<4>::himask() {return _mm_set1_epi8(0xCC);}

template<> inline SIMD_type simd<8>::himask() {return _mm_set1_epi8(0xF0);}

template<> inline SIMD_type simd<16>::himask() {return _mm_set1_epi16(0xFF00);}

template<> inline SIMD_type simd<32>::himask() {return _mm_set1_epi32(0xFFFF0000);}

template<> inline SIMD_type simd<64>::himask() {return _mm_set_epi32(-1,0,-1,0);}

template<> inline SIMD_type simd<128>::himask() {return _mm_set_epi32(-1,-1,0,0);}

template<> template <int n> inline SIMD_type simd<4>::constant() {return _mm_set1_epi8((n)<<4|(n));}

template<> template <int n> inline SIMD_type simd<8>::constant() {return _mm_set1_epi8(n);}

template<> template <int n> inline SIMD_type simd<16>::constant() {return _mm_set1_epi16(n);}

template<> template <int n> inline SIMD_type simd<32>::constant() {return _mm_set1_epi32(n);}

template<> template <> inline SIMD_type simd<1>::constant<0>() {return simd<8>::constant<0>();}
template<> template <> inline SIMD_type simd<1>::constant<1>() {return simd<8>::constant<-1>();}

template<> template <int n> inline SIMD_type simd<2>::constant() {return simd<4>::constant<(n<<2|n)>();}

// Splat the first 16-bit int into all positions.
template <> inline SIMD_type simd<16>::splat(SIMD_type x) {
  SIMD_type t = _mm_shufflelo_epi16(x,0);
  return _mm_shuffle_epi32(t,0);
}

// Splat the first 32-bit int into all positions.
template <> inline SIMD_type simd<32>::splat(SIMD_type x) {
  return _mm_shuffle_epi32(x,0);
}

/* Shift immediate operations with direct implementation by built-ins. */

template<> template<int sh> inline SIMD_type simd<16>::slli(SIMD_type r) {return _mm_slli_epi16(r, sh);}

template<> template<int sh> inline SIMD_type simd<32>::slli(SIMD_type r) {return _mm_slli_epi32(r, sh);}

template<> template<int sh> inline SIMD_type simd<64>::slli(SIMD_type r) {return _mm_slli_epi64(r, sh);}

template<> template<int sh> inline SIMD_type simd<16>::srli(SIMD_type r) {return _mm_srli_epi16(r, sh);}

template<> template<int sh> inline SIMD_type simd<32>::srli(SIMD_type r) {return _mm_srli_epi32(r, sh);}

template<> template<int sh> inline SIMD_type simd<64>::srli(SIMD_type r) {return _mm_srli_epi64(r, sh);}

/* simd_srai
 * fw: 16,32*/
template<> template<int sh> inline SIMD_type simd<16>::srai(SIMD_type r) {return _mm_srai_epi16(r, sh);}

template<> template<int sh> inline SIMD_type simd<32>::srai(SIMD_type r) {return _mm_srai_epi32(r, sh);}
                  


/* General rules for slli/srli for field widths 2, 4, 8 in terms of 32-bit shifts. */


// Doesn't work:
//template<int fw> template<int sh>
//inline SIMD_type simd<fw>::slli(SIMD_type r) {
//	return simd_and(simd<32>::slli<sh>(r), simd<fw>::constant<6>());
//}
//


template<> template<int sh>
inline SIMD_type simd<2>::slli(SIMD_type r) {
	return simd_and(simd<32>::slli<sh>(r),simd<2>::constant<((3<<sh)&3)>());
}

template<> template<int sh>
inline SIMD_type simd<4>::slli(SIMD_type r) {
	return simd_and(simd<32>::slli<sh>(r),simd<4>::constant<((15<<sh)&15)>());
}

template<> template<int sh>
inline SIMD_type simd<8>::slli(SIMD_type r) {
	return simd_and(simd<32>::slli<sh>(r),simd<8>::constant<((255<<sh)&255)>());
}


//template<int fw> template<int sh>
//inline SIMD_type simd<fw>::srli(SIMD_type r) {
//	return simd_and(simd<32>::srli<sh>(r),simd<fw>::constant<((1<<(fw-sh))-1)>());
//}
//


template<> template<int sh>
inline SIMD_type simd<2>::srli(SIMD_type r) {
	return simd_and(simd<32>::srli<sh>(r),simd<2>::constant<(3>>sh)>());
}

template<> template<int sh>
inline SIMD_type simd<4>::srli(SIMD_type r) {
	return simd_and(simd<32>::srli<sh>(r),simd<4>::constant<(15>>sh)>());
}

template<> template<int sh>
inline SIMD_type simd<8>::srli(SIMD_type r) {
	return simd_and(simd<32>::srli<sh>(r),simd<8>::constant<(255>>sh)>());
}




/* Shift immediate for 128-bit fields */

template<> template<int shft>
inline SIMD_type simd<128>::slli(SIMD_type r) {
	return (shft % 8 == 0 ? _mm_slli_si128(r, shft/8) :
		shft >= 64 ? simd<64>::slli<shft-64>(_mm_slli_si128(r, 8)) :
		simd_or(simd<64>::slli<shft>(r), _mm_slli_si128(simd<64>::srli<64-shft>(r), 8)));
}

template<> template<int shft>
inline SIMD_type simd<128>::srli(SIMD_type r) {
	return (shft % 8 == 0 ? _mm_srli_si128(r, shft/8) :
   		shft >= 64 ? simd<64>::srli<shft-64>(_mm_srli_si128(r, 8)) :
		simd_or(simd<64>::srli<shft>(r), _mm_srli_si128(simd<64>::slli<64-shft>(r), 8)));
}


/* Shifts with shift values specified in an operand register. */

template<>
inline SIMD_type simd<128>::srl(SIMD_type r, SIMD_type shft) {
	return simd_or(_mm_srl_epi64(r, shft),
           	       simd_or(_mm_srli_si128(_mm_srl_epi64(r, _mm_sub_epi32(shft, sisd_from_int(64))), 8),
			       _mm_srli_si128(_mm_sll_epi64(r, _mm_sub_epi32(sisd_from_int(64), shft)), 8)));
}

template<>
inline SIMD_type simd<128>::sll(SIMD_type r, SIMD_type shft) {
	return simd_or(_mm_sll_epi64(r, shft),
           	       simd_or(_mm_slli_si128(_mm_sll_epi64(r, _mm_sub_epi32(shft, sisd_from_int(64))), 8),
			       _mm_slli_si128(_mm_srl_epi64(r, _mm_sub_epi32(sisd_from_int(64), shft)), 8)));
}

template<>
inline SIMD_type simd<64>::srl(SIMD_type r, SIMD_type shft) {
	return simd_if(simd<128>::himask(),
		       _mm_srl_epi64(r, _mm_srli_si128(shft, 8)),
		       _mm_srl_epi64(r, simd_andc(shft, simd<128>::himask())));
}

template<>
inline SIMD_type simd<64>::sll(SIMD_type r, SIMD_type shft) {
	return simd_if(simd<128>::himask(),
		       _mm_sll_epi64(r, _mm_srli_si128(shft, 8)),
		       _mm_sll_epi64(r, simd_andc(shft, simd<128>::himask())));
}


/* simd_add
 * fw: 2,4,8,16,32,64

   Use built-ins for 8, 16, 32, 64, simulations for 2, 4. */

template<> inline SIMD_type simd<8>::add(SIMD_type r1, SIMD_type r2) {return _mm_add_epi8(r1, r2);}

template<> inline SIMD_type simd<16>::add(SIMD_type r1, SIMD_type r2) {return _mm_add_epi16(r1, r2);}

template<> inline SIMD_type simd<32>::add(SIMD_type r1, SIMD_type r2) {return _mm_add_epi32(r1, r2);}

template<> inline SIMD_type simd<64>::add(SIMD_type r1, SIMD_type r2) {return _mm_add_epi64(r1, r2);}

template<>
inline SIMD_type simd<2>::add(SIMD_type r1, SIMD_type r2) {
	 SIMD_type c1 = simd_xor(r1,r2);
	 SIMD_type borrow = simd_and(r1,r2);
	 SIMD_type c2 = simd_xor(c1,(simd<128>::slli<1>(borrow)));
	 return simd_if(simd<2>::himask(),c2,c1);
}

template<>
SIMD_type simd<4>::add(SIMD_type r1, SIMD_type r2) {
	return simd_if(simd<8>::himask(), 
	               simd<8>::add(r1,simd_and(r2,simd<8>::himask())),
	               simd<8>::add(r1, r2));
}

/* simd_sub
 * fw: 2,4,8,16,32,64

   Use built-ins for 8, 16, 32, 64, simulations for 2, 4. */

template<> inline SIMD_type simd<8>::sub(SIMD_type r1, SIMD_type r2) {return _mm_sub_epi8(r1, r2);}

template<> inline SIMD_type simd<16>::sub(SIMD_type r1, SIMD_type r2) {return _mm_sub_epi16(r1, r2);}

template<> inline SIMD_type simd<32>::sub(SIMD_type r1, SIMD_type r2) {return _mm_sub_epi32(r1, r2);}

template<> inline SIMD_type simd<64>::sub(SIMD_type r1, SIMD_type r2) {return _mm_sub_epi64(r1, r2);}


template<>
inline SIMD_type simd<2>::sub(SIMD_type r1, SIMD_type r2)
{
	 SIMD_type c1 = simd_xor(r1,r2);
	 SIMD_type borrow = simd_andc(r2,r1);
	 SIMD_type c2 = simd_xor(c1,(simd<128>::slli<1>(borrow)));
	 return simd_if(simd<2>::himask(),c2,c1);
}

template<>
inline SIMD_type simd<4>::sub(SIMD_type r1, SIMD_type r2){
	return simd_if(simd<8>::himask(), 
	               simd<8>::sub(r1, simd_and(r2,simd<8>::himask())),
	               simd<8>::sub(r1, r2));
}

/* simd_mult for 16 bits only. */

template<> inline SIMD_type simd<16>::mult(SIMD_type r1, SIMD_type r2) {return _mm_mullo_epi16(r1, r2);}

/* simd_max for 8 bits only. */

template<> inline SIMD_type simd<8>::max(SIMD_type r1, SIMD_type r2) {return _mm_max_epu8(r1, r2);}


/* simd_eq
 * fw: 8,16,32*/

template<> inline SIMD_type simd<8>::eq(SIMD_type r1, SIMD_type r2) {return _mm_cmpeq_epi8(r1, r2);}

template<> inline SIMD_type simd<16>::eq(SIMD_type r1, SIMD_type r2) {return _mm_cmpeq_epi16(r1, r2);}

template<> inline SIMD_type simd<32>::eq(SIMD_type r1, SIMD_type r2) {return _mm_cmpeq_epi32(r1, r2);}



/*simd_pack
 * fw: 2,4,8,16*/

/* Built-in operation for fw = 16. */
template<>
inline SIMD_type simd<16>::pack(SIMD_type r1, SIMD_type r2) {
	return _mm_packus_epi16(simd_andc(r2, simd<16>::himask()), simd_andc(r1, simd<16>::himask()));
}

/* fw: 2, 4, 8 */
template<int fw>
inline SIMD_type simd<fw>::pack(SIMD_type r1, SIMD_type r2){
	return simd<fw*2>::pack(simd_if(simd<fw>::himask(),simd<128>::srli<fw/2>(r1),r1),
				simd_if(simd<fw>::himask(),simd<128>::srli<fw/2>(r2),r2));
}

/* simd_mergeh
 * fw: 1,2,4,8,16,32,64*/
template<int fw>
inline SIMD_type simd<fw>::mergeh(SIMD_type r1, SIMD_type r2){
	/*fw: 1,2,4*/
	return simd<fw*2>::mergeh(simd_if(simd<fw*2>::himask(),r1,simd<fw*2>::srli<fw>(r2)),
				  simd_if(simd<fw*2>::himask(),simd<fw*2>::slli<fw>(r1),r2));
}

template<> inline SIMD_type simd<8>::mergeh(SIMD_type r1, SIMD_type r2) {return _mm_unpackhi_epi8(r2, r1);}
template<> inline SIMD_type simd<16>::mergeh(SIMD_type r1, SIMD_type r2) {return _mm_unpackhi_epi16(r2, r1);}
template<> inline SIMD_type simd<32>::mergeh(SIMD_type r1, SIMD_type r2) {return _mm_unpackhi_epi32(r2, r1);}
template<> inline SIMD_type simd<64>::mergeh(SIMD_type r1, SIMD_type r2) {return _mm_unpackhi_epi64(r2, r1);}


/* simd_mergel
 * fw: 1,2,4,8,16,32,64*/
template<int fw>
inline SIMD_type simd<fw>::mergel(SIMD_type r1, SIMD_type r2){
	/*fw: 1,2,4*/
	return simd<fw*2>::mergel(simd_if(simd<fw*2>::himask(),r1,simd<fw*2>::srli<fw>(r2)),
				  simd_if(simd<fw*2>::himask(),simd<fw*2>::slli<fw>(r1),r2));
}

template<> inline SIMD_type simd<8>::mergel(SIMD_type r1, SIMD_type r2) {return _mm_unpacklo_epi8(r2, r1);}
template<> inline SIMD_type simd<16>::mergel(SIMD_type r1, SIMD_type r2) {return _mm_unpacklo_epi16(r2, r1);}
template<> inline SIMD_type simd<32>::mergel(SIMD_type r1, SIMD_type r2) {return _mm_unpacklo_epi32(r2, r1);}
template<> inline SIMD_type simd<64>::mergel(SIMD_type r1, SIMD_type r2) {return _mm_unpacklo_epi64(r2, r1);}





#define simd_all_eq_8(v1, v2) simd_all_true<8>(_mm_cmpeq_epi8(v1, v2))
#define simd_all_le_8(v1, v2) simd_all_eq_8(simd_max_8(v1, v2), v2)

#define simd_all_signed_gt_8(v1, v2) simd_all_true_8(_mm_cmpgt_epi8(v1, v2))

#define simd_cmpgt_8(v1,v2) _mm_cmpgt_epi8(v1, v2)



/* simd_all_true
 * fw: 8*/
template<int fw>
static inline int simd_all_true(SIMD_type r);
template<>
static inline int simd_all_true<8>(SIMD_type r) {
	return _mm_movemask_epi8(r) == 0xFFFF;
}

/* simd_any_true
 * fw: 8*/
template<int fw>
static inline int simd_any_true(SIMD_type r);
template<>
static inline int simd_any_true<8>(SIMD_type r) {
	return _mm_movemask_epi8(r) != 0;
}

/* simd_any_sign_bit
 * fw: 8*/
template<int fw>
static inline int simd_any_sign_bit(SIMD_type r);
template<>
static inline int simd_any_sign_bit<8>(SIMD_type r) {
	return _mm_movemask_epi8(r) != 0;
}



/* IV.  Half operand modifiers - implementations. */
/* Half operand modifier functions.*/

/* Half operand modifier*/
/* Half operand modifier*/
template <int fw, HOM_t m>
struct SIMD {
	static inline SIMD_type hom(SIMD_type r) {}
};

template <int fw>
struct SIMD<fw, x> {
	static inline SIMD_type hom(SIMD_type r) {return r;}
	static inline SIMD_type l2x(SIMD_type r) {return r;}
};

template <int fw>
struct SIMD<fw, l> {
	static inline SIMD_type hom(SIMD_type r) {return simd_andc(r, simd<fw>::himask());}
	static inline SIMD_type l2x(SIMD_type r) {return r;}
};

//template <int fw>
//struct SIMD<fw, h> {
//	static inline SIMD_type hom(SIMD_type r) {return simd<fw>::srli<fw/2>(r);}
//	static inline SIMD_type l2x(SIMD_type r) {return simd<fw>::srli<fw/2>(r);}
//};
//
template <>
struct SIMD<2, h> {
	static inline SIMD_type hom(SIMD_type r) {return simd<2>::srli<1>(r);}
	static inline SIMD_type l2x(SIMD_type r) {return simd<2>::srli<1>(r);}
};

template <>
struct SIMD<4, h> {
	static inline SIMD_type hom(SIMD_type r) {return simd<4>::srli<2>(r);}
	static inline SIMD_type l2x(SIMD_type r) {return simd<4>::srli<2>(r);}
};

template <>
struct SIMD<8, h> {
	static inline SIMD_type hom(SIMD_type r) {return simd<8>::srli<4>(r);}
	static inline SIMD_type l2x(SIMD_type r) {return simd<8>::srli<4>(r);}
};

template <>
struct SIMD<16, h> {
	static inline SIMD_type hom(SIMD_type r) {return simd<16>::srli<8>(r);}
	static inline SIMD_type l2x(SIMD_type r) {return simd<16>::srli<8>(r);}
};

template <>
struct SIMD<32, h> {
	static inline SIMD_type hom(SIMD_type r) {return simd<32>::srli<16>(r);}
	static inline SIMD_type l2x(SIMD_type r) {return simd<32>::srli<16>(r);}
};


/* SIMD operations extended with HOM*/
template<int fw> template <HOM_t m1, HOM_t m2>
inline SIMD_type simd<fw>::add(SIMD_type r1, SIMD_type r2){
	return simd<fw>::add(SIMD<fw,m1>::hom(r1),SIMD<fw,m2>::hom(r2));
}

template<int fw> template <HOM_t m1, HOM_t m2>
inline SIMD_type simd<fw>::sub(SIMD_type r1, SIMD_type r2){
	return simd<fw>::sub(SIMD<fw,m1>::hom(r1),SIMD<fw,m2>::hom(r2));
}

template<int fw> template <HOM_t m1, HOM_t m2>
inline SIMD_type simd<fw>::pack(SIMD_type r1, SIMD_type r2){
	return simd<fw>::pack(SIMD<fw,m1>::l2x(r1),SIMD<fw,m2>::l2x(r2));
}

template<int fw> template <HOM_t m1, HOM_t m2>
inline SIMD_type simd<fw>::mergeh(SIMD_type r1, SIMD_type r2){
	return simd<fw>::mergeh(SIMD<fw,m1>::hom(r1),SIMD<fw,m2>::hom(r2));
}

template<int fw> template <HOM_t m1, HOM_t m2>
inline SIMD_type simd<fw>::mergel(SIMD_type r1, SIMD_type r2){
	return simd<fw>::mergel(SIMD<fw,m1>::hom(r1),SIMD<fw,m2>::hom(r2));
}


//
//template <HOM_t m>
//struct HOM {
//template<int fw> SIMD_type hom(SIMD_type r) {return r;}
//template<int fw> SIMD_type l2x(SIMD_type r) {return r;}
//};
//
//template <> 
//template <int fw>
//SIMD_type HOM<l>::hom(SIMD_type r) {return simd_andc(r, simd<fw>::himask());}
//
//template <> 
//template <int fw>
//SIMD_type HOM<h>::hom(SIMD_type r) {return simd<fw>::srli<fw/2>(r);}
//
//template <> 
//template <int fw>
//SIMD_type HOM<h>::l2x(SIMD_type r) {return simd<fw>::srli<fw/2>(r);}
//
//
///* SIMD operations extended with Half-Operand Modifiers */
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::add(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::add(HOM<m1>::hom<fw>, HOM<m2>::hom<fw>(r2));
//}
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::sub(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::sub(HOM<m1>::hom<fw>, HOM<m2>::hom<fw>(r2));
//}
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::mult(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::mult(HOM<m1>::hom<fw>, HOM<m2>::hom<fw>(r2));
//}
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::pack(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::pack(HOM<m1>::l2x<fw>, HOM<m2>::hom<fw>::hom(r2));
//}
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::mergeh(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::mergeh(HOM<m1>::hom<fw>, HOM<m2>::hom<fw>::hom(r2));
//}
//
//template<int fw> template <HOM_t m1, HOM_t m2>
//inline SIMD_type simd<fw>::mergel(SIMD_type r1, SIMD_type r2){
//	return simd<fw>::mergel(HOM<m1>::hom<fw>, HOM<m2>::hom<fw>::hom(r2));
//} 
 
/* V.  sisd operations on full 128-bit register width. */

//struct sisd {
//	template <int shft> inline SIMD_type slli(SIMD_type r) {return simd<128>::slli<shft>(r);}
//	template <int shft> inline SIMD_type srli(SIMD_type r) {return simd<128>::srli<shft>(r);}
//	inline SIMD_type sll(SIMD_type r, SIMD_type shft) {return simd<128>::sll<shft>(r, shft);}
//	inline SIMD_type srl(SIMD_type r, SIMD_type shft) {return simd<128>::srl<shft>(r, shft);}
//};


#define sisd_store_aligned(r, addr) _mm_store_si128(addr, r)
#define sisd_store_unaligned(r, addr) _mm_storeu_si128(addr, r)
#define sisd_load_aligned(addr) _mm_load_si128(addr)
#ifndef USE_LDDQU
#define sisd_load_unaligned(addr) _mm_loadu_si128(addr)
#endif
#ifdef USE_LDDQU
#define sisd_load_unaligned(addr) _mm_lddqu_si128(addr)
#endif


#define bitblock_test_bit(blk, n) \
   sisd_to_int(sisd_srli(sisd_slli(blk, ((BLOCKSIZE-1)-(n))), BLOCKSIZE-1))


#if (BYTE_ORDER == BIG_ENDIAN)
void print_bit_block(char * var_name, SIMD_type v) {
  union {SIMD_type vec; unsigned char elems[8];} x;
  x.vec = v;
  unsigned char c, bit_reversed;
  int i;
  printf("%20s = ", var_name);
  for (i = 0; i < sizeof(SIMD_type); i++) {
    c = x.elems[i];
     printf("%02X ", c); 
  }
  printf("\n");
}
#endif

#if (BYTE_ORDER == LITTLE_ENDIAN)
void print_bit_block(char * var_name, SIMD_type v) {
  union {SIMD_type vec; unsigned char elems[8];} x;
  x.vec = v;
  unsigned char c, bit_reversed;
  int i;
  printf("%20s = ", var_name);
  for (i = sizeof(SIMD_type)-1; i >= 0; i--) {
    c = x.elems[i];
     printf("%02X ", c); 
  }
  printf("\n");
}
#endif


static inline int bitblock_has_bit(SIMD_type v) {
  return !simd_all_true<8>(simd<8>::eq(v, simd<8>::constant<0>()));
}

static inline int bitblock_bit_count(SIMD_type v) {
  int bit_count = 0;
  SIMD_type cts_2 = simd<2>::add<l,h>(v, v);
  SIMD_type cts_4 = simd<4>::add<l,h>(cts_2, cts_2);
  SIMD_type cts_8 = simd<8>::add<l,h>(cts_4, cts_4);
  SIMD_type cts_64 = _mm_sad_epu8(cts_8, simd<8>::constant<0>());
  /* SIMD_type cts_128 = simd<a28>::add<l,h>(cts_64, cts_64) */;
  SIMD_type cts_128 = simd<64>::add(cts_64, simd<128>::srli<64>(cts_64));
  return (int) sisd_to_int(cts_128);
}
#endif