/xmlbench/trunk

/*  byteplex.c - Parallel byte stream module.
    Copyright (c) 2008, Robert D. Cameron.
    Licensed to the public under the Open Software License 3.0.
    Licensed to International Characters, Inc., under the Academic
    Free License 3.0.

*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "byteplex.h"
#include "xml_error.h"
#include "multiliteral.h"
#include "bytelex.h"



/*  Space for sentinels in bytescans of the pseudo-ASCII stream. */
const int SENTINEL_PACKS = 1;

Byteplex::~Byteplex() {
	if (infile != NULL) fclose(infile);
}


template <CodeUnit_Base C>
X8_Buffer<C>::X8_Buffer() : Byteplex() {
	/* For 8-bit code units, the input buffer is a also used directly
	   as the pseudo-ASCII buffer; make sure that there is room for
	   sentinels. */
	src_buffer = simd_new(BYTEPLEX_SIZE/PACKSIZE+SENTINEL_PACKS);
	x8data = src_buffer;
	// Set the sentinel for ScanToQuote,ScanWS in reading XML/text decls.
	((unsigned char *) x8data)[BYTEPLEX_SIZE] = Ord<C, '"'>::value;
}


template <CodeUnit_Base C>
X8_Buffer<C>::~X8_Buffer() {
  simd_delete((SIMD_type *) src_buffer);
}

U16_Buffer::U16_Buffer()
	: Byteplex() {

	src_buffer = simd_new((BYTEPLEX_SIZE/PACKSIZE)*2);
	x16hi = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x16lo = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x8data = simd_new(BYTEPLEX_SIZE/PACKSIZE+SENTINEL_PACKS);
	// Set the sentinel for ScanToQuote,ScanWS in reading XML/text decls.
	((unsigned char *) x8data)[BYTEPLEX_SIZE] = '"';
}

U16_Buffer::~U16_Buffer() {
  simd_delete((SIMD_type *) x16hi);
  simd_delete((SIMD_type *) x16lo);
  simd_delete((SIMD_type *) x8data);
  simd_delete((SIMD_type *) src_buffer);
}

U16LE_Buffer::U16LE_Buffer()
	: U16_Buffer() {
}

U16BE_Buffer::U16BE_Buffer()
	: U16_Buffer() {
}

U32_Buffer::U32_Buffer()
	: Byteplex() {

	src_buffer = simd_new((BYTEPLEX_SIZE/PACKSIZE)*4);
	x32hh = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x32hl = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x32lh = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x32ll = simd_new(BYTEPLEX_SIZE/PACKSIZE);
	x8data = simd_new(BYTEPLEX_SIZE/PACKSIZE+SENTINEL_PACKS);
	x8data = simd_new(BYTEPLEX_SIZE/PACKSIZE+SENTINEL_PACKS);
	// Set the sentinel for ScanToQuote,ScanWS in reading XML/text decls.
	((unsigned char *) x8data)[BYTEPLEX_SIZE] = '"';
}

U32_Buffer::~U32_Buffer() {
  simd_delete((SIMD_type *) x32hh);
  simd_delete((SIMD_type *) x32hl);
  simd_delete((SIMD_type *) x32lh);
  simd_delete((SIMD_type *) x32ll);
  simd_delete((SIMD_type *) x8data);
  simd_delete((SIMD_type *) src_buffer);
}

U32LE_Buffer::U32LE_Buffer()
	: U32_Buffer() {
}

U32BE_Buffer::U32BE_Buffer()
	: U32_Buffer() {
}

U32_2143_Buffer::U32_2143_Buffer()
	: U32_Buffer() {
}

U32_3412_Buffer::U32_3412_Buffer()
	: U32_Buffer() {
}


/* Byteplex methods.

   No byteplexing is required for 8-bit code units; byteplex methods are no-ops.

*/

template <CodeUnit_Base C>
void X8_Buffer<C>::DoByteplex() {
	x8data = src_buffer;
}


void DoDuplex(BytePack * src_data, int packs_in_buffer,
				 	 BytePack * p0, BytePack * p1) {

	for (int pk = 0; pk < packs_in_buffer; pk++) {
		BytePack s0 = src_data[2*pk];
		BytePack s1 = src_data[2*pk+1];
#if (BYTE_ORDER == LITTLE_ENDIAN)
#ifdef TEMPLATED_SIMD_LIB
		p0[pk] = simd<16>::pack<l,l>(s1, s0);
		p1[pk] = simd<16>::pack<h,h>(s1, s0);
#endif
#ifndef TEMPLATED_SIMD_LIB
		p0[pk] = simd_pack_16_ll(s1, s0);
		p1[pk] = simd_pack_16_hh(s1, s0);
#endif
#endif
#if (BYTE_ORDER == BIG_ENDIAN)
#ifdef TEMPLATED_SIMD_LIB
		p0[pk] = simd<16>::pack<l,l>(s0, s1);
		p1[pk] = simd<16>::pack<h,h>(s0, s1);
#endif
#ifndef TEMPLATED_SIMD_LIB
		p0[pk] = simd_pack_16_ll(s0, s1);
		p1[pk] = simd_pack_16_hh(s0, s1);
#endif
#endif
	}
}
					 
void U16LE_Buffer::DoByteplex() {
	DoDuplex(src_buffer, packs_in_buffer, x16lo, x16hi);
}

void U16BE_Buffer::DoByteplex() {
	DoDuplex(src_buffer, packs_in_buffer, x16hi, x16lo);
}

void DoQuadplex(BytePack * src_data, int packs_in_buffer,
				BytePack * p0, BytePack * p1, BytePack * p2, BytePack * p3) {

	for (int pk = 0; pk < packs_in_buffer; pk++) {
		BytePack s0 = src_data[4*pk];
		BytePack s1 = src_data[4*pk+1];
		BytePack s2 = src_data[4*pk+2];
		BytePack s3 = src_data[4*pk+3];
#if (BYTE_ORDER == LITTLE_ENDIAN)
#ifdef TEMPLATED_SIMD_LIB
		BytePack p02_0 = simd<16>::pack<l,l>(s1, s0);
		BytePack p13_0 = simd<16>::pack<h,h>(s1, s0);
		BytePack p02_1 = simd<16>::pack<l,l>(s3, s2);
		BytePack p13_1 = simd<16>::pack<h,h>(s3, s2);
		p0[pk] = simd<16>::pack<l,l>(p02_1, p02_0);
		p1[pk] = simd<16>::pack<l,l>(p13_1, p13_0);
		p2[pk] = simd<16>::pack<h,h>(p02_1, p02_0);
		p3[pk] = simd<16>::pack<h,h>(p13_1, p13_0);
#endif
#ifndef TEMPLATED_SIMD_LIB
		BytePack p02_0 = simd_pack_16_ll(s1, s0);
		BytePack p13_0 = simd_pack_16_hh(s1, s0);
		BytePack p02_1 = simd_pack_16_ll(s3, s2);
		BytePack p13_1 = simd_pack_16_hh(s3, s2);
		p0[pk] = simd_pack_16_ll(p02_1, p02_0);
		p1[pk] = simd_pack_16_ll(p13_1, p13_0);
		p2[pk] = simd_pack_16_hh(p02_1, p02_0);
		p3[pk] = simd_pack_16_hh(p13_1, p13_0);
#endif
#endif
#if (BYTE_ORDER == BIG_ENDIAN)
#ifdef TEMPLATED_SIMD_LIB
		BytePack p02_0 = simd<16>::pack<h,h>(s0, s1);
		BytePack p13_0 = simd<16>::pack<l,l>(s0, s1);
		BytePack p02_1 = simd<16>::pack<h,h>(s2, s3);
		BytePack p13_1 = simd<16>::pack<l,l>(s2, s3);
		p0[pk] = simd<16>::pack<h,h>(p02_0, p02_1);
		p1[pk] = simd<16>::pack<h,h>(p13_0, p13_1);
		p2[pk] = simd<16>::pack<l,l>(p02_0, p02_1);
		p3[pk] = simd<16>::pack<l,l>(p13_0, p13_1);
#endif
#ifndef TEMPLATED_SIMD_LIB
		BytePack p02_0 = simd_pack_16_hh(s0, s1);
		BytePack p13_0 = simd_pack_16_ll(s0, s1);
		BytePack p02_1 = simd_pack_16_hh(s2, s3);
		BytePack p13_1 = simd_pack_16_ll(s2, s3);
		p0[pk] = simd_pack_16_hh(p02_0, p02_1);
		p1[pk] = simd_pack_16_hh(p13_0, p13_1);
		p2[pk] = simd_pack_16_ll(p02_0, p02_1);
		p3[pk] = simd_pack_16_ll(p13_0, p13_1);
#endif
#endif
	}
}

void U32LE_Buffer::DoByteplex() {
	DoQuadplex(src_buffer, packs_in_buffer, x32ll, x32lh, x32hl, x32hh);
}

void U32BE_Buffer::DoByteplex() {
	DoQuadplex(src_buffer, packs_in_buffer, x32hh, x32hl, x32lh, x32ll);
}

void U32_2143_Buffer::DoByteplex() {
	DoQuadplex(src_buffer, packs_in_buffer, x32hl, x32hh, x32ll, x32lh);
}

void U32_3412_Buffer::DoByteplex() {
	DoQuadplex(src_buffer, packs_in_buffer, x32lh, x32ll, x32hh, x32hl);
}


/* Pseudo-ASCII stream methods */

template <CodeUnit_Base C>
void X8_Buffer<C>::PreparePseudoASCII_Stream() {
	x8data = src_buffer;
}

void U16_Buffer::PreparePseudoASCII_Stream() {
	for (int pk = 0; pk < packs_in_buffer; pk++) {
#ifdef TEMPLATED_SIMD_LIB
		x8data[pk] = simd_or(x16lo[pk], simd_andc(simd<8>::constant<(0x80)>(), 
					       simd<8>::eq(x16hi[pk], simd<8>::constant<0>())));
#endif
#ifndef TEMPLATED_SIMD_LIB
		x8data[pk] = simd_or(x16lo[pk], simd_andc(simd_const_8(0x80), 
					       simd_eq_8(x16hi[pk], simd_const_8(0))));
#endif
	}
}

void U32_Buffer::PreparePseudoASCII_Stream() {
	for (int pk = 0; pk < packs_in_buffer; pk++) {
		BytePack hi = simd_or(simd_or(x32hh[pk], x32hl[pk]), x32lh[pk]);
#ifdef TEMPLATED_SIMD_LIB
		x8data[pk] = simd_or(x32ll[pk], simd_andc(simd<8>::constant<(0x80)>(), 
					       simd<8>::eq(hi, simd<8>::constant<0>())));
#endif
#ifndef TEMPLATED_SIMD_LIB
		x8data[pk] = simd_or(x32ll[pk], simd_andc(simd_const_8(0x80), 
					       simd_eq_8(hi, simd_const_8(0))));
#endif
	}
}


int Byteplex::CopyAndFill(unsigned char * bytes_to_copy, int lgth, int bytes_to_read) {
	memcpy(src_buffer, bytes_to_copy, lgth);
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == FILE_READING)
	code_clocker->start_interval();
#endif
	unsigned char * end_ptr = &((unsigned char *)src_buffer)[lgth];
	int bytes_read = fread(end_ptr, 1, bytes_to_read, infile);
	if (bytes_read < bytes_to_read) end_ptr[bytes_read] = '\0'; /* sentinel */
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == FILE_READING)
	code_clocker->end_interval(bytes_read);
#endif
	return bytes_read;
}

void Byteplex::Set_limits(int units) {
	units_in_buffer = units;
	packs_in_buffer = (units_in_buffer + PACKSIZE -1)/PACKSIZE;
	//buffer_limit_pos = min(units_in_buffer, BUFFER_SIZE);
}

template <CodeUnit_Base C>
void X8_Buffer<C>::InitializeBuffer(unsigned char * src, int lgth){	
	int byte_advance = BYTEPLEX_SIZE - lgth;	
	int bytes_read = CopyAndFill(src, lgth, byte_advance);
	Set_limits(bytes_read + lgth);
}

void U16_Buffer::InitializeBuffer(unsigned char * src, int lgth){	
	int byte_advance = BYTEPLEX_SIZE * 2 - lgth;	
	int bytes_read = CopyAndFill(src, lgth, byte_advance);
	if (bytes_read % 2 != 0) {
		IncompleteCodeUnitError();
	}
	Set_limits((bytes_read + lgth)/2);
}

void U32_Buffer::InitializeBuffer(unsigned char * src, int lgth){	
	int byte_advance = BYTEPLEX_SIZE * 4 - lgth;	
	int bytes_read = CopyAndFill(src, lgth, byte_advance);
	if (bytes_read % 4 != 0) {
		IncompleteCodeUnitError();
	}
	Set_limits((bytes_read + lgth)/4);
}



template <CodeUnit_Base C>
void X8_Buffer<C>::AdvanceInputBuffer(int advance_amt){	
	int bytes_to_keep = units_in_buffer - advance_amt;	
	int bytes_read = CopyAndFill(&((unsigned char *)src_buffer)[advance_amt],
					bytes_to_keep, advance_amt);
	Set_limits(bytes_read + bytes_to_keep);
}

void U16_Buffer::AdvanceInputBuffer(int advance_amt){	
	int bytes_to_keep = (units_in_buffer - advance_amt)*2;	
	int bytes_read = CopyAndFill(&((unsigned char *)src_buffer)[advance_amt*2],
					bytes_to_keep, advance_amt*2);
	if (bytes_read % 2 != 0) {
		IncompleteCodeUnitError();
	}
	Set_limits((bytes_read + bytes_to_keep)/2);
}

void U32_Buffer::AdvanceInputBuffer(int advance_amt){	
	int bytes_to_keep = (units_in_buffer - advance_amt)*4;	
	int bytes_read = CopyAndFill(&((unsigned char *)src_buffer)[advance_amt*4],
					bytes_to_keep, advance_amt*4);
	if (bytes_read % 4 != 0) {
		IncompleteCodeUnitError();
	}
	Set_limits((bytes_read + bytes_to_keep)/4);
}

void U16_Buffer::Validate_UTF16() {
	BytePack surrogate_select;
	BytePack hi_surrogate;
	BytePack lo_surrogate;
#ifdef TEMPLATED_SIMD_LIB
	BytePack hi_surrogate_pending = simd<8>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
	BytePack hi_surrogate_pending = simd_const_8(0);
#endif
	BytePack surrogate_scope;
	BytePack u16_surrogate_error;
//	BytePack u16_surrogate_accum = simd<8>::constant<0>();
//	BytePack u16_FFFE_FFFF_accum = simd<8>::constant<0>();
	BytePack u16_FFFE_FFFF;
	for (int pk = 0; pk < packs_in_buffer; pk++) {
		/* UTF-16 code units in the range D800-DBFF and DC00-DFFF are
		   reserved for the first and second elements, respectively
		   of surrogate pairs.  Validation requires that these values
		   only occur in well-formed pairs. */
#ifdef TEMPLATED_SIMD_LIB
		surrogate_select = simd_and(x16hi[pk], simd<8>::constant<0xDC>());
		hi_surrogate = simd<8>::eq(surrogate_select, simd<8>::constant<0xD8>());
		lo_surrogate = simd<8>::eq(surrogate_select, simd<8>::constant<0xDC>());
		surrogate_scope = simd_or(hi_surrogate_pending,
					  sisd_sfli(hi_surrogate, 8));
					  
		u16_surrogate_error = simd_xor(surrogate_scope, lo_surrogate);
		hi_surrogate_pending = sisd_sbli(hi_surrogate, 8 * (PACKSIZE-1));
		/* The values FFFE and FFFF are excluded. */
		u16_FFFE_FFFF = simd<8>::eq(simd_and(x16hi[pk],
						   simd_or(x16lo[pk], simd<8>::constant<1>())),
					  simd<8>::constant<0xFF>());
#endif
#ifndef TEMPLATED_SIMD_LIB
		surrogate_select = simd_and(x16hi[pk], simd_const_8(0xDC));
		hi_surrogate = simd_eq_8(surrogate_select, simd_const_8(0xD8));
		lo_surrogate = simd_eq_8(surrogate_select, simd_const_8(0xDC));
		surrogate_scope = simd_or(hi_surrogate_pending,
					  sisd_sfli(hi_surrogate, 8));
					  
		u16_surrogate_error = simd_xor(surrogate_scope, lo_surrogate);
		hi_surrogate_pending = sisd_sbli(hi_surrogate, 8 * (PACKSIZE-1));
		/* The values FFFE and FFFF are excluded. */
		u16_FFFE_FFFF = simd_eq_8(simd_and(x16hi[pk],
						   simd_or(x16lo[pk], simd_const_8(1))),
					  simd_const_8(0xFF));
#endif
//		u16_FFFE_FFFF_accum = simd_or(u16_FFFE_FFFF_accum, u16_FFFE_FFFF);
		u16_surrogate_error = simd_or(u16_surrogate_error, u16_FFFE_FFFF);
	
		if (bitblock_has_bit(u16_surrogate_error)) {
			CharSetValidationError("UTF-16 (relative position reported)",
						pk * PACKSIZE + count_forward_zeroes(u16_surrogate_error)/8);
		}
	}
};


void U16_Buffer::Validate_UCS2() {
#ifdef X16HILO_ACCESS
	int packs = (buffer_units - 1)/PACKSIZE + 1;
#ifdef TEMPLATED_SIMD_LIB
	BytePack u16_surrogate_accum = simd<8>::constant<0>();
	BytePack u16_FFFE_FFFF_accum = simd<8>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
	BytePack u16_surrogate_accum = simd_const_8(0);
	BytePack u16_FFFE_FFFF_accum = simd_const_8(0);
#endif
	BytePack u16_FFFE_FFFF;
	for (int pk = 0; pk < packs; pk++) {
		/* The high byte of UCS-2 code units cannot be in the range D8-DF.
		   This corresponds to the D800-DFFF range of illegal codepoints
		   reserved for UTF-16 surrogate pairs. Accumulate the results. 
		   To check, 0x20 is added to each such octet, mapping the D8-DF
                   range to F8-FF and wrapping E0-FF values around.  The max value
                   is then accumulated.  */  
#ifdef TEMPLATED_SIMD_LIB 
		u16_surrogate_accum =
			simd_max_8(u16_surrogate_accum, simd<8>::add(x16hi[pk], simd<8>::constant<0x20>()));
		/* The values FFFE and FFFF are excluded. */
		u16_FFFE_FFFF = simd<8>::eq(simd_and(x16hi[pk],
						   simd_or(x16lo[pk], simd<8>::constant<1>())), simd<8>::constant<0xFF>());
		u16_FFFE_FFFF_accum = simd_or(u16_FFFE_FFFF_accum, u16_FFFE_FFFF);
#endif
#ifndef TEMPLATED_SIMD_LIB
 		u16_surrogate_accum =
			simd_max_8(u16_surrogate_accum, simd_add_8(x16hi[pk], simd_const_8(0x20)));
		/* The values FFFE and FFFF are excluded. */
		u16_FFFE_FFFF = simd_eq_8(simd_and(x16hi[pk],
						   simd_or(x16lo[pk], simd_const_8(1))), simd_const_8(0xFF));
		u16_FFFE_FFFF_accum = simd_or(u16_FFFE_FFFF_accum, u16_FFFE_FFFF);
#endif
	}
#ifdef TEMPLATED_SIMD_LIB 
	u16_surrogate_accum = simd<8>::eq(simd_or(u16_surrogate_accum, simd<8>::constant<0x07>()),
					simd<8>::constant<0xFF>());
#endif
#ifndef TEMPLATED_SIMD_LIB
	u16_surrogate_accum = simd_eq_8(simd_or(u16_surrogate_accum, simd_const_8(0x07)),
					simd_const_8(0xFF));
#endif

	if (bitblock_has_bit(simd_or(u16_surrogate_accum, u16_FFFE_FFFF_accum)))
		CharSetValidationError("UCS-2");
	}
#endif
#ifndef X16HILO_ACCESS
	printf("UCS_2_Lexer::Do_CharsetValidation not yet complete; assuming OK.\n");
#endif
};


void U32_Buffer::Validate_UTF32() {
#ifdef X32BYTEPLEX_ACCESS
	int packs = (buffer_units - 1)/PACKSIZE + 1;
#ifdef TEMPLATED_SIMD_LIB
	BytePack u32hh_accum = simd<8>::constant<0>();
	BytePack u32hl_accum = simd<8>::constant<0>();
	BytePack u32_surrogate_accum = simd<8>::constant<0>();
	BytePack u32_FFFE_FFFF_accum = simd<8>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
	BytePack u32hh_accum = simd_const_8(0);
	BytePack u32hl_accum = simd_const_8(0);
	BytePack u32_surrogate_accum = simd_const_8(0);
	BytePack u32_FFFE_FFFF_accum = simd_const_8(0);
#endif
	BytePack u32_BMP_select;
	BytePack u32l_FFFE_FFFF;
	for (int pk = 0; pk < packs; pk++) {
		/* There can be no bits set in the high octet; "or" together
		   all octet values to check for any bit set. */
		u32hh_accum = simd_or(u32hh_accum, x32hh[pk]);
		/* The second octet has a max value of 0x10, corresponding to the
		   maximum Unicode code point value of 0x10FFFF.  Accumulate the
		   maximum of all u32hl values observed. */ 
		u32hl_accum = simd_max_8(u32hl_accum, x32hl[pk]);
		/* The third octet cannot be in the range D8-DF if the second octet
		   is 0.  This corresponds to the D800-DFFF range of illegal codepoints
		   reserved for UTF-16 surrogate pairs. Accumulate the results. 
		   To check, 0x20 is added to each such octet, mapping the D8-DF
                   range to F8-FF and wrapping E0-FF values around.  The max value
                   is then accumulated.  */
#ifdef TEMPLATED_SIMD_LIB
		u32_BMP_select = simd<8>::eq(x32hl[pk], simd<8>::constant<0>());
		u32_surrogate_accum = simd_max_8(u32_surrogate_accum, 
				  				 simd_and(u32_BMP_select, simd<8>::add(x32lh[pk], simd<8>::constant<0x20>())));
		/* The low two octets cannot have the value FFFE or FFFF if
		   we're in the BMP (second octet is 0). */
		u32l_FFFE_FFFF = simd<8>::eq(simd_and(x32lh[pk],
						    simd_or(x32ll[pk], simd<8>::constant<1>())),simd<8>::constant<0xFF>());
		u32_FFFE_FFFF_accum = simd_or(u32_FFFE_FFFF_accum,
					      simd_and(u32_BMP_select, u32l_FFFE_FFFF));
#endif
#ifndef TEMPLATED_SIMD_LIB
		u32_BMP_select = simd_eq_8(x32hl[pk], simd_const_8(0));
		u32_surrogate_accum = simd_max_8(u32_surrogate_accum, 
				  				 simd_and(u32_BMP_select, simd<8>::add(x32lh[pk], simd_const_8(0x20))));
		/* The low two octets cannot have the value FFFE or FFFF if
		   we're in the BMP (second octet is 0). */
		u32l_FFFE_FFFF = simd_eq_8(simd_and(x32lh[pk],
						    simd_or(x32ll[pk], simd_const_8(1))),simd_const_8(0xFF));
		u32_FFFE_FFFF_accum = simd_or(u32_FFFE_FFFF_accum,
					      simd_and(u32_BMP_select, u32l_FFFE_FFFF));
#endif
	}
#ifdef TEMPLATED_SIMD_LIB
	u32hl_accum = simd_gt_8(u32hl_accum, simd<8>::constant<0x10>());
	u32_surrogate_accum = simd<8>::eq(simd_or(u32_surrogate_accum, simd<8>::constant<0x07>()),
					simd<8>::constant<0xFF>());
#endif
#ifndef TEMPLATED_SIMD_LIB
	u32hl_accum = simd_gt_8(u32hl_accum, simd_const_8(0x10));
	u32_surrogate_accum = simd_eq_8(simd_or(u32_surrogate_accum, simd_const_8(0x07)),
					simd_const_8(0xFF));
#endif
	if (bitblock_has_bit(simd_or(simd_or(u32hh_accum, u32hl_accum),
					 simd_or(u32_surrogate_accum, u32_FFFE_FFFF_accum)))) {
		CharSetValidationError("UTF-32");
	}
#endif
#ifndef X32BYTEPLEX_ACCESS
	printf("UTF_32_Lexer::Do_CharsetValidation not yet complete; assuming OK.\n");
#endif
};

Byteplex * Byteplex::ByteplexFactory(Entity_Info * e) {
	Byteplex * b;
	if (likely(e->code_unit_size == SingleByte)) {
		if (likely(e->code_unit_base == ASCII)) 
			b = new X8_Buffer<ASCII>();
		else b = new X8_Buffer<EBCDIC>();
	}
	else if (likely(e->code_unit_size == DoubleByte)) {
		if (likely(e->byte_order == BigEndian))
			b = new U16BE_Buffer();
		else b = new U16LE_Buffer();
	}
	else switch (e->byte_order) {
		case BigEndian: b = new U32BE_Buffer(); break;
		case LittleEndian: b = new U32LE_Buffer(); break;
		case Unusual_2143: b = new U32_2143_Buffer(); break;
		case Unusual_3412: b = new U32_3412_Buffer(); break;
	}	
	return b;
}

Byteplex * Byteplex::ByteplexFactory(Entity_Info * e, FILE * inputfile) {
	Byteplex * b = ByteplexFactory(e);
	b->infile = inputfile;
	return b;
}
	
Byteplex * Byteplex::ByteplexFactory(Entity_Info * e, unsigned char * buffer_bytes, int buffer_size) {
	Byteplex * b = ByteplexFactory(e);
	b->infile = NULL;
	memcpy(b->src_buffer, buffer_bytes, buffer_size);
//printf("buffer_bytes = %s\n", buffer_bytes);
 	b->units_in_buffer = buffer_size / e->code_unit_size;
	b->packs_in_buffer = (b->units_in_buffer + PACKSIZE -1)/PACKSIZE;
	return b;
}

template <>
int X8_Buffer<EBCDIC>::UTF8_Length(int name_pos, int lgth){
	int u8_lgth = 0;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		u8_lgth += /*TEMPORARY - NEED TO USE A TABLE FOR LOOKUP*/ 2;
	}
	return u8_lgth;
}

template <>
int X8_Buffer<ASCII>::UTF8_Length(int name_pos, int lgth){
	int u8_lgth = 0;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		if (((unsigned char *)x8data)[i] < 0x80) u8_lgth += 1;
		else u8_lgth += /*TEMPORARY - NEED TO USE A TABLE FOR LOOKUP*/ 1;
	}
	return u8_lgth;
}

int UTF8_Buffer::UTF8_Length(int name_pos, int lgth){
	return lgth;
}

int U16_Buffer::UTF8_Length(int name_pos, int lgth){
	int u8_lgth = 0;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		if (((unsigned char *)x8data)[i] < 0x80) u8_lgth += 1;
		else if(((unsigned char *)x16hi)[i]<=0x7 || (((unsigned char *)x16hi)[i] >= 0xD8 &&((unsigned char *)x16hi)[i]<= 0xDF))
			u8_lgth += 2;
		else
			u8_lgth += 3;
	}
	return u8_lgth;
}

int U32_Buffer::UTF8_Length(int name_pos, int lgth){
	int u8_lgth = 0;
	unsigned char * u32hl = (unsigned char *) x32hl;
	unsigned char * u32lh = (unsigned char *) x32lh;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		if (((unsigned char *)x8data)[i] < 0x80)  {
			u8_lgth += 1;
		}
		else if(u32hl[i] > 0)
			u8_lgth += 4;
		else if(u32lh[i]<=0x7)
			u8_lgth += 2;
		else
			u8_lgth += 3;
	}
	return u8_lgth;
}

template <>
void X8_Buffer<ASCII>::to_UTF8(int name_pos, int lgth, char * u8_ptr){
	memcpy(u8_ptr, &((char *)x8data)[name_pos], lgth);
	u8_ptr[lgth] = '\0';
//	u8_ptr = copy_name(&((char *)x8data)[name_pos], lgth);
}

template <>
void X8_Buffer<EBCDIC>::to_UTF8(int name_pos, int lgth, char * u8_ptr){
	
}
void U16_Buffer::to_UTF8(int name_pos, int lgth, char * u8_ptr){
	int u8_lgth = 0;
	unsigned char * u16h = (unsigned char *) x16hi;
	unsigned char * u16l = (unsigned char *) x16lo;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		if (((unsigned char *)x8data)[i] < 0x80) {
			u8_ptr[u8_lgth] = ((unsigned char *)x8data)[i];
			u8_lgth += 1;
		}
		else if (u16h[i]<=0x7) {
			u8_ptr[u8_lgth] = 0xC0 + (u16h[i] << 2) + (u16l[i] >> 6);
			u8_ptr[u8_lgth+1] = 0x80 + (u16l[i] & 0x3F);
			u8_lgth += 2;
		}
		else if ((u16h[i] >= 0xD8) && (u16h[i]<= 0xDB)){
			char temp =  ((u16h[i] & 0x03) << 2) + (u16l[i] >> 6) + 1;
			u8_ptr[u8_lgth] = 0xF0 + (temp >> 2);
			u8_ptr[u8_lgth+1] = 0x80 + ((temp & 0x03) << 4) + ((u16l[i] & 0x3F) >> 2);
			u8_ptr[u8_lgth+2] = 0x80 + ((u16l[i] & 0x03) << 4) + ((u16h[i+1] & 0x03) << 2) + (u16l[i+1] >> 6);
			u8_ptr[u8_lgth+3] = 0x80 + (u16l[i+1] & 0x3F);
			i++;
			u8_lgth += 4;
		}
		else{
			u8_ptr[u8_lgth] = 0xE0 + (u16h[i] >> 4);
			u8_ptr[u8_lgth+1] = 0x80 + ((u16h[i] & 0x0F) << 2) + (u16l[i] >> 6);
			u8_ptr[u8_lgth+2] = 0x80 + (u16l[i] & 0x3F);
			u8_lgth += 3;
		}
	}
	u8_ptr[u8_lgth] = '\0';
}



void U32_Buffer::to_UTF8(int name_pos, int lgth, char * u8_ptr){
	int u8_lgth = 0;
	unsigned char * u32hl = (unsigned char *) x32hl;
	unsigned char * u32lh = (unsigned char *) x32lh;
	unsigned char * u32ll = (unsigned char *) x32ll;
	for (int i = name_pos; i < name_pos+lgth; i++) {
		if (((unsigned char *)x8data)[i] < 0x80)  {
			u8_ptr[u8_lgth] = ((unsigned char *)x8data)[i];
			u8_lgth += 1;
		}
		else if(u32hl[i] > 0) {
			u8_ptr[u8_lgth] = 0xF0 + (u32hl[i] >> 2);
			u8_ptr[u8_lgth+1] = 0x80 + ((u32hl[i] & 0x03) << 4) + (u32lh[i] >> 4);
			u8_ptr[u8_lgth+2] = 0x80 + ((u32lh[i] & 0x0F) << 2) + (u32ll[i] >> 6);
			u8_ptr[u8_lgth+3] = 0x80 + (u32ll[i] & 0x3F);		
			u8_lgth += 4;
		}
		else if(u32lh[i]<=0x7) {
			u8_ptr[u8_lgth] = 0xC0 + (u32lh[i] << 2) + (u32ll[i] >> 6);
			u8_ptr[u8_lgth+1] = 0x80 + (u32ll[i] & 0x3F);
			u8_lgth += 2;
		}
		else {
			u8_ptr[u8_lgth] = 0xE0 + (u32lh[i] >> 4);
			u8_ptr[u8_lgth+1] = 0x80 + ((u32lh[i] & 0x0F) << 2) + (u32ll[i] >> 6);
			u8_ptr[u8_lgth+2] = 0x80 + (u32ll[i] & 0x3F);
			u8_lgth += 3;
		}
	}
	u8_ptr[u8_lgth] = '\0';
}