/xmlbench/trunk

/*  bitlex - Lexical Item Stream Module.
    Copyright (c) 2007, 2008, Robert D. Cameron.
    Licensed to the public under the Open Software License 3.0.
    Licensed to International Characters, Inc., under the Academic
    Free License 3.0.

*/

#include "bitlex.h"
#include "../lib/lib_simd.h"
#include "xml_error.h"

#ifdef CODE_CLOCKING
/*
#include "../codeclocker/clocker/code_clocker_session.h"
Code_Clocker * transpose_clocker;
Code_Clocker * WS_Control_clocker;
Code_Clocker * MarkupStreams_clocker;
Code_Clocker * char_validation_clocker;
*/
#endif


Lexer_Interface::Lexer_Interface(Entity_Info * e, LexicalStreamSet *l) {
	entity_Info = e;
	parsing_engine_data = l;

	x8basis = (BitBlockBasis *) simd_new(BUFFER_SIZE/PACKSIZE);
	validation_stream = (BitBlock *) simd_new(BUFFER_BLOCKS+SENTINEL_BLOCKS);
#ifdef TEMPLATED_SIMD_LIB
	validation_stream[BUFFER_BLOCKS] = simd<1>::constant<1>();
#endif
#ifndef TEMPLATED_SIMD_LIB
	validation_stream[BUFFER_BLOCKS] = simd_const_1(1);
#endif
};


Lexer_Interface::~Lexer_Interface() {
	simd_delete((SIMD_type *) validation_stream);
};


template <>
Lexer_Interface * Lexer<ASCII>::LexerFactory(Entity_Info * e, LexicalStreamSet *l) {
#ifdef CODE_CLOCKING
/*
	WS_Control_clocker = register_Code_Clocker("WS", "WhiteSpace/Control\n");
	MarkupStreams_clocker = register_Code_Clocker("bitlex", "Markup streams\n");
	char_validation_clocker = register_Code_Clocker("charcheck", "Character validation\n");
*/
#endif
	if (!(e->has_encoding_decl)) {
		// Must be UTF-8 or UTF-16; UTF-16 requires a ByteOrderMark.
		if (e->code_unit_size == SingleByte) return new UTF_8_Lexer(e, l);
		else if ((e->code_unit_size == DoubleByte))
			if (e->BOM_units == 1) return new UTF_16_Lexer(e, l);
			else NoEncodingError("UTF-16 implied but no byte order found.");
		else NoEncodingError("UTF-32 without an encoding declaration.\n");
	}
	else {
		int lgth = strlen((const char *) e->encoding);
		CodeUnit_ByteOrder order = e->byte_order;
		switch (e->code_unit_size) {
		case SingleByte:
			if ((lgth == 5) && at_UTF_8(e->encoding))
				return new UTF_8_Lexer(e, l);
			else if ((lgth == 5) && at_ASCII(e->encoding))
				return new ASCII_7_Lexer(e, l);
			else if ((lgth == 6)  && at_Latin1(e->encoding))
				return new EASCII_8_Lexer(e, l);
			/* Really need a table-based lookup here */
			else EncodingError("8-bit", e->encoding, lgth);
		case DoubleByte:
			if (e->BOM_units == 1)
				if ((lgth == 6) && at_UTF_16(e->encoding))
					return new UTF_16_Lexer(e, l);
				else if ((lgth == 5) && at_UCS_2(e->encoding))
					return new UCS_2_Lexer(e, l);
				else EncodingError("16-bit", e->encoding, lgth);
			else if (order == BigEndian)
				if ((lgth == 8) && at_UTF_16BE(e->encoding))
					return new UTF_16_Lexer(e, l);
				else if ((lgth == 7) && at_UCS_2BE(e->encoding))
					return new UCS_2_Lexer(e, l);
				else EncodingError("16BE", e->encoding, lgth);
			else /*if (order == LittleEndian)*/
				if ((lgth == 8) && at_UTF_16LE(e->encoding))
					return new UTF_16_Lexer(e, l);
				else if ((lgth == 7) && at_UCS_2LE(e->encoding))
					return new UCS_2_Lexer(e, l);
				else EncodingError("16LE", e->encoding, lgth);
		case QuadByte:
			if (e->BOM_units == 1)
				if ((lgth == 6) && at_UTF_32(e->encoding))
					return new UTF_32_Lexer(e, l);
				else if ((lgth == 5) && at_UCS_4(e->encoding))
					return new UTF_32_Lexer(e, l);
				else EncodingError("32-bit", e->encoding, lgth);
			else if (order == BigEndian)
				if ((lgth == 8) && at_UTF_32BE(e->encoding))
					return new UTF_32_Lexer(e, l);
				else if ((lgth == 7) && at_UCS_4BE(e->encoding))
					return new UTF_32_Lexer(e, l);
				else EncodingError("32BE", e->encoding, lgth);
			else if (order == LittleEndian)
				if ((lgth == 8) && at_UTF_32LE(e->encoding))
					return new UTF_32_Lexer(e, l);
				else if ((lgth == 7) && at_UCS_4LE(e->encoding))
					return new UTF_32_Lexer(e, l);
				else EncodingError("32LE", e->encoding, lgth);
			else EncodingError("32-bit", e->encoding, lgth);
		}
	}
}

template <>
Lexer_Interface * Lexer<EBCDIC>::LexerFactory(Entity_Info * e, LexicalStreamSet *l) {
	if (!(e->has_encoding_decl)) {
		// Must be UTF-8 or UTF-16; UTF-16 requires a ByteOrderMark.
		NoEncodingError("EBCDIC-family inferred, but no encoding declaration present.\n");
	}
	else {
		int lgth = strlen((const char *) e->encoding);
		/* Really need a table-based lookup here */
		if ((lgth == 6) && at_EBCDIC(e->encoding))
			return new EBCDIC_Lexer(e, l);
		else EncodingError("EBCDIC family", e->encoding, lgth);
	}
}

template <CodeUnit_Base C>
Lexer<C>::Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer_Interface::Lexer_Interface(e, l) {
}

UTF_8_Lexer::UTF_8_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<ASCII>::Lexer(e, l) {
}

ASCII_7_Lexer::ASCII_7_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<ASCII>::Lexer(e, l) {
}

EASCII_8_Lexer::EASCII_8_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<ASCII>::Lexer(e, l) {
}

U16_Lexer::U16_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<ASCII>::Lexer(e, l) {
}

UTF_16_Lexer::UTF_16_Lexer(Entity_Info * e, LexicalStreamSet *l) : U16_Lexer::U16_Lexer(e, l) {
}

UCS_2_Lexer::UCS_2_Lexer(Entity_Info * e, LexicalStreamSet *l) : U16_Lexer::U16_Lexer(e, l) {
}

UTF_32_Lexer::UTF_32_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<ASCII>::Lexer(e, l) {
}

EBCDIC_Lexer::EBCDIC_Lexer(Entity_Info * e, LexicalStreamSet *l) : Lexer<EBCDIC>::Lexer(e, l) {
}

template <CodeUnit_Base C>
static inline void WS_Control_Blocks(BitBlock bit[], BitBlock& WS, BitBlock& Control);

template <>
static inline void WS_Control_Blocks<ASCII>(BitBlock bit[], BitBlock& WS, BitBlock& Control) {
	BitBlock temp1 = simd_or(bit[0], bit[1]);
	BitBlock temp2 = simd_or(temp1, bit[2]);
#ifdef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd<1>::constant<1>(), temp2);
#endif
#ifndef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd_const_1(1), temp2);
#endif
	BitBlock temp3 = simd_or(bit[2], bit[3]);
	BitBlock temp4 = simd_or(temp1, temp3);
	BitBlock temp5 = simd_and(bit[4], bit[5]);
	BitBlock temp6 = simd_andc(bit[7], bit[6]);
	BitBlock temp7 = simd_and(temp5, temp6);
	BitBlock CR = simd_andc(temp7, temp4);
	BitBlock temp8 = simd_andc(bit[4], bit[5]);
	BitBlock temp9 = simd_andc(bit[6], bit[7]);
	BitBlock temp10 = simd_and(temp8, temp9);
	BitBlock LF = simd_andc(temp10, temp4);
	BitBlock temp11 = simd_and(temp8, temp6);
	BitBlock HT = simd_andc(temp11, temp4);
	BitBlock temp12 = simd_andc(bit[2], bit[3]);
	BitBlock temp13 = simd_andc(temp12, temp1);
	BitBlock temp14 = simd_or(bit[4], bit[5]);
	BitBlock temp15 = simd_or(bit[6], bit[7]);
	BitBlock temp16 = simd_or(temp14, temp15);
	BitBlock SP = simd_andc(temp13, temp16);
	WS = simd_or(simd_or(CR, LF), simd_or(HT, SP));
}

template <>
static inline void WS_Control_Blocks<EBCDIC>(BitBlock bit[], BitBlock& WS, BitBlock& Control) {
	BitBlock temp1 = simd_or(bit[0], bit[1]);
	BitBlock temp2 = simd_or(bit[2], bit[3]);
	BitBlock temp3 = simd_or(temp1, temp2);
	BitBlock temp4 = simd_or(bit[4], bit[5]);
	BitBlock temp5 = simd_or(temp3, temp4);
	BitBlock temp6 = simd_and(bit[2], bit[3]);
	BitBlock temp7 = simd_andc(temp6, temp1);
	BitBlock temp8 = simd_andc(bit[5], bit[4]);
	BitBlock temp9 = simd_and(bit[6], bit[7]);
	BitBlock temp10 = simd_and(temp8, temp9);
	BitBlock temp11 = simd_and(temp7, temp10);
	BitBlock temp12 = simd_andc(temp5, temp11);
	BitBlock temp13 = simd_andc(bit[2], bit[3]);
	BitBlock temp14 = simd_andc(temp13, temp1);
	BitBlock temp15 = simd_and(bit[4], bit[5]);
	BitBlock temp16 = simd_and(temp14, temp15);
	BitBlock temp17 = simd_andc(bit[6], bit[7]);
	BitBlock temp18 = simd_andc(temp16, temp17);
	BitBlock temp19 = simd_andc(temp12, temp18);
	BitBlock temp20 = simd_andc(bit[3], bit[2]);
	BitBlock temp21 = simd_andc(temp20, temp1);
	BitBlock temp22 = simd_and(temp8, temp17);
	BitBlock temp23 = simd_and(temp21, temp22);
	BitBlock temp24 = simd_andc(temp19, temp23);
	BitBlock temp25 = simd_or(temp1, bit[2]);
	BitBlock temp26 = simd_or(bit[5], temp9);
	BitBlock temp27 = simd_and(bit[4], temp26);
#ifdef TEMPLATED_SIMD_LIB
	BitBlock temp28 = simd_andc(simd<1>::constant<1>(), temp4);
#endif
#ifndef TEMPLATED_SIMD_LIB
	BitBlock temp28 = simd_andc(simd_const_1(1), temp4);
#endif
	BitBlock temp29 = simd_if(bit[3], temp27, temp28);
	BitBlock temp30 = simd_andc(temp29, temp25);
	BitBlock temp31 = simd_andc(temp24, temp30);
	BitBlock temp32 = simd_andc(temp15, bit[6]);
	BitBlock temp33 = simd_and(temp7, temp32);
	BitBlock temp34 = simd_andc(temp31, temp33);
	BitBlock temp35 = simd_andc(temp17, temp4);
	BitBlock temp36 = simd_and(temp7, temp35);
	BitBlock temp37 = simd_andc(temp34, temp36);
	BitBlock temp38 = simd_and(temp8, bit[6]);
	BitBlock temp39 = simd_and(temp14, temp38);
	BitBlock temp40 = simd_andc(temp37, temp39);
	BitBlock temp41 = simd_andc(bit[4], bit[5]);
	BitBlock temp42 = simd_andc(temp41, bit[6]);
	BitBlock temp43 = simd_and(temp21, temp42);
	BitBlock temp44 = simd_andc(temp40, temp43);
	BitBlock temp45 = simd_and(temp15, temp9);
	BitBlock temp46 = simd_and(temp7, temp45);
	BitBlock temp47 = simd_andc(temp44, temp46);
	BitBlock temp48 = simd_and(temp21, temp15);
	BitBlock temp49 = simd_andc(temp47, temp48);
#ifdef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd<1>::constant<1>(), temp49);
#endif
#ifndef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd_const_1(1), temp49);
#endif
	BitBlock temp50 = simd_andc(bit[7], bit[6]);
	BitBlock temp51 = simd_and(temp15, temp50);
	BitBlock CR = simd_andc(temp51, temp3);
	BitBlock temp52 = simd_and(temp8, temp50);
	BitBlock LF = simd_and(temp14, temp52);
	BitBlock HT = simd_andc(temp52, temp3);
	BitBlock temp53 = simd_andc(bit[1], bit[0]);
	BitBlock temp54 = simd_andc(temp53, temp2);
	BitBlock temp55 = simd_or(bit[6], bit[7]);
	BitBlock temp56 = simd_or(temp4, temp55);
	BitBlock SP = simd_andc(temp54, temp56);
	WS = simd_or(simd_or(CR, LF), simd_or(HT, SP));
}



template <CodeUnit_Base C>
void Lexer<C>::Do_XML_10_WS_Control() {
	BitBlock Control, WS;
	for (int i = 0; i < buffer_blocks; i++) {
		WS_Control_Blocks<C>(x8basis[i].bit,
				     WS,
                                     Control);
		parsing_engine_data->item_stream[NonWS][i] = simd_not(WS);
		validation_stream[i] = simd_andc(Control, WS);
	}
};



template <CodeUnit_Base C>
static inline void ComputeLexicalItemBlocks(BitBlock bit[], BitBlock LexItem[]);

/* Given the bit[] array of one BitBlock each for the 8 bits of
   an ASCII-family character representation, compute the parallel
   lexical item streams needed for XML parsing.

   WARNING: the following is generated code by charset_compiler.py.
   Do not edit.

*/

template <>
static inline void ComputeLexicalItemBlocks<ASCII>(BitBlock bit[], BitBlock LexItem[]) {
  BitBlock temp1 = simd_or(bit[0], bit[1]);
  BitBlock temp2 = simd_and(bit[2], bit[3]);
  BitBlock temp3 = simd_andc(temp2, temp1);
  BitBlock temp4 = simd_and(bit[4], bit[5]);
  BitBlock temp5 = simd_or(bit[6], bit[7]);
  BitBlock temp6 = simd_andc(temp4, temp5);
  BitBlock temp7 = simd_and(temp3, temp6);
  BitBlock temp8 = simd_andc(bit[2], bit[3]);
  BitBlock temp9 = simd_andc(temp8, temp1);
  BitBlock temp10 = simd_andc(bit[5], bit[4]);
  BitBlock temp11 = simd_andc(bit[6], bit[7]);
  BitBlock temp12 = simd_and(temp10, temp11);
  BitBlock temp13 = simd_and(temp9, temp12);
  LexItem[MarkupStart] = simd_or(temp7, temp13);
  BitBlock temp14 = simd_and(temp4, temp11);
  BitBlock RAngle = simd_and(temp3, temp14);
  BitBlock temp15 = simd_andc(bit[1], bit[0]);
  BitBlock temp16 = simd_andc(bit[3], bit[2]);
  BitBlock temp17 = simd_and(temp15, temp16);
  BitBlock temp18 = simd_andc(bit[7], bit[6]);
  BitBlock temp19 = simd_and(temp4, temp18);
  BitBlock RBracket = simd_and(temp17, temp19);
  LexItem[Hyphen] = simd_and(temp9, temp19);
  BitBlock temp20 = simd_and(bit[6], bit[7]);
  BitBlock temp21 = simd_and(temp4, temp20);
  LexItem[QMark] = simd_and(temp3, temp21);
  BitBlock temp22 = simd_or(bit[4], bit[5]);
  BitBlock temp23 = simd_andc(temp11, temp22);
  BitBlock temp24 = simd_and(temp10, temp20);
  BitBlock temp25 = simd_or(temp23, temp24);
  BitBlock temp26 = simd_and(temp9, temp25);
  BitBlock temp27 = simd_or(temp26, temp7);
  LexItem[Quote] = simd_or(temp27, temp13);
  BitBlock temp28 = simd_and(bit[5], temp5);
  BitBlock temp29 = simd_and(bit[4], temp28);
  BitBlock temp30 = simd_andc(temp29, temp21);
  BitBlock temp31 = simd_andc(temp9, temp30);
  BitBlock temp32 = simd_and(temp3, bit[4]);
  BitBlock temp33 = simd_or(bit[5], temp20);
  BitBlock temp34 = simd_and(temp32, temp33);
  BitBlock temp35 = simd_or(temp31, temp34);
  BitBlock temp36 = simd_and(temp17, bit[4]);
#ifdef TEMPLATED_SIMD_LIB
  BitBlock temp37 = simd_andc(simd<1>::constant<1>(), temp20);
#endif
#ifndef TEMPLATED_SIMD_LIB
  BitBlock temp37 = simd_andc(simd_const_1(1), temp20);
#endif
  BitBlock temp38 = simd_if(bit[5], temp37, temp20);
  BitBlock temp39 = simd_and(temp36, temp38);
  BitBlock temp40 = simd_or(temp35, temp39);
  BitBlock temp41 = simd_and(temp15, temp2);
  BitBlock temp42 = simd_and(temp41, bit[4]);
  BitBlock temp43 = simd_and(temp42, temp38);
  LexItem[NameFollow] = simd_or(temp40, temp43);
#ifdef DIGIT_AND_HEX_ITEMS
  BitBlock temp44 = simd_or(bit[5], bit[6]);
  BitBlock temp45 = simd_and(bit[4], temp44);
  BitBlock Digit = simd_andc(temp3, temp45);
  BitBlock temp46 = simd_or(bit[2], bit[3]);
  BitBlock temp47 = simd_andc(temp15, temp46);
  BitBlock temp48 = simd_andc(temp47, bit[4]);
  BitBlock temp49 = simd_if(bit[5], temp37, temp5);
  BitBlock temp50 = simd_and(temp48, temp49);
  BitBlock temp51 = simd_or(Digit, temp50);
  BitBlock temp52 = simd_and(temp15, temp8);
  BitBlock temp53 = simd_andc(temp52, bit[4]);
  BitBlock temp54 = simd_and(temp53, temp49);
  BitBlock Hex = simd_or(temp51, temp54);
  LexItem[NonDigit] = simd_not(Digit);
  LexItem[NonHex] = simd_not(Hex);
#endif
#ifdef MARKUP_SORTING
  BitBlock temp55 = simd_andc(temp20, temp22);
  BitBlock temp56 = simd_or(temp12, temp55);
  BitBlock temp57 = simd_or(temp56, temp21);
  LexItem[AmpHashSlash] = simd_and(temp9, temp57);
#endif

	/* Mark potential occurrences of ']]>'  These are all actual
	occurrences of ]]> as well as occurrences of ]] or ] at
	the block end. Shifting the RBracket and RAngle streams in
	negated forms ensures that a potential CD_End is not ruled
	out at the block boundary. */
	LexItem[CD_End_check] = simd_andc(RBracket,
					  simd_or(sisd_sbli(simd_not(RBracket), 1),
						  sisd_sbli(simd_not(RAngle), 2)));
#ifndef OMIT_CD_End_check_In_Markup_Scan
	LexItem[MarkupStart] = simd_or(LexItem[MarkupStart], LexItem[CD_End_check]);
#endif
}

template <>
static inline void ComputeLexicalItemBlocks<EBCDIC>(BitBlock bit[], BitBlock LexItem[]) {
	  BitBlock temp1 = simd_andc(bit[1], bit[0]);
  BitBlock temp2 = simd_or(bit[2], bit[3]);
  BitBlock temp3 = simd_andc(temp1, temp2);
  BitBlock temp4 = simd_and(bit[4], bit[5]);
  BitBlock temp5 = simd_or(bit[6], bit[7]);
  BitBlock temp6 = simd_andc(temp4, temp5);
  BitBlock temp7 = simd_and(temp3, temp6);
  BitBlock temp8 = simd_andc(bit[3], bit[2]);
  BitBlock temp9 = simd_and(temp1, temp8);
  BitBlock temp10 = simd_or(bit[4], bit[5]);
  BitBlock temp11 = simd_or(temp10, temp5);
  BitBlock temp12 = simd_andc(temp9, temp11);
  LexItem[MarkupStart] = simd_or(temp7, temp12);
  BitBlock temp13 = simd_andc(bit[2], bit[3]);
  BitBlock temp14 = simd_and(temp1, temp13);
  BitBlock temp15 = simd_andc(bit[6], bit[7]);
  BitBlock temp16 = simd_and(temp4, temp15);
  BitBlock RAngle = simd_and(temp14, temp16);
  BitBlock temp17 = simd_andc(bit[0], bit[1]);
  BitBlock temp18 = simd_and(bit[2], bit[3]);
  BitBlock temp19 = simd_and(temp17, temp18);
  BitBlock temp20 = simd_andc(bit[4], bit[5]);
  BitBlock temp21 = simd_and(bit[6], bit[7]);
  BitBlock temp22 = simd_and(temp20, temp21);
  BitBlock RBracket = simd_and(temp19, temp22);
  LexItem[Hyphen] = simd_andc(temp14, temp11);
  BitBlock temp23 = simd_and(temp4, temp21);
  BitBlock QMark = simd_and(temp14, temp23);
  BitBlock temp24 = simd_and(temp1, temp18);
  BitBlock temp25 = simd_and(temp4, bit[7]);
  BitBlock temp26 = simd_and(temp24, temp25);
  BitBlock temp27 = simd_or(temp26, temp7);
  LexItem[Quote] = simd_or(temp27, temp12);
  BitBlock temp28 = simd_andc(temp3, temp11);
  BitBlock temp29 = simd_and(temp20, temp15);
  BitBlock temp30 = simd_and(temp9, temp29);
  BitBlock temp31 = simd_or(temp28, temp30);
  BitBlock temp32 = simd_and(temp24, temp23);
  BitBlock temp33 = simd_or(temp31, temp32);
  BitBlock temp34 = simd_and(temp24, temp22);
  BitBlock temp35 = simd_or(temp33, temp34);
  BitBlock temp36 = simd_and(temp9, temp22);
  BitBlock temp37 = simd_or(temp35, temp36);
  BitBlock temp38 = simd_and(temp14, temp6);
  BitBlock temp39 = simd_or(temp37, temp38);
  BitBlock temp40 = simd_or(temp39, temp12);
  BitBlock temp41 = simd_andc(bit[7], bit[6]);
  BitBlock temp42 = simd_and(temp4, temp41);
  BitBlock temp43 = simd_and(temp24, temp42);
  BitBlock temp44 = simd_or(temp40, temp43);
  BitBlock temp45 = simd_and(temp3, temp42);
  BitBlock temp46 = simd_or(temp44, temp45);
  BitBlock temp47 = simd_and(temp9, temp42);
  BitBlock temp48 = simd_or(temp46, temp47);
  BitBlock temp49 = simd_and(temp9, temp6);
  BitBlock temp50 = simd_or(temp48, temp49);
  BitBlock temp51 = simd_and(temp3, temp16);
  BitBlock temp52 = simd_or(temp50, temp51);
  BitBlock temp53 = simd_and(temp14, temp22);
  BitBlock temp54 = simd_or(temp52, temp53);
  BitBlock temp55 = simd_andc(temp41, temp10);
  BitBlock temp56 = simd_and(temp14, temp55);
  BitBlock temp57 = simd_or(temp54, temp56);
  BitBlock temp58 = simd_and(temp9, temp16);
  BitBlock temp59 = simd_or(temp57, temp58);
  BitBlock temp60 = simd_or(temp59, temp7);
  BitBlock temp61 = simd_and(temp24, temp16);
  BitBlock temp62 = simd_or(temp60, temp61);
  BitBlock temp63 = simd_or(temp62, RAngle);
  BitBlock temp64 = simd_or(temp63, QMark);
  BitBlock temp65 = simd_and(temp19, temp29);
  BitBlock temp66 = simd_or(temp64, temp65);
  BitBlock temp67 = simd_and(bit[0], bit[1]);
  BitBlock temp68 = simd_and(temp67, temp13);
  BitBlock temp69 = simd_andc(temp68, temp11);
  BitBlock temp70 = simd_or(temp66, temp69);
  BitBlock temp71 = simd_or(temp70, RBracket);
  BitBlock temp72 = simd_andc(temp19, temp11);
  BitBlock temp73 = simd_or(temp71, temp72);
  BitBlock temp74 = simd_andc(temp67, temp2);
  BitBlock temp75 = simd_andc(temp74, temp11);
  BitBlock temp76 = simd_or(temp73, temp75);
  BitBlock temp77 = simd_and(temp3, temp23);
  BitBlock temp78 = simd_or(temp76, temp77);
  BitBlock temp79 = simd_and(temp67, temp8);
  BitBlock temp80 = simd_andc(temp79, temp11);
  BitBlock temp81 = simd_or(temp78, temp80);
  BitBlock temp82 = simd_and(temp17, temp13);
  BitBlock temp83 = simd_and(temp82, temp55);
  LexItem[NameFollow] = simd_or(temp81, temp83);
#ifdef DIGIT_AND_HEX_ITEMS
  BitBlock temp84 = simd_and(temp67, temp18);
  BitBlock temp85 = simd_andc(temp11, temp55);
  BitBlock temp86 = simd_andc(temp15, temp10);
  BitBlock temp87 = simd_andc(temp85, temp86);
  BitBlock temp88 = simd_andc(temp21, temp10);
  BitBlock temp89 = simd_andc(temp87, temp88);
  BitBlock temp90 = simd_andc(bit[5], bit[4]);
  BitBlock temp91 = simd_andc(temp90, temp5);
  BitBlock temp92 = simd_andc(temp89, temp91);
  BitBlock temp93 = simd_and(temp90, temp41);
  BitBlock temp94 = simd_andc(temp92, temp93);
  BitBlock temp95 = simd_and(temp90, temp15);
  BitBlock temp96 = simd_andc(temp94, temp95);
  BitBlock temp97 = simd_and(temp90, temp21);
  BitBlock temp98 = simd_andc(temp96, temp97);
  BitBlock temp99 = simd_andc(temp20, temp5);
  BitBlock temp100 = simd_andc(temp98, temp99);
  BitBlock temp101 = simd_and(temp20, temp41);
  BitBlock temp102 = simd_andc(temp100, temp101);
  BitBlock Digit = simd_andc(temp84, temp102);
  BitBlock temp103 = simd_and(temp74, temp55);
  BitBlock temp104 = simd_or(Digit, temp103);
  BitBlock temp105 = simd_and(temp74, temp86);
  BitBlock temp106 = simd_or(temp104, temp105);
  BitBlock temp107 = simd_and(temp74, temp88);
  BitBlock temp108 = simd_or(temp106, temp107);
  BitBlock temp109 = simd_and(temp74, temp91);
  BitBlock temp110 = simd_or(temp108, temp109);
  BitBlock temp111 = simd_and(temp74, temp93);
  BitBlock temp112 = simd_or(temp110, temp111);
  BitBlock temp113 = simd_and(temp74, temp95);
  BitBlock temp114 = simd_or(temp112, temp113);
  BitBlock temp115 = simd_andc(temp17, temp2);
  BitBlock temp116 = simd_and(temp115, temp55);
  BitBlock temp117 = simd_or(temp114, temp116);
  BitBlock temp118 = simd_and(temp115, temp86);
  BitBlock temp119 = simd_or(temp117, temp118);
  BitBlock temp120 = simd_and(temp115, temp88);
  BitBlock temp121 = simd_or(temp119, temp120);
  BitBlock temp122 = simd_and(temp115, temp91);
  BitBlock temp123 = simd_or(temp121, temp122);
  BitBlock temp124 = simd_and(temp115, temp93);
  BitBlock temp125 = simd_or(temp123, temp124);
  BitBlock temp126 = simd_and(temp115, temp95);
  BitBlock Hex = simd_or(temp125, temp126);

  LexItem[NonDigit] = simd_not(Digit);
  LexItem[NonHex] = simd_not(Hex);

#endif

	/* Mark potential occurrences of ']]>'  These are all actual
	occurrences of ]]> as well as occurrences of ]] or ] at
	the block end. Shifting the RBracket and RAngle streams in
	negated forms ensures that a potential CD_End is not ruled
	out at the block boundary. */
	LexItem[CD_End_check] = simd_andc(RBracket,
					  simd_or(sisd_sbli(simd_not(RBracket), 1),
						  sisd_sbli(simd_not(RAngle), 2)));
#ifndef OMIT_CD_End_check_In_Markup_Scan
	LexItem[MarkupStart] = simd_or(LexItem[MarkupStart], LexItem[CD_End_check]);
#endif
}


/* A temporary structure for internal use in ComputeLexicalItemStreams. */
typedef struct {
	BitBlock LexicalItems[LexicalItemCount];
} LexicalItemBlock;




template <CodeUnit_Base C>
void Lexer<C>::Do_MarkupStreams() {
	LexicalItemBlock lx_blk[BUFFER_BLOCKS];
	for (int i = 0; i < buffer_blocks; i++) {
		ComputeLexicalItemBlocks<C>(x8basis[i].bit, lx_blk[i].LexicalItems);
	}
	/* NonWS stream already completed by WS_Control method. */
	for (int j = MarkupStart; j < LexicalItemCount; j++) {
		for (int i = 0; i < buffer_blocks; i++) {
			parsing_engine_data->item_stream[j][i] = lx_blk[i].LexicalItems[j];
		}
	}
	for (int i = 0; i < buffer_blocks; i++) {
		parsing_engine_data->item_stream[NameFollow][i] =
			simd_or(parsing_engine_data->item_stream[NameFollow][i],
				simd_not(parsing_engine_data->item_stream[NonWS][i]));
	}
};


void UTF_8_Lexer::Do_CharsetValidation() {
	BitBlock u8prefix, u8suffix, u8prefix2, u8prefix3or4, u8prefix3, u8prefix4;
	BitBlock error_mask;
	/*  UTF-8 sequences may cross block boundaries.  If a
	    prefix is found near the end of a block that requires
	    one or more suffixes in the next block, then
	    prefix_pending is set to mark the positions.
	    However, at the beginning of the buffer, no suffixes
	    are expected, so this value is initialized to zeroes. */
#ifdef TEMPLATED_SIMD_LIB
	BitBlock prefix_pending = simd<1>::constant<0>();
	/*  If a suffix is pending, then it may involve one of
	    the special case prefixes E0, ED. F0, F4, or the
            EF prefix or EF_BF combination for FFFF/FFFE detection.*/
	BitBlock E0ED_pending = simd<1>::constant<0>();
	BitBlock F0F4_pending = simd<1>::constant<0>();
	BitBlock bit5_pending = simd<1>::constant<0>();
	BitBlock EF_pending = simd<1>::constant<0>();
	BitBlock EF_BF_pending = simd<1>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
	BitBlock prefix_pending = simd_const_1(0);
	/*  If a suffix is pending, then it may involve one of
	    the special case prefixes E0, ED. F0, F4, or the
            EF prefix or EF_BF combination for FFFF/FFFE detection.*/
	BitBlock E0ED_pending = simd_const_1(0);
	BitBlock F0F4_pending = simd_const_1(0);
	BitBlock bit5_pending = simd_const_1(0);
	BitBlock EF_pending = simd_const_1(0);
	BitBlock EF_BF_pending = simd_const_1(0);
#endif
	/* Temporary variables used within the block. */
	BitBlock suffix_required_scope;
	BitBlock prefix_E0ED, E0ED_scope, bit5_scope, E0ED_constraint;
	BitBlock prefix_F5FF, prefix_F0F4, F0F4_scope, F0F4_constraint;
	BitBlock X111x, B111x, prefix_EF, BF, EF_BF, EF_scope, EF_BF_scope;

	for (int i = 0; i < buffer_blocks; i++) {
#ifdef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd<1>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd_const_1(0);
#endif
		/* If there is no pending suffix and no bit 0, then there
		   are no possible validation issues for this block. */
		if (!bitblock_has_bit(simd_or(prefix_pending, x8basis[i].bit[0])))
			continue;
		/*  Compute classifications of UTF-8 bytes. */
		u8prefix = simd_and(x8basis[i].bit[0], x8basis[i].bit[1]);
		u8suffix = simd_andc(x8basis[i].bit[0], x8basis[i].bit[1]);
		u8prefix3or4 = simd_and(u8prefix, x8basis[i].bit[2]);
		u8prefix2 = simd_andc(u8prefix, x8basis[i].bit[2]);
		u8prefix3 = simd_andc(u8prefix3or4, x8basis[i].bit[3]);
		u8prefix4 = simd_and(u8prefix3or4, x8basis[i].bit[3]);

		/*  Initiate validation for two-byte sequences. */
		error_mask = simd_andc(u8prefix2,
					simd_or(simd_or(x8basis[i].bit[3], x8basis[i].bit[4]),
						simd_or(x8basis[i].bit[5], x8basis[i].bit[6])));
		suffix_required_scope = simd_or(prefix_pending, sisd_sfli(u8prefix, 1));

		prefix_pending = sisd_sbli(u8prefix, BLOCKSIZE - 1);
		E0ED_scope = E0ED_pending;
		F0F4_scope = F0F4_pending;
		bit5_scope = bit5_pending;
		EF_scope = EF_pending;
		EF_BF_scope = EF_BF_pending;

		/* Default values of pending variables for next iteration. */
#ifdef TEMPLATED_SIMD_LIB
		E0ED_pending = simd<1>::constant<0>();
		F0F4_pending = simd<1>::constant<0>();
		bit5_pending = simd<1>::constant<0>();
		EF_pending = simd<1>::constant<0>();
		EF_BF_pending = simd<1>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
		E0ED_pending = simd_const_1(0);
		F0F4_pending = simd_const_1(0);
		bit5_pending = simd_const_1(0);
		EF_pending = simd_const_1(0);
		EF_BF_pending = simd_const_1(0);
#endif

		X111x = simd_and(simd_and(x8basis[i].bit[4], x8basis[i].bit[5]), x8basis[i].bit[6]);
		B111x = simd_and(simd_and(u8suffix, simd_and(x8basis[i].bit[2], x8basis[i].bit[3])),
                                 X111x);
		BF = simd_and(B111x, x8basis[i].bit[7]);
		EF_BF = simd_and(EF_scope, BF);

		if (bitblock_has_bit(u8prefix3or4)) {
			/*  Extend validation for errors in three-byte sequences. */
			suffix_required_scope = simd_or(suffix_required_scope,
							sisd_sfli(u8prefix3or4, 2));
			bit5_scope = simd_or(bit5_scope, sisd_sfli(x8basis[i].bit[5], 1));
			prefix_E0ED = simd_andc(u8prefix3,
						simd_or(simd_or(x8basis[i].bit[6],
								simd_xor(x8basis[i].bit[4], x8basis[i].bit[7])),
							simd_xor(x8basis[i].bit[4], x8basis[i].bit[5])));
			E0ED_scope = simd_or(E0ED_scope, sisd_sfli(prefix_E0ED, 1));
			prefix_EF = simd_and(u8prefix3, simd_and(X111x, x8basis[i].bit[7]));
			EF_scope = simd_or(EF_scope, sisd_sfli(prefix_EF, 1));
			EF_BF = simd_and(EF_scope, BF);

			/* Values for next iteration. */
			prefix_pending = simd_or(prefix_pending,
						 sisd_sbli(u8prefix3or4, BLOCKSIZE - 2));
			bit5_pending = sisd_sbli(x8basis[i].bit[5], BLOCKSIZE - 1);
			E0ED_pending = sisd_sbli(prefix_E0ED, BLOCKSIZE - 1);
			EF_pending = sisd_sbli(prefix_EF, BLOCKSIZE - 1);
			EF_BF_pending = sisd_sbli(EF_BF, BLOCKSIZE - 2);
			if (bitblock_has_bit(u8prefix4)) {
				/*  Extend validation for errors in four-byte sequences. */
				suffix_required_scope = simd_or(suffix_required_scope,
								sisd_sfli(u8prefix4, 3));
				prefix_pending = simd_or(prefix_pending,
							 sisd_sbli(u8prefix4, BLOCKSIZE - 3));
				prefix_F5FF = simd_and(u8prefix4,
						       simd_or(x8basis[i].bit[4],
								simd_and(x8basis[i].bit[5],
									simd_or(x8basis[i].bit[6], x8basis[i].bit[7]))));
				error_mask = simd_or(error_mask, prefix_F5FF);
				prefix_F0F4 = simd_andc(u8prefix4,
							simd_or(x8basis[i].bit[4],
								simd_or(x8basis[i].bit[6], x8basis[i].bit[7])));
				F0F4_scope = simd_or(F0F4_scope, sisd_sfli(prefix_F0F4, 1));
				F0F4_pending = sisd_sbli(prefix_F0F4, BLOCKSIZE - 1);
			}
		}
		E0ED_constraint = simd_xor(bit5_scope, x8basis[i].bit[2]);
		error_mask = simd_or(error_mask, simd_andc(E0ED_scope, E0ED_constraint));
		F0F4_constraint = simd_xor(bit5_scope,
					   simd_or(x8basis[i].bit[2], x8basis[i].bit[3]));
		error_mask = simd_or(error_mask, simd_andc(F0F4_scope, F0F4_constraint));
		/*  Complete validation by checking for prefix-suffix mismatches. */
		error_mask = simd_or(error_mask, simd_xor(suffix_required_scope, u8suffix));

		EF_BF_scope = simd_or(EF_BF_scope, sisd_sfli(EF_BF, 1));
		error_mask = simd_or(error_mask, simd_and(EF_BF_scope, B111x));
		validation_stream[i] = error_mask;
#ifdef DEBUG_UTF8_VALIDATION
//		if (bitblock_has_bit(error_mask)) {
printf("-%i----------------------\n", i);
print_bit_block("x8basis[i].bit[0]", x8basis[i].bit[0]);
print_bit_block("x8basis[i].bit[1]", x8basis[i].bit[1]);
print_bit_block("x8basis[i].bit[2]", x8basis[i].bit[2]);
print_bit_block("x8basis[i].bit[3]", x8basis[i].bit[3]);
	print_bit_block("u8prefix2", u8prefix2);
	print_bit_block("u8prefix3", u8prefix3);
	print_bit_block("u8prefix4", u8prefix4);
	print_bit_block("suffix_required_scope", suffix_required_scope);
	print_bit_block("prefix_pending", prefix_pending);
	print_bit_block("E0ED_pending", E0ED_pending);
	print_bit_block("F0F4_pending", F0F4_pending);
	print_bit_block("bit5_pending", bit5_pending);
		print_bit_block("error_mask", error_mask);

//}
#endif
	}
};


void ASCII_7_Lexer::Do_CharsetValidation() {
	for (int blk = 0; blk < buffer_blocks; blk++) {
		validation_stream[blk] = x8basis[blk].bit[0];
	}
};


void EASCII_8_Lexer::Do_CharsetValidation() {
	/* Nothing required for most charsets - but perhaps should have tables. */
	for (int i = 0; i < buffer_blocks; i++) {
#ifdef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd<1>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd_const_1(0);
#endif
	}
};


void UTF_16_Lexer::Do_CharsetValidation() {
};


void UCS_2_Lexer::Do_CharsetValidation() {
};


void UTF_32_Lexer::Do_CharsetValidation() {
};


void EBCDIC_Lexer::Do_CharsetValidation() {
	/* Nothing required for most cases - but perhaps should have tables. */
	for (int i = 0; i < buffer_blocks; i++) {
#ifdef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd<1>::constant<0>();
#endif
#ifndef TEMPLATED_SIMD_LIB
		validation_stream[i] = simd_const_1(0);
#endif
	}
}



/* Stub out XML 1.1 routines initially. */

void UTF_8_Lexer::Do_XML_11_WS_Control() {
	printf("UTF_8_Lexer::Do_XML_11_WS_Control not yet implemented; using XML 1.0 rules.\n");
	Do_XML_10_WS_Control();
};


static inline void ASCII_7_WS_Control_Blocks_11(BitBlock bit[], BitBlock& WS, BitBlock& Control) {
	BitBlock temp1 = simd_or(bit[0], bit[1]);
	BitBlock temp2 = simd_or(temp1, bit[2]);
	BitBlock temp3 = simd_andc(bit[1], bit[0]);
	BitBlock temp4 = simd_and(bit[2], bit[3]);
	BitBlock temp5 = simd_and(temp3, temp4);
	BitBlock temp6 = simd_and(bit[4], bit[5]);
	BitBlock temp7 = simd_and(bit[6], bit[7]);
	BitBlock temp8 = simd_and(temp6, temp7);
	BitBlock temp9 = simd_and(temp5, temp8);
	BitBlock temp10 = simd_andc(temp2, temp9);
	BitBlock temp11 = simd_andc(temp10, bit[0]);
#ifdef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd<1>::constant<1>(), temp11);
#endif
#ifndef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd_const_1(1), temp11);
#endif
	BitBlock temp12 = simd_or(bit[2], bit[3]);
	BitBlock temp13 = simd_or(temp1, temp12);
	BitBlock temp14 = simd_andc(bit[7], bit[6]);
	BitBlock temp15 = simd_and(temp6, temp14);
	BitBlock CR = simd_andc(temp15, temp13);
	BitBlock temp16 = simd_andc(bit[4], bit[5]);
	BitBlock temp17 = simd_andc(bit[6], bit[7]);
	BitBlock temp18 = simd_and(temp16, temp17);
	BitBlock LF = simd_andc(temp18, temp13);
	BitBlock temp19 = simd_and(temp16, temp14);
	BitBlock HT = simd_andc(temp19, temp13);
	BitBlock temp20 = simd_andc(bit[2], bit[3]);
	BitBlock temp21 = simd_andc(temp20, temp1);
	BitBlock temp22 = simd_or(bit[4], bit[5]);
	BitBlock temp23 = simd_or(bit[6], bit[7]);
	BitBlock temp24 = simd_or(temp22, temp23);
	BitBlock SP = simd_andc(temp21, temp24);
	WS = simd_or(simd_or(CR, LF), simd_or(HT, SP));
}


void ASCII_7_Lexer::Do_XML_11_WS_Control() {
	BitBlock WS, Control;
	for (int i = 0; i < buffer_blocks; i++) {
		ASCII_7_WS_Control_Blocks_11(x8basis[i].bit, WS, Control);
		parsing_engine_data->item_stream[NonWS][i] = simd_not(WS);
		validation_stream[i] = simd_andc(Control, WS);
	}
};

static inline void EASCII_8_WS_Control_Blocks_11(BitBlock bit[], BitBlock& WS, BitBlock& Control) {
	BitBlock temp1 = simd_or(bit[0], bit[1]);
	BitBlock temp2 = simd_or(temp1, bit[2]);
	BitBlock temp3 = simd_andc(bit[1], bit[0]);
	BitBlock temp4 = simd_and(bit[2], bit[3]);
	BitBlock temp5 = simd_and(temp3, temp4);
	BitBlock temp6 = simd_and(bit[4], bit[5]);
	BitBlock temp7 = simd_and(bit[6], bit[7]);
	BitBlock temp8 = simd_and(temp6, temp7);
	BitBlock temp9 = simd_and(temp5, temp8);
	BitBlock temp10 = simd_andc(temp2, temp9);
	BitBlock temp11 = simd_andc(bit[0], bit[1]);
	BitBlock temp12 = simd_andc(temp11, bit[2]);
	BitBlock temp13 = simd_andc(temp10, temp12);
#ifdef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd<1>::constant<1>(), temp13);
#endif
#ifndef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd_const_1(1), temp13);
#endif
	BitBlock temp14 = simd_or(bit[2], bit[3]);
	BitBlock temp15 = simd_or(temp1, temp14);
	BitBlock temp16 = simd_andc(bit[7], bit[6]);
	BitBlock temp17 = simd_and(temp6, temp16);
	BitBlock CR = simd_andc(temp17, temp15);
	BitBlock temp18 = simd_andc(bit[4], bit[5]);
	BitBlock temp19 = simd_andc(bit[6], bit[7]);
	BitBlock temp20 = simd_and(temp18, temp19);
	BitBlock LF = simd_andc(temp20, temp15);
	BitBlock temp21 = simd_and(temp18, temp16);
	BitBlock HT = simd_andc(temp21, temp15);
	BitBlock temp22 = simd_andc(bit[2], bit[3]);
	BitBlock temp23 = simd_andc(temp22, temp1);
	BitBlock temp24 = simd_or(bit[4], bit[5]);
	BitBlock temp25 = simd_or(bit[6], bit[7]);
	BitBlock temp26 = simd_or(temp24, temp25);
	BitBlock SP = simd_andc(temp23, temp26);
	BitBlock temp27 = simd_andc(temp11, temp14);
	BitBlock temp28 = simd_andc(bit[5], bit[4]);
	BitBlock temp29 = simd_and(temp28, temp16);
	BitBlock NEL = simd_and(temp27, temp29);
	WS = simd_or(simd_or(simd_or(CR, LF), simd_or(HT, SP)), NEL);
}

void EASCII_8_Lexer::Do_XML_11_WS_Control() {
	BitBlock WS, Control;
	for (int i = 0; i < buffer_blocks; i++) {
		EASCII_8_WS_Control_Blocks_11(x8basis[i].bit, WS, Control);
		parsing_engine_data->item_stream[NonWS][i] = simd_not(WS);
		validation_stream[i] = simd_andc(Control, WS);
	}
};


void U16_Lexer::Do_XML_11_WS_Control() {
	printf("U16_Lexer::Do_XML_11_WS_Control not yet implemented; using XML 1.0 rules.\n");
	Do_XML_10_WS_Control();
};


void UTF_32_Lexer::Do_XML_11_WS_Control() {
	printf("UTF_32_Lexer::Do_XML_11_WS_Control not yet implemented; using XML 1.0 rules.\n");
	Do_XML_10_WS_Control();
};

static inline void EBCDIC_WS_Control_Blocks_11(BitBlock bit[], BitBlock& WS, BitBlock& Control) {
	BitBlock temp1 = simd_or(bit[0], bit[1]);
	BitBlock temp2 = simd_and(bit[0], bit[1]);
	BitBlock temp3 = simd_and(bit[2], bit[3]);
	BitBlock temp4 = simd_and(temp2, temp3);
	BitBlock temp5 = simd_and(bit[4], bit[5]);
	BitBlock temp6 = simd_and(bit[6], bit[7]);
	BitBlock temp7 = simd_and(temp5, temp6);
	BitBlock temp8 = simd_and(temp4, temp7);
	BitBlock temp9 = simd_andc(temp1, temp8);
#ifdef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd<1>::constant<1>(), temp9);
#endif
#ifndef TEMPLATED_SIMD_LIB
	Control = simd_andc(simd_const_1(1), temp9);
#endif
	BitBlock temp10 = simd_or(bit[2], bit[3]);
	BitBlock temp11 = simd_or(temp1, temp10);
	BitBlock temp12 = simd_andc(bit[7], bit[6]);
	BitBlock temp13 = simd_and(temp5, temp12);
	BitBlock CR = simd_andc(temp13, temp11);
	BitBlock temp14 = simd_andc(bit[2], bit[3]);
	BitBlock temp15 = simd_andc(temp14, temp1);
	BitBlock temp16 = simd_andc(bit[5], bit[4]);
	BitBlock temp17 = simd_and(temp16, temp12);
	BitBlock LF = simd_and(temp15, temp17);
	BitBlock HT = simd_andc(temp17, temp11);
	BitBlock temp18 = simd_andc(bit[1], bit[0]);
	BitBlock temp19 = simd_andc(temp18, temp10);
	BitBlock temp20 = simd_or(bit[4], bit[5]);
	BitBlock temp21 = simd_or(bit[6], bit[7]);
	BitBlock temp22 = simd_or(temp20, temp21);
	BitBlock SP = simd_andc(temp19, temp22);
	BitBlock temp23 = simd_andc(bit[3], bit[2]);
	BitBlock temp24 = simd_andc(temp23, temp1);
	BitBlock NEL = simd_and(temp24, temp17);
	WS = simd_or(simd_or(simd_or(CR, LF), simd_or(HT, SP)), NEL);
}

void EBCDIC_Lexer::Do_XML_11_WS_Control() {
	BitBlock WS, Control;
	for (int i = 0; i < buffer_blocks; i++) {
		EBCDIC_WS_Control_Blocks_11(x8basis[i].bit, WS, Control);
		parsing_engine_data->item_stream[NonWS][i] = simd_not(WS);
		validation_stream[i] = simd_andc(Control, WS);
	}
};



void Lexer_Interface::AnalyzeBuffer(BitBlockBasis * basis, int base_pos, int start_pos, int buffer_limit_pos) {
#ifdef DEBUG
	printf("Entered AnalyzeBuffer, buffer_limit_pos = %i\n", buffer_limit_pos);
#endif
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == BITLEX_ALL)
	code_clocker->start_interval();
#endif
	x8basis = basis;
	lexer_base_pos = base_pos; /* for error reporting. */
	int err_pos;
	buffer_blocks = (buffer_limit_pos + BLOCKSIZE - 1)/BLOCKSIZE;
	buffer_units = buffer_limit_pos;
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == CHARSET_VALIDATION)
	code_clocker->start_interval();
#endif

        Do_CharsetValidation();
	/* Ignore error bits before start_pos which only arise
	   due to UTF8 pending scope streams at buffer boundaries.*/
	err_pos = bitstream_scan(validation_stream, start_pos);
	/* Detect validation errors up to the end of file plus one more
	   position in case there is an incomplete code unit at EOF. */
	if ((err_pos <= buffer_units) && (err_pos < BUFFER_SIZE)) {
//		printf("start_pos =%i\n, err_pos = %i\n", start_pos, err_pos);
//		print_bit_block("validation_stream[0]", validation_stream[0]);

//		print_bit_block("validation_stream[err_pos/128]", validation_stream[err_pos/128]);

		CharSetValidationError((const char *) entity_Info->encoding, lexer_base_pos + err_pos);
	}
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == CHARSET_VALIDATION)
		code_clocker->end_interval(buffer_units);
#endif
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == WS_CONTROL)
	code_clocker->start_interval();
#endif

	if (entity_Info->version == XML_1_1) Do_XML_11_WS_Control();
        else Do_XML_10_WS_Control();
#ifdef DEBUG
	printf("Do_WS_Control() complete.\n");
#endif
  	err_pos = bitstream_scan0(validation_stream);
	if (err_pos < buffer_units) XMLCharacterError(lexer_base_pos + err_pos);
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == WS_CONTROL)
	code_clocker->end_interval(buffer_units);
#endif
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == MARKUP_STREAMS)
	code_clocker->start_interval();
#endif
	Do_MarkupStreams();
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == MARKUP_STREAMS)
	code_clocker->end_interval(buffer_units);
#endif
#ifdef DEBUG
	printf("Do_MarkupStreams() complete.\n");
#endif

	if (buffer_units < BUFFER_SIZE) {
#ifdef TEMPLATED_SIMD_LIB
		BitBlock final_block_mask =
			sisd_sfl(simd<1>::constant<1>(), sisd_from_int(buffer_units % BLOCKSIZE));
#endif
#ifndef TEMPLATED_SIMD_LIB
		BitBlock final_block_mask =
			sisd_sfl(simd_const_1(1), sisd_from_int(buffer_units % BLOCKSIZE));
#endif
		int lastblk = buffer_units/BLOCKSIZE;
		for (int j = minLexicalItem; j < LexicalItemCount; j++) {
        		parsing_engine_data->item_stream[j][lastblk] =
				simd_or(parsing_engine_data->item_stream[j][lastblk],
					final_block_mask);
		}
	}
#if defined(PAPI) and defined(CODE_CLOCKING) and (CODE_CLOCKING == BITLEX_ALL)
		code_clocker->end_interval(buffer_units);
#endif

}