LexatomLoader_Converter (6732B)
1 /* -*- C++ -*- vim: set syntax=cpp: 2 * (C) 2007-2008 Frank-Rene Schaefer */ 3 #ifndef __QUEX_INCLUDE_GUARD__BUFFER__LEXATOMS__LEXATOM_LOADER_CONVERTER 4 #define __QUEX_INCLUDE_GUARD__BUFFER__LEXATOMS__LEXATOM_LOADER_CONVERTER 5 6 /* 7 * PURPOSE: (NOTE: This has to be reworked) 8 * 9 * Reads data from a stream and converts the incoming lexatoms into a 10 * lexatom format required by the user. The user can specify the coding 11 * format of the input stream as well as the coding format in which he 12 * wants it to be put into his buffer. 13 * 14 * NOTE: Streams for the quex::buffer comply to three simple requirements: 15 * 16 * tell() returning a value of the current input position 17 * seek(pos) sets the the current input position in the stream 18 * read_lexatoms(buffer, N) fills the buffer with N lexatoms from the stream. 19 * returns the number of actually filled lexatoms. 20 * 21 * 22 * MAIN GOAL: The main goal of a 'FixedSizeCharacterStream' class is to 23 * fill a buffer with N lexatoms starting from consecutive 24 * stream positions. 25 * 26 * ASSUMPTION: 27 * 28 * -- The input lexatom format is arbitrary. It can be anything that is 29 * supported by 'iconv'. Flexible lexatom length codings are supported. 30 * 31 * -- The user's lexatoms format has a fixed lexatom size for each 32 * lexatom. Any coding that involves different byte numbers for different 33 * lexatoms or 'history' cannot be provided as user format. 34 * 35 * PRINCIPLE: 36 * 37 * ________ _______ ________ 38 * | | | ¦ | | 39 * | FILE | ------> | raw | ------> | Buffer | 40 * |________| |_______| |________| 41 * 42 * A data stream (FILE) delivers lexatoms in an arbitrary coding format. 43 * These lexatoms are stored bytewise inside a 'raw buffer'. Then this data is 44 * transformed into the users coding format. The result is directly put into 45 * the user's buffer. 46 * 47 * (1) Fill as many bytes from the stream into the raw buffer 48 * stream_position 49 * | 50 * Stream: ||aa.aa.aa.bb.cc.cc.cc.cc.dd.dd.ee.ee.ee.ee.ff.ff.gg.|| 51 * 52 * raw_buffer.iterator 53 * | 54 * Raw Buffer: ||aa.aa.aa.bb.cc.cc.cc.cc.dd.dd.ee.ee.ee|| 55 * 56 * 57 * (2) Convert lexatoms in the raw buffer and store into users buffer 58 * 59 * 60 * User's Buffer: ||AA.AA.BB.BB.CC.CC________________________|| 61 * 62 * 63 * yes 64 * (2b) User's buffer is filled? --------> **EXIT/DONE** 65 * | 66 * | no 67 * * 68 * (3a) Copy unconverted bytes to the beginning of the raw buffer 69 * 70 * Raw Buffer: ||ee.ee.ee______________________________|| 71 * 72 * 73 * (3b) Load remaining bytes from the stream into raw buffer 74 * 75 * raw_buffer.iterator 76 * | 77 * Raw Buffer: ||ee.ee.ee.ee.ff.ff.gg.hh.ii.ii.________|| 78 * 79 * Goto (2) 80 * 81 * (4) Convert lexatoms from raw buffer and store them into user's buffer 82 * 83 * User's Buffer: ||AA.AA.BB.BB.CC.CC.DD.DD.EE.EE.FF.FF.GG.GG|| 84 * 85 * (4b) goto (2b) 86 * 87 * NOTE: The 'raw_buffer.iterator' remains between calls to "read_lexatoms()". 88 * This is equivalent to having some bytes in the pipeline. 89 **********************************************************************************/ 90 #if ! defined (__QUEX_OPTION_PLAIN_C) 91 # include <iostream> 92 # include <cerrno> 93 # include <stdexcept> 94 # include <cstddef> 95 #else 96 # include <stddef.h> /* gets: ptrdiff_t */ 97 #endif 98 #include "definitions" 99 #include "bufferBuffer" 100 #include "LexatomLoader" 101 #include "Converter" 102 103 QUEX_NAMESPACE_MAIN_OPEN 104 105 typedef struct QUEX_SETTING_USER_CLASS_DECLARATION_EPILOG { 106 E_Ownership ownership; 107 108 uint8_t* begin; 109 uint8_t* fill_end_p; 110 uint8_t* memory_end; 111 112 uint8_t* next_to_convert_p; 113 114 } QUEX_NAME(RawBuffer); 115 116 typedef struct QUEX_SETTING_USER_CLASS_DECLARATION_EPILOG { 117 /* LexatomLoader_Converter: 118 * *---------------------> QUEX_NAME(ByteLoader 119 )* *---------------------> Converter 120 * */ 121 QUEX_NAME(LexatomLoader) base; 122 123 QUEX_NAME(ByteLoader)* byte_loader; 124 QUEX_NAME(Converter)* converter; 125 QUEX_NAME(RawBuffer) raw_buffer; 126 127 } QUEX_NAME(LexatomLoader_Converter); 128 129 130 QUEX_INLINE QUEX_NAME(LexatomLoader)* 131 QUEX_NAME(LexatomLoader_Converter_new)(QUEX_NAME(ByteLoader)* byte_loader, 132 QUEX_NAME(Converter)* converter, 133 size_t RawBufferSize); 134 135 QUEX_INLINE void QUEX_NAME(LexatomLoader_Converter_move_away_passed_content)(QUEX_NAME(LexatomLoader_Converter)*); 136 QUEX_INLINE size_t QUEX_NAME(__LexatomLoader_Converter_fill_raw_buffer)(QUEX_NAME(LexatomLoader_Converter)*); 137 138 QUEX_INLINE void QUEX_NAME(RawBuffer_init)(QUEX_NAME(RawBuffer)* me, 139 uint8_t* Begin, size_t SizeInBytes); 140 141 #ifdef QUEX_OPTION_ASSERTS 142 # define QUEX_ASSERT_RAW_BUFFER(BI) \ 143 __quex_assert(BI); \ 144 __quex_assert((BI)->next_to_convert_p >= (BI)->begin); \ 145 __quex_assert((BI)->next_to_convert_p <= (BI)->fill_end_p);\ 146 __quex_assert((BI)->fill_end_p >= (BI)->begin); \ 147 __quex_assert((BI)->fill_end_p <= (BI)->memory_end); 148 149 # define QUEX_ASSERT_RAW_BUFFER_EASY(BI) \ 150 __quex_assert(BI); \ 151 __quex_assert((BI)->next_to_convert_p >= (BI)->begin); \ 152 __quex_assert((BI)->next_to_convert_p <= (BI)->fill_end_p); 153 154 #else 155 # define QUEX_ASSERT_RAW_BUFFER(BI) /* empty */ 156 # define QUEX_ASSERT_RAW_BUFFER_EASY(BI) /* empty */ 157 #endif 158 159 QUEX_NAMESPACE_MAIN_CLOSE 160 161 162 #ifdef QUEX_OPTION_CONVERTER_ICONV 163 # include <quex/code_base/buffer/lexatoms/converter/iconv/Converter_IConv> 164 #endif 165 #ifdef QUEX_OPTION_CONVERTER_ICU 166 # include <quex/code_base/buffer/lexatoms/converter/icu/Converter_ICU> 167 #endif 168 169 170 #endif /* __QUEX_INCLUDE_GUARD__BUFFER__LEXATOMS__LEXATOM_LOADER_CONVERTER */