Buffer.i (42386B)
1 /* vim:set ft=c: -*- C++ -*- */ 2 #ifndef __QUEX_INCLUDE_GUARD__BUFFER__BUFFER_I 3 #define __QUEX_INCLUDE_GUARD__BUFFER__BUFFER_I 4 5 #include "asserts" 6 #include "bufferAsserts" 7 #include "definitions" 8 #include "bufferBuffer" 9 #include "Buffer_debug.i" 10 #include "LexatomLoader" 11 #include "MemoryManager" 12 13 QUEX_NAMESPACE_MAIN_OPEN 14 15 QUEX_INLINE ptrdiff_t QUEX_NAME(Buffer_move_forward)(QUEX_NAME(Buffer)* me, 16 ptrdiff_t move_distance); 17 QUEX_INLINE void QUEX_NAME(Buffer_move_forward_undo)(QUEX_NAME(Buffer)* me, 18 intmax_t move_distance, 19 ptrdiff_t move_size); 20 QUEX_INLINE ptrdiff_t QUEX_NAME(Buffer_move_backward)(QUEX_NAME(Buffer)* me, 21 ptrdiff_t move_distance); 22 23 QUEX_INLINE void* QUEX_NAME(Buffer_fill)(QUEX_NAME(Buffer)* me, 24 const void* ContentBegin, 25 const void* ContentEnd); 26 QUEX_INLINE void QUEX_NAME(Buffer_fill_prepare)(QUEX_NAME(Buffer)* me, 27 void** begin_p, 28 const void** end_p); 29 QUEX_INLINE void QUEX_NAME(Buffer_fill_finish)(QUEX_NAME(Buffer)* me, 30 const void* FilledEndP); 31 QUEX_INLINE bool QUEX_NAME(Buffer_is_end_of_stream_inside)(QUEX_NAME(Buffer)* me); 32 QUEX_INLINE void QUEX_NAME(Buffer_init_content)(QUEX_NAME(Buffer)* me, 33 QUEX_TYPE_LEXATOM* EndOfFileP); 34 QUEX_INLINE void QUEX_NAME(Buffer_init_analyzis)(QUEX_NAME(Buffer)* me); 35 36 QUEX_INLINE void QUEX_NAME(Buffer_on_overflow_DEFAULT)(QUEX_NAME(Buffer)*, bool ForwardF); 37 QUEX_INLINE void QUEX_NAME(Buffer_on_content_change_DEFAULT)(const QUEX_TYPE_LEXATOM*, 38 const QUEX_TYPE_LEXATOM*); 39 40 QUEX_INLINE void 41 QUEX_NAME(Buffer_construct)(QUEX_NAME(Buffer)* me, 42 QUEX_NAME(LexatomLoader)* filler, 43 QUEX_TYPE_LEXATOM* memory, 44 const size_t MemorySize, 45 QUEX_TYPE_LEXATOM* EndOfFileP, 46 E_Ownership Ownership) 47 { 48 /* Ownership of InputMemory is passed to 'me->_memory'. */ 49 QUEX_NAME(BufferMemory_construct)(&me->_memory, memory, MemorySize, 50 Ownership); 51 52 /* By setting begin and end to zero, we indicate to the loader that 53 * this is the very first load procedure. */ 54 me->filler = filler; 55 me->fill = QUEX_NAME(Buffer_fill); 56 me->fill_prepare = QUEX_NAME(Buffer_fill_prepare); 57 me->fill_finish = QUEX_NAME(Buffer_fill_finish); 58 59 /* Event handlers. */ 60 me->on_content_change = QUEX_NAME(Buffer_on_content_change_DEFAULT); 61 me->on_overflow = QUEX_NAME(Buffer_on_overflow_DEFAULT); 62 63 /* Initialize. */ 64 QUEX_NAME(Buffer_init_content)(me, EndOfFileP); 65 QUEX_NAME(Buffer_init_analyzis)(me); 66 67 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 68 } 69 70 QUEX_INLINE void 71 QUEX_NAME(Buffer_destruct)(QUEX_NAME(Buffer)* me) 72 { 73 if( me->filler && me->filler->ownership == E_Ownership_LEXICAL_ANALYZER ) { 74 me->filler->delete_self(me->filler); 75 } 76 QUEX_NAME(BufferMemory_destruct)(&me->_memory); 77 } 78 79 QUEX_INLINE void 80 QUEX_NAME(Buffer_init_analyzis)(QUEX_NAME(Buffer)* me) 81 /* Initialize: _read_p 82 * _lexeme_start_p 83 * _lexatom_at_lexeme_start 84 * _lexatom_before_lexeme_start */ 85 { 86 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 87 88 if( ! me->_memory._front ) { 89 /* No memory => Analyzer is put into a non-functional state. */ 90 me->_read_p = (QUEX_TYPE_LEXATOM*)0; 91 me->_lexeme_start_p = (QUEX_TYPE_LEXATOM*)0; 92 me->_lexatom_at_lexeme_start = (QUEX_TYPE_LEXATOM)0; 93 # ifdef __QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION 94 me->_lexatom_before_lexeme_start = (QUEX_TYPE_LEXATOM)0; 95 # endif 96 } 97 else { 98 /* The first state in the state machine does not increment. 99 * => input pointer is set to the first position, not before. */ 100 me->_read_p = BeginP; 101 me->_lexeme_start_p = BeginP; 102 me->_lexatom_at_lexeme_start = '\0'; /* Nothing covered. */ 103 # ifdef __QUEX_OPTION_SUPPORT_BEGIN_OF_LINE_PRE_CONDITION 104 /* When the buffer is initialized, a line begins. Set 'newline'. */ 105 me->_lexatom_before_lexeme_start = QUEX_SETTING_CHARACTER_NEWLINE_IN_ENGINE_CODEC; 106 # endif 107 } 108 } 109 110 QUEX_INLINE void 111 QUEX_NAME(Buffer_init_content)(QUEX_NAME(Buffer)* me, QUEX_TYPE_LEXATOM* EndOfFileP) 112 /* Initialize: input.lexatom_index_begin 113 * input.lexatom_index_end_of_stream 114 * input.end_p */ 115 { 116 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 117 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 118 QUEX_TYPE_STREAM_POSITION ci_begin = (QUEX_TYPE_STREAM_POSITION)0; 119 QUEX_TYPE_STREAM_POSITION ci_end_of_stream = (QUEX_TYPE_STREAM_POSITION)-1; 120 QUEX_TYPE_LEXATOM* end_p = (QUEX_TYPE_LEXATOM*)0; 121 (void)EndP; 122 123 if( ! me->_memory._front ) { 124 ci_end_of_stream = (QUEX_TYPE_STREAM_POSITION)-1; 125 end_p = (QUEX_TYPE_LEXATOM*)0; 126 ci_begin = (QUEX_TYPE_STREAM_POSITION)-1; 127 } 128 else if( me->filler && me->filler->byte_loader ) { 129 __quex_assert(! EndOfFileP); 130 131 # if 0 132 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, BeginP, ContentSize, 133 0, &end_of_stream_f, &encoding_error_f); 134 ci_end_of_stream = ((! loaded_n) || end_of_stream_f) ? loaded_n 135 : (QUEX_TYPE_STREAM_POSITION)-1; 136 end_p = &BeginP[loaded_n]; 137 # endif 138 /* Setup condition to initiate immediate load when the state machine 139 * is entered: 'read pointer hits buffer limit code'. */ 140 ci_begin = (QUEX_TYPE_STREAM_POSITION)0; 141 ci_end_of_stream = (QUEX_TYPE_STREAM_POSITION)-1; 142 end_p = &BeginP[0]; 143 } 144 else { 145 __quex_assert(me->_memory._front); /* See first condition. */ 146 __quex_assert(! EndOfFileP || (EndOfFileP >= BeginP && EndOfFileP <= EndP)); 147 148 if( EndOfFileP ) { 149 ci_end_of_stream = EndOfFileP - BeginP; 150 end_p = EndOfFileP; 151 } 152 else { 153 ci_end_of_stream = (QUEX_TYPE_STREAM_POSITION)-1; 154 end_p = BeginP; 155 } 156 } 157 me->input.lexatom_index_begin = ci_begin; 158 me->input.lexatom_index_end_of_stream = ci_end_of_stream; 159 me->input.end_p = end_p; 160 if( me->input.end_p ) { 161 *(me->input.end_p) = QUEX_SETTING_BUFFER_LIMIT_CODE; 162 } 163 164 QUEX_IF_ASSERTS_poison(&me->input.end_p[1], me->_memory._back); 165 } 166 167 QUEX_INLINE void 168 QUEX_NAME(Buffer_register_content)(QUEX_NAME(Buffer)* me, 169 QUEX_TYPE_LEXATOM* EndOfInputP, 170 QUEX_TYPE_STREAM_POSITION CharacterIndexBegin) 171 /* Registers information about the stream that fills the buffer and its 172 * relation to the buffer. 173 * 174 * EndOfInputP --> Position behind the last lexatom in the buffer that has 175 * been streamed. 176 * '0' --> No change. 177 * 178 * CharacterIndexBegin --> Character index of the first lexatom in the 179 * buffer. 180 * '-1' --> No change. */ 181 { 182 if( EndOfInputP ) { 183 __quex_assert(EndOfInputP <= me->_memory._back); 184 __quex_assert(EndOfInputP > me->_memory._front); 185 186 me->input.end_p = EndOfInputP; 187 *(me->input.end_p) = QUEX_SETTING_BUFFER_LIMIT_CODE; 188 } 189 190 if( CharacterIndexBegin != (QUEX_TYPE_STREAM_POSITION)-1 ) { 191 me->input.lexatom_index_begin = CharacterIndexBegin; 192 } 193 194 QUEX_IF_ASSERTS_poison(&me->input.end_p[1], me->_memory._back); 195 /* NOT: assert(QUEX_NAME(Buffer_input_lexatom_index_begin)(me) >= 0); 196 * This function may be called before content is setup/loaded propperly. */ 197 } 198 199 QUEX_INLINE void 200 QUEX_NAME(Buffer_register_eos)(QUEX_NAME(Buffer)* me, 201 QUEX_TYPE_STREAM_POSITION CharacterIndexEndOfStream) 202 { 203 me->input.lexatom_index_end_of_stream = CharacterIndexEndOfStream; 204 } 205 206 QUEX_INLINE bool 207 QUEX_NAME(Buffer_is_empty)(QUEX_NAME(Buffer)* me) 208 /* RETURNS: true, if buffer does not contain anything. 209 * false, else. */ 210 { 211 return me->input.end_p == &me->_memory._front[1] 212 && me->input.lexatom_index_begin == 0; 213 } 214 215 QUEX_INLINE QUEX_TYPE_STREAM_POSITION 216 QUEX_NAME(Buffer_input_lexatom_index_end)(QUEX_NAME(Buffer)* me) 217 /* RETURNS: Character index of the lexatom to which '.input.end_p' points. 218 * */ 219 { 220 __quex_assert(me->input.lexatom_index_begin >= 0); 221 QUEX_BUFFER_ASSERT_pointers_in_range(me); 222 223 return me->input.lexatom_index_begin 224 + (me->input.end_p - &me->_memory._front[1]); 225 } 226 227 QUEX_INLINE void 228 QUEX_NAME(Buffer_read_p_add_offset)(QUEX_NAME(Buffer)* buffer, const size_t Offset) 229 /* Add offset to '._read_p'. No check applies whether this is admissible. 230 * */ 231 { 232 QUEX_BUFFER_ASSERT_pointers_in_range(buffer); 233 buffer->_read_p += Offset; 234 QUEX_BUFFER_ASSERT_pointers_in_range(buffer); 235 } 236 237 QUEX_INLINE size_t 238 QUEX_NAME(Buffer_content_size)(QUEX_NAME(Buffer)* me) 239 { 240 return QUEX_NAME(BufferMemory_size)(&(me->_memory)) - 2; 241 } 242 243 QUEX_INLINE bool 244 QUEX_NAME(Buffer_is_end_of_file)(QUEX_NAME(Buffer)* me) 245 { 246 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 247 if ( me->_read_p != me->input.end_p ) return false; 248 else if( me->input.lexatom_index_end_of_stream == -1 ) return false; 249 250 return QUEX_NAME(Buffer_input_lexatom_index_end)(me) 251 == me->input.lexatom_index_end_of_stream; 252 } 253 254 QUEX_INLINE bool 255 QUEX_NAME(Buffer_is_end_of_stream_inside)(QUEX_NAME(Buffer)* me) 256 { 257 const ptrdiff_t ContentSize = (ptrdiff_t)QUEX_NAME(Buffer_content_size)(me); 258 259 if ( me->input.lexatom_index_end_of_stream == -1 ) return false; 260 else if( me->input.lexatom_index_end_of_stream < me->input.lexatom_index_begin ) return false; 261 262 return me->input.lexatom_index_end_of_stream - me->input.lexatom_index_begin < ContentSize; 263 } 264 265 QUEX_INLINE bool 266 QUEX_NAME(Buffer_is_begin_of_file)(QUEX_NAME(Buffer)* buffer) 267 { 268 QUEX_BUFFER_ASSERT_CONSISTENCY(buffer); 269 if ( buffer->_read_p != buffer->_memory._front ) return false; 270 else if( QUEX_NAME(Buffer_input_lexatom_index_begin)(buffer) ) return false; 271 else return true; 272 } 273 274 QUEX_INLINE bool 275 QUEX_NAME(Buffer_move_and_load_forward)(QUEX_NAME(Buffer)* me, 276 QUEX_TYPE_STREAM_POSITION NewCharacterIndexBegin, 277 QUEX_TYPE_STREAM_POSITION MinCharacterIndexInBuffer) 278 /* RETURNS: true -- if the the buffer could be filled start from 279 * NewCharacterIndexBegin. 280 * false, else. 281 * 282 * In case, that the loading fails, the buffer is setup as it was BEFORE the call 283 * to this function. 284 * 285 * EXPLANATION: 286 * 287 * Before: .-------------------------------------- prev lexatom_index_begin 288 * : 289 * | . . . . . . . . .x.x.x.x.x.x.x.x.x.x.x| 290 * |<---- move size ---->| 291 * After: |<- move distance | 292 * .-------------------------------------- new lexatom_index_begin 293 * : .---------------- prev lexatom index begin 294 * : : 295 * |x.x.x.x.x.x.x.x.x.x.x|N.N.N.N.N.N.N. . | 296 * |- move_size -------->|- loaded_n ->| 297 * 298 * Moves the region of size 'Size' from the end of the buffer to the beginning 299 * of the buffer and tries to load as many lexatoms as possible behind it. */ 300 { 301 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 302 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 303 const ptrdiff_t ContentSize = (ptrdiff_t)QUEX_NAME(Buffer_content_size)(me); 304 QUEX_TYPE_STREAM_POSITION load_lexatom_index; 305 ptrdiff_t load_request_n; 306 QUEX_TYPE_LEXATOM* load_p; 307 ptrdiff_t loaded_n; 308 intmax_t move_distance; 309 ptrdiff_t move_size; 310 bool end_of_stream_f = false; 311 bool encoding_error_f = false; 312 313 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 314 __quex_assert(me->input.lexatom_index_begin <= NewCharacterIndexBegin); 315 __quex_assert(NewCharacterIndexBegin <= MinCharacterIndexInBuffer); 316 __quex_assert(NewCharacterIndexBegin + ContentSize >= MinCharacterIndexInBuffer ); 317 318 if( me->input.lexatom_index_end_of_stream != -1 319 && MinCharacterIndexInBuffer >= me->input.lexatom_index_end_of_stream ) { 320 /* If the end of the stream is INSIDE the buffer already, then there 321 * is no need, no chance, of loading more content. */ 322 return false; 323 } 324 325 /* Move existing content in the buffer to appropriate position. */ 326 move_distance = NewCharacterIndexBegin - me->input.lexatom_index_begin; 327 move_size = QUEX_NAME(Buffer_move_forward)(me, (ptrdiff_t)move_distance); 328 load_lexatom_index = NewCharacterIndexBegin + move_size; 329 load_request_n = ContentSize - move_size; 330 load_p = &BeginP[move_size]; 331 332 __quex_assert(load_lexatom_index == NewCharacterIndexBegin + (load_p - BeginP)); 333 __quex_assert(load_p >= BeginP); 334 __quex_assert(&load_p[load_request_n] <= EndP); 335 (void)EndP; 336 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, load_p, load_request_n, 337 load_lexatom_index, 338 &end_of_stream_f, &encoding_error_f); 339 340 if( (! loaded_n) || end_of_stream_f ) { /* End of stream detected. */ 341 QUEX_NAME(Buffer_register_eos)(me, load_lexatom_index + loaded_n); 342 } 343 344 /* (3) In case of failure, restore previous buffer content. */ 345 if( MinCharacterIndexInBuffer >= load_lexatom_index + loaded_n ) { 346 QUEX_NAME(Buffer_move_forward_undo)(me, move_distance, move_size); 347 return false; 348 } 349 350 QUEX_NAME(Buffer_register_content)(me, &load_p[loaded_n], NewCharacterIndexBegin); 351 return true; 352 } 353 354 QUEX_INLINE bool 355 QUEX_NAME(Buffer_move_and_load_backward)(QUEX_NAME(Buffer)* me, 356 QUEX_TYPE_STREAM_POSITION NewCharacterIndexBegin) 357 /* Before: 358 * .------------------------------------- prev lexatom index begin 359 * : 360 * |x.x.x.x.x.x.x.x.x.x. . . . . . . . . . . . . | 361 * |<--- move size---->| 362 * After: 363 * .------------------------------------- new lexatom index begin 364 * : .--------------- prev lexatom index begin 365 * : : 366 * :--- move distance--->| 367 * |N.N.N.N.N.N.N.N.N.N.N.x.x.x.x.x.x.x.x.x.x. . | 368 * 369 * Moves the region of size 'Size' from the beginning of the buffer to the end 370 * and tries to load as many lexatoms as possible behind it. If the try fails 371 * something is seriously wrong. */ 372 { 373 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 374 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 375 QUEX_TYPE_STREAM_POSITION ci_begin = QUEX_NAME(Buffer_input_lexatom_index_begin)(me); 376 ptrdiff_t load_request_n; 377 ptrdiff_t loaded_n; 378 ptrdiff_t move_distance; 379 QUEX_TYPE_LEXATOM* end_p; 380 bool end_of_stream_f = false; 381 bool encoding_error_f = false; 382 383 __quex_assert(NewCharacterIndexBegin >= 0); 384 __quex_assert(ci_begin >= NewCharacterIndexBegin); 385 386 /* (1) Move away content, so that previous content can be reloaded. */ 387 move_distance = (ptrdiff_t)(ci_begin - NewCharacterIndexBegin); 388 load_request_n = QUEX_NAME(Buffer_move_backward)(me, (ptrdiff_t)move_distance); 389 390 __quex_assert(&BeginP[load_request_n] <= EndP); 391 392 /* (2) Move away content, so that previous content can be reloaded. */ 393 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, BeginP, load_request_n, 394 NewCharacterIndexBegin, 395 &end_of_stream_f, &encoding_error_f); 396 397 /* (3) In case of error, the stream must have been corrupted. Previously 398 * present content is not longer available. Continuation impossible. */ 399 if( loaded_n != load_request_n ) { 400 QUEX_ERROR_EXIT("Buffer filler failed to load content that has been loaded before.!"); 401 return false; 402 } 403 404 end_p = EndP - me->input.end_p < move_distance ? 405 EndP : &me->input.end_p[move_distance]; 406 407 QUEX_NAME(Buffer_register_content)(me, end_p, NewCharacterIndexBegin); 408 return true; 409 } 410 411 QUEX_INLINE ptrdiff_t 412 QUEX_NAME(Buffer_move_away_passed_content)(QUEX_NAME(Buffer)* me, 413 QUEX_TYPE_LEXATOM** position_register, 414 const size_t PositionRegisterN) 415 /* Free some space AHEAD so that new content can be loaded. Content that 416 * is still used, or expected to be used shall remain inside the buffer. 417 * Following things need to be respected: 418 * 419 * _lexeme_start_p --> points to the lexeme that is currently treated. 420 * MUST BE INSIDE BUFFER! 421 * _read_p --> points to the lexatom that is currently used 422 * for triggering. MUST BE INSIDE BUFFER! 423 * fall back region --> A used defined buffer backwards from the lexeme 424 * start. Shall help to avoid extensive backward 425 * loading. 426 * 427 * RETURNS: Pointer to the end of the maintained content. */ 428 { 429 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 430 const QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 431 const ptrdiff_t ContentSize = (ptrdiff_t)QUEX_NAME(Buffer_content_size)(me); 432 QUEX_TYPE_LEXATOM* move_begin_p; 433 ptrdiff_t move_size; 434 ptrdiff_t move_distance; 435 const ptrdiff_t FallBackN = (ptrdiff_t)QUEX_SETTING_BUFFER_MIN_FALLBACK_N; 436 QUEX_TYPE_LEXATOM** pr_it = 0x0; 437 438 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 439 __quex_assert(FallBackN >= 0); 440 441 if( me->_read_p - me->_lexeme_start_p >= ContentSize - FallBackN ) { 442 /* OVERFLOW: If stretch from _read_p to _lexeme_start_p 443 * spans the whole buffer, then nothing can be loaded. */ 444 me->on_overflow(me, /* Forward */ true); 445 return 0; 446 } 447 else if( QUEX_NAME(Buffer_is_end_of_stream_inside)(me) ) { 448 /* Refuse the move, if the end of stream is inside buffer. */ 449 return 0; 450 } 451 452 /* Determine from where the region-to-be-moved BEGINS, what its size is 453 * and how far it is to be moved. */ 454 move_begin_p = me->_read_p; 455 if( me->_lexeme_start_p ) { 456 move_begin_p = QUEX_MIN(move_begin_p, me->_lexeme_start_p); 457 } 458 if( move_begin_p - BeginP <= FallBackN ) { 459 return 0; 460 } 461 move_begin_p = &move_begin_p[- FallBackN]; 462 move_distance = move_begin_p - BeginP; 463 464 if( ! move_distance ) { 465 return 0; 466 } 467 468 move_size = QUEX_NAME(Buffer_move_forward)(me, move_distance); 469 470 /* Pointer Adaption: _read_p, _lexeme_start_p, position registers. 471 * input.end_p, input.end_lexatom_index */ 472 me->_read_p -= move_distance; 473 if( me->_lexeme_start_p ) me->_lexeme_start_p -= move_distance; 474 475 if( position_register ) { 476 /* All position registers MUST point behind '_lexeme_start_p'. */ 477 for(pr_it = position_register; pr_it != &position_register[PositionRegisterN]; ++pr_it) { 478 if( ! *pr_it ) continue; 479 *pr_it = (*pr_it - BeginP) >= move_distance ? *pr_it - move_distance : 0; 480 } 481 } 482 483 /* input.end_p/end_lexatom_index: End lexatom index remains the SAME, 484 * since no new content has been loaded into the buffer. */ 485 __quex_assert(me->input.end_p - BeginP >= move_distance); 486 487 QUEX_NAME(Buffer_register_content)(me, &me->input.end_p[- move_distance], 488 me->input.lexatom_index_begin + move_distance); 489 490 /*_______________________________________________________________________*/ 491 __quex_assert(me->input.end_p == &move_begin_p[move_size - move_distance]); 492 (void)move_size; 493 QUEX_IF_ASSERTS_poison(&EndP[- move_distance + 1], EndP); 494 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 495 (void)EndP; 496 497 return move_distance; 498 } 499 500 QUEX_INLINE E_LoadResult 501 QUEX_NAME(Buffer_load_forward)(QUEX_NAME(Buffer)* me, 502 QUEX_TYPE_LEXATOM** position_register, 503 const size_t PositionRegisterN) 504 /* Load as much new content into the buffer as possible--from what lies ahead 505 * in the input stream. Maintains '_read_p', '_lexeme_start_p' inside the 506 * buffer (if possible also fallback region). The 'input.end_p' pointer and 507 * 'input.end_lexatom_index' are adapted according to the newly loaded 508 * content, i.e. the point to exactly the same lexatom as before the load. 509 * 510 * BEHAVIOR: BLOCKING wait for incoming stream content. 511 * No return without content--except at end of stream. 512 * 513 * Buffer and pointers are adapted are adapted IN ANY CASE! 514 * 515 * (i) Content present: 516 * => return 'true'. 517 * 518 * (ii) No content: 519 * => pointers are 'disabled' because 'end_p = _read_p'. 520 * return 'false'. 521 * 522 * RETURNS: 523 * 524 * DONE => Something has been loaded (analysis MAY CONTINUE) 525 * FAILURE => General load failure. (analysis MUST STOP) 526 * NO_SPACE_FOR_LOAD => Lexeme exceeds buffer size. (analysis MUST STOP) 527 * ENCODING_ERROR => Failed. conversion error (analysis MUST STOP) 528 * NO_MORE_DATA => No more data available. (analysis MUST STOP) 529 * BAD_LEXATOM => Read pointer on buffer limit code, 530 * but it was not a buffer limit. 531 * 532 * The case of 'end-of-stream' may be true in both cases. When 'end-of-stream' 533 * is detected, the lexatom index of the 'end-of-stream' is registered. This 534 * prevents future attemps to load beyond that index. Again, even if 535 * 'end-of-stream' has been detected, there might be lexatoms for the lexer 536 * to chew on. */ 537 { 538 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 539 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 540 const ptrdiff_t ContentSize = (ptrdiff_t)QUEX_NAME(Buffer_content_size)(me); 541 QUEX_TYPE_STREAM_POSITION ci_begin = QUEX_NAME(Buffer_input_lexatom_index_begin)(me); 542 QUEX_TYPE_STREAM_POSITION ci_load_begin; 543 ptrdiff_t move_distance; 544 ptrdiff_t load_request_n; 545 ptrdiff_t loaded_n; 546 bool end_of_stream_f = false; 547 bool encoding_error_f = false; 548 549 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 550 551 if( me->_read_p != me->input.end_p) { 552 /* If the read pointer does not stand on the end of input pointer, then 553 * the 'buffer limit code' at the read pointer is a bad lexatom. 554 * Buffer limit codes cannot be possibly be part of buffer content. */ 555 return E_LoadResult_BAD_LEXATOM; 556 } 557 else if( ! me->filler || ! me->filler->byte_loader ) { 558 QUEX_NAME(Buffer_register_eos)(me, ci_begin + (me->input.end_p - BeginP)); 559 return E_LoadResult_NO_MORE_DATA; /* No filler/loader => no load! */ 560 } 561 562 /* Move remaining content. 563 * Maintain lexeme and fallback. 564 * Adapt pointers. */ 565 move_distance = QUEX_NAME(Buffer_move_away_passed_content)(me, position_register, 566 PositionRegisterN); 567 if( ! move_distance && me->input.end_p == EndP ) { 568 return E_LoadResult_NO_SPACE_FOR_LOAD; /* No free space for loading. */ 569 } 570 571 /* Load new content. */ 572 ci_load_begin = me->input.lexatom_index_begin + (me->input.end_p - BeginP); 573 load_request_n = ContentSize - (me->input.end_p - BeginP); 574 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, 575 me->input.end_p, load_request_n, 576 ci_load_begin, 577 &end_of_stream_f, 578 &encoding_error_f); 579 QUEX_NAME(Buffer_register_content)(me, &me->input.end_p[loaded_n], -1); 580 581 if( ! loaded_n ) { 582 /* If filler returned, then either some lexatoms have been filled, 583 * or it indicates 'end-of-stream' by 'load_n = 0'. */ 584 end_of_stream_f = true; 585 } 586 if( end_of_stream_f ) { 587 QUEX_NAME(Buffer_register_eos)(me, ci_load_begin + loaded_n); 588 } 589 590 __quex_debug_buffer_load(me, "LOAD FORWARD(exit)\n"); 591 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 592 if ( encoding_error_f ) return E_LoadResult_BAD_LEXATOM; 593 else if( loaded_n ) return E_LoadResult_DONE; 594 else return E_LoadResult_NO_MORE_DATA; 595 } 596 597 QUEX_INLINE ptrdiff_t 598 QUEX_NAME(Buffer_move_away_upfront_content)(QUEX_NAME(Buffer)* me) 599 /* Free some space in the REAR so that previous content can be re-loaded. Some 600 * content is to be left in front, so that no immediate reload is necessary 601 * once the analysis goes forward again. Following things need to be respected: 602 * 603 * _lexeme_start_p --> points to the lexeme that is currently treated. 604 * MUST BE INSIDE BUFFER! 605 * _read_p --> points to the lexatom that is currently used 606 * for triggering. MUST BE INSIDE BUFFER! 607 * 608 * RETURNS: Distance the the buffer content has been freed to be filled. */ 609 { 610 const QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 611 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 612 QUEX_TYPE_LEXATOM* LastP = &me->_memory._back[-1]; 613 const ptrdiff_t ContentSize = EndP - BeginP; 614 ptrdiff_t move_distance; 615 ptrdiff_t move_distance_max; 616 ptrdiff_t move_distance_nominal; 617 QUEX_TYPE_LEXATOM* end_p; 618 619 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 620 __quex_assert(me->_read_p <= LastP); 621 622 if( me->input.lexatom_index_begin == 0 && BeginP == me->_read_p ) { 623 return 0; /* Begin of stream. */ 624 } 625 else if( me->_lexeme_start_p >= LastP ) { 626 /* If _lexeme_start_p at back, then no new content can be loaded. */ 627 me->on_overflow(me, /* Forward */ false); 628 return 0; 629 } 630 631 /* Max. possible move distance: where 'read_p' or 'lexeme_start_p' 632 * land on last position in the buffer. */ 633 move_distance_max = LastP - QUEX_MAX(me->_read_p, me->_lexeme_start_p); 634 /* Also, move before lexatom index '0' is impossible. */ 635 move_distance_max = QUEX_MIN(move_distance_max, (ptrdiff_t)me->input.lexatom_index_begin); 636 637 /* Desired move distance: ContentSize / 3 */ 638 move_distance_nominal = ContentSize > 3 ? ContentSize / 3 : ContentSize; 639 640 move_distance = QUEX_MIN(move_distance_max, move_distance_nominal); 641 if( ! move_distance ) { 642 return 0; 643 } 644 645 (void)QUEX_NAME(Buffer_move_backward)(me, move_distance); 646 647 /* Pointer Adaption: _read_p, _lexeme_start_p. */ 648 me->_read_p += move_distance; 649 if( me->_lexeme_start_p ) me->_lexeme_start_p += move_distance; 650 651 /* Adapt and of content pointer and new lexatom index at begin. */ 652 end_p = EndP - me->input.end_p < move_distance ? EndP 653 : &me->input.end_p[move_distance]; 654 655 QUEX_NAME(Buffer_register_content)(me, end_p, 656 me->input.lexatom_index_begin - move_distance); 657 658 /*_______________________________________________________________________*/ 659 QUEX_IF_ASSERTS_poison(BeginP, &BeginP[move_distance]); 660 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 661 662 return move_distance; 663 } 664 665 QUEX_INLINE E_LoadResult 666 QUEX_NAME(Buffer_load_backward)(QUEX_NAME(Buffer)* me) 667 /* Load *previous* content into the buffer so that the analyzer can continue 668 * seeminglessly (in backward direction). 669 * 670 * BEHAVIOR: BLOCKING wait for incoming stream content. 671 * No return without content--except at end of stream. 672 * 673 * Buffer and pointers are adapted are adapted IN ANY CASE! 674 * 675 * RETURNS: 676 * 677 * DONE => Something has been loaded (analysis MAY CONTINUE) 678 * FAILURE => General load failure. (analysis MUST STOP) 679 * NO_SPACE_FOR_LOAD => Lexeme exceeds buffer size. (analysis MUST STOP) 680 * ENCODING_ERROR => Failed. Conversion error. (analysis MUST STOP) 681 * NO_MORE_DATA => Begin of stream reached. (analysis MUST STOP) 682 * 683 * __________________________________________________________________________ 684 * ! In the false case, the range from 'Begin' to '_lexeme_start_p' may ! 685 * ! have ARBITRARY CONTENT. Then the '_read_p' MUST be reset IMMEDIATELY and ! 686 * ! only forward analysis may work. ! 687 * '--------------------------------------------------------------------------' 688 * 689 *_____________________________________________________________________________ 690 * NO ADAPTIONS OF POST-CONTEXT POSITIONS. Reason: Backward analysis appears 691 * only in the following two cases. 692 * 693 * (1) When checking for a pre-condition. This does not involve pre-contexts. 694 * 695 * (2) When tracing back along a 'pseudo-ambigous post context'. However, 696 * the stretch from 'end-of-core' pattern to 'end-of-post context' lies 697 * completely in between 'lexeme start' to 'read '. Thus, one never has 698 * to go farther back then the buffer's begin. */ 699 { 700 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 701 ptrdiff_t move_distance; 702 ptrdiff_t loaded_n; 703 bool end_of_stream_f = false; 704 bool encoding_error_f = false; 705 706 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 707 708 __quex_debug_buffer_load(me, "BACKWARD(entry)\n"); 709 if( me->_read_p != me->_memory._front ) { 710 /* If the read pointer does not stand on 'front', then the buffer limit 711 * code is a bad character, Buffer limit codes are not supposed to 712 * appear in buffer content. */ 713 return E_LoadResult_BAD_LEXATOM; 714 } 715 else if( me->input.lexatom_index_begin == 0 ) { 716 return E_LoadResult_NO_MORE_DATA; /* Begin of stream. */ 717 } 718 else if( ! me->filler || ! me->filler->byte_loader ) { 719 return E_LoadResult_NO_MORE_DATA; /* No filler/loader => no loading! */ 720 } 721 else if( ! QUEX_NAME(ByteLoader_seek_is_enabled)(me->filler->byte_loader) ) { 722 return E_LoadResult_NO_MORE_DATA; /* Stream cannot go backwards. */ 723 } 724 725 move_distance = QUEX_NAME(Buffer_move_away_upfront_content)(me); 726 727 if( ! move_distance ) { 728 return E_LoadResult_NO_SPACE_FOR_LOAD; /* Cannot be further back. */ 729 } 730 731 /* Load new content. */ 732 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, 733 BeginP, move_distance, 734 me->input.lexatom_index_begin, 735 &end_of_stream_f, &encoding_error_f); 736 if( encoding_error_f ) { 737 return E_LoadResult_BAD_LEXATOM; 738 } 739 740 if( loaded_n != move_distance ) { 741 /* Serious: previously loaded content could not be loaded again! 742 * => Buffer has now hole: 743 * from BeginP[loaded_n] to Begin[move_distance] 744 * The analysis can continue in forward direction, but not backwards.*/ 745 return E_LoadResult_FAILURE; 746 } 747 748 __quex_debug_buffer_load(me, "BACKWARD(exit)\n"); 749 QUEX_BUFFER_ASSERT_CONSISTENCY(me); 750 return E_LoadResult_DONE; 751 } 752 753 QUEX_INLINE ptrdiff_t 754 QUEX_NAME(Buffer_move_forward)(QUEX_NAME(Buffer)* me, 755 ptrdiff_t move_distance) 756 /* Moves the entire (meaningful) content of the buffer by 'move_distance' 757 * forward. It does NOT MODIFY any pointers about the buffer content! 758 * 759 * EndP 760 * |<---- move size ---->| 761 * Before: | . . . . . . . . . . . . .x.x.x.x.x.x.x.x.x.x.x| 762 * |<---- move distance -----| | 763 * .----------------------' | 764 * .-' .----------------' 765 * | .-------' 766 * After: |x.x.x.x.x.x.x.x.x.x.x| . . . . . . . . . . . . | 767 * |<---- move_size ---->| 768 * 769 * The callback 'on_content_change()' informs the user that any pointer into 770 * the buffer must now copy its pointed objects, because the pointers will 771 * become invalid. 772 * 773 * RETURNS: Number of lexatoms that have been moved. */ 774 { 775 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 776 const ptrdiff_t FilledSize = me->input.end_p - BeginP; 777 ptrdiff_t move_size; 778 779 me->on_content_change(BeginP, me->input.end_p); 780 781 if( move_distance >= FilledSize ) { 782 return 0; 783 } 784 785 move_size = FilledSize - move_distance; 786 787 if( move_distance && move_size ) { 788 __QUEX_STD_memmove((void*)BeginP, (void*)&BeginP[move_distance], 789 (size_t)move_size * sizeof(QUEX_TYPE_LEXATOM)); 790 } 791 return move_size; 792 } 793 794 QUEX_INLINE ptrdiff_t 795 QUEX_NAME(Buffer_move_backward)(QUEX_NAME(Buffer)* me, 796 ptrdiff_t move_distance) 797 /* Moves content so that previous content may be filled into the buffer. 798 * 799 * BeginP 800 * |<--- move size---->| 801 * |x.x.x.x.x.x.x.x.x.x| . . . . . . . . . . . . . . | 802 * | '--------------. 803 * '------------. '-------------. 804 * '---------------. | 805 * :------- move distance------->| | 806 * | . . . . . . . . . . . . . . |x.x.x.x.x.x.x.x.x.x| 807 * 808 * 809 * RETURNS: Number of lexatom that need to be filled into the gap. 810 * */ 811 { 812 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 813 const ptrdiff_t ContentSize = (ptrdiff_t)QUEX_NAME(Buffer_content_size)(me); 814 ptrdiff_t move_size; 815 816 me->on_content_change(BeginP, me->input.end_p); 817 818 if( move_distance > ContentSize ) { 819 return ContentSize; 820 } 821 822 move_size = ContentSize - move_distance; 823 824 if( move_distance && move_size ) { 825 __QUEX_STD_memmove((void*)&BeginP[move_distance], BeginP, 826 (size_t)move_size * sizeof(QUEX_TYPE_LEXATOM)); 827 } 828 return (ptrdiff_t)move_distance; 829 } 830 831 QUEX_INLINE void 832 QUEX_NAME(Buffer_move_forward_undo)(QUEX_NAME(Buffer)* me, 833 intmax_t move_distance, 834 ptrdiff_t move_size) 835 /* Restore the buffer's raw memory to what it was before in the 'FORWARD' case. 836 * It is assumed that the buffer's parameters in 837 * 838 * me->input 839 * 840 * remained UNTOUCHED during the moving and loading of the caller function. 841 * That is, they indicate the situation to be restored. */ 842 { 843 QUEX_TYPE_LEXATOM* BeginP = &me->_memory._front[1]; 844 QUEX_TYPE_LEXATOM* EndP = me->_memory._back; 845 ptrdiff_t load_request_n; 846 ptrdiff_t loaded_n; 847 bool end_of_stream_f = false; 848 bool encoding_error_f = false; 849 850 /* Character with lexatom index 'MinCharacterIndexInBuffer' has 851 * not been loaded. => Buffer must be setup as before. */ 852 if( move_size ) { 853 __QUEX_STD_memmove((void*)&BeginP[move_distance], (void*)BeginP, 854 (size_t)move_size * sizeof(QUEX_TYPE_LEXATOM)); 855 load_request_n = (ptrdiff_t)move_distance; 856 } 857 else { 858 load_request_n = (me->input.end_p - BeginP); 859 } 860 __quex_assert(&BeginP[load_request_n] <= EndP); 861 (void)EndP; 862 loaded_n = QUEX_NAME(LexatomLoader_load)(me->filler, BeginP, load_request_n, 863 me->input.lexatom_index_begin, 864 &end_of_stream_f, &encoding_error_f); 865 866 if( loaded_n != load_request_n ) { 867 QUEX_ERROR_EXIT("Buffer filler failed to load content that has been loaded before.!"); 868 } 869 else { 870 /* Ensure, that the buffer limit code is restored. */ 871 *(me->input.end_p) = (QUEX_TYPE_LEXATOM)QUEX_SETTING_BUFFER_LIMIT_CODE; 872 } 873 } 874 875 QUEX_INLINE void 876 QUEX_NAME(Buffer_on_content_change_DEFAULT)(const QUEX_TYPE_LEXATOM* BeginP, 877 const QUEX_TYPE_LEXATOM* EndP) 878 { (void)BeginP; (void)EndP; } 879 880 QUEX_INLINE void 881 QUEX_NAME(Buffer_on_overflow_DEFAULT)(QUEX_NAME(Buffer)* me, bool ForwardF) 882 { 883 (void)me; (void)ForwardF; 884 # ifdef QUEX_OPTION_INFORMATIVE_BUFFER_OVERFLOW_MESSAGE 885 uint8_t utf8_encoded_str[512]; 886 char message[1024]; 887 const size_t MessageSize = (size_t)1024; 888 uint8_t* WEnd = 0x0; 889 uint8_t* witerator = 0x0; 890 QUEX_TYPE_LEXATOM* End = 0x0; 891 const QUEX_TYPE_LEXATOM* iterator = 0x0; 892 893 /* Print out the lexeme start, so that the user has a hint. */ 894 WEnd = utf8_encoded_str + 512 - 7; 895 witerator = utf8_encoded_str; 896 End = me->_memory._back; 897 iterator = me->_lexeme_start_p; 898 899 QUEX_CONVERTER_STRING(QUEX_SETTING_CHARACTER_CODEC, utf8)(&iterator, End, &witerator, WEnd); 900 901 message[0] = '\0'; 902 /* No use of 'snprintf()' because not all systems seem to support it propperly. */ 903 __QUEX_STD_strncat(message, 904 "Distance between lexeme start and current pointer exceeds buffer size.\n" 905 "(tried to load buffer", 906 MessageSize); 907 __QUEX_STD_strncat(message, ForwardF ? "forward)" : "backward)", MessageSize); 908 __QUEX_STD_strncat(message, "As a hint consider the beginning of the lexeme:\n[[", MessageSize); 909 __QUEX_STD_strncat(message, (char*)utf8_encoded_str, MessageSize); 910 __QUEX_STD_strncat(message, "]]\n", MessageSize); 911 912 QUEX_ERROR_EXIT(message); 913 # else 914 QUEX_ERROR_EXIT("Distance between lexeme start and current pointer exceeds buffer size.\n" 915 "(tried to load buffer forward). Please, compile with option\n\n" 916 " QUEX_OPTION_INFORMATIVE_BUFFER_OVERFLOW_MESSAGE\n\n" 917 "in order to get a more informative output. Most likely, one of your patterns\n" 918 "eats more than you intended. Alternatively you might want to set the buffer\n" 919 "size to a greater value or use skippers (<skip: [ \\n\\t]> for example).\n"); 920 # endif /* QUEX_OPTION_INFORMATIVE_BUFFER_OVERFLOW_MESSAGE */ 921 } 922 923 QUEX_NAMESPACE_MAIN_CLOSE 924 925 #include "LexatomLoader.i" 926 #include "Buffer_debug.i" 927 #include "Buffer_navigation.i" 928 #include "Buffer_fill.i" 929 #include "BufferMemory.i" 930 931 #endif /* __QUEX_INCLUDE_GUARD__BUFFER__BUFFER_I */ 932 933