ucn_token_buffer.cc

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
/*! \file
  \brief Token buffer.

  Definition of ucn_token_buffer class representing buffer
  of simple tokens, with methods for extracting literals.
  \author pt
*/
#include <lestes/common.hh>
#include <lestes/lang/cplus/lex/ucn_token_buffer.hh>
#include <lestes/lang/cplus/lex/ucn_token_buffer.m.hh>
#include <lestes/lang/cplus/lex/ucn_token.hh>
#include <lestes/lang/cplus/lex/token_value.hh>
#include <lestes/lang/cplus/lex/line_control.hh>
#include <lestes/std/source_location.hh>

package(lestes);
package(lang);
package(cplus);
package(lex);

using namespace ::std;

/*!
  Creates empty ucn token buffer.
  \post length() == 0
  \pre a_lines != NULL
  \param a_lines  The associated line control.
*/
ucn_token_buffer::ucn_token_buffer(const ptr<line_control> &a_lines):
        buffer(buffer_type::create()),
        lines(checked(a_lines))
{
}

/*!
  Adds single item to the end of the buffer.
  \param item  The item to be added.
*/
void ucn_token_buffer::add_back(const ptr<ucn_token> &item)
{
        buffer->push_back(item);
}

/*!
  Discards \a len items from the beginning.
  Used for skipping unimportant values.
  \pre len <= length()
  \param len  The length of the discarded sequence.
*/
void ucn_token_buffer::advance(ucn_token_buffer::size_type len)
{
        lassert(len <= length());
        while (len != 0) {
                buffer->pop_front();
                len--;
        }
}

/*!
  Returns item currently at the beginning of the buffer.
  \pre buffer.size() != 0
  \return The item at the beginning of the buffer.
*/
ptr<ucn_token> ucn_token_buffer::peek_front(void) const
{
        lassert(length() != 0);
        return buffer->front();
}

/*!
  Removes \a len items from the beginning of the buffer.
  Returns token value representing the removed items.
  Performs no scanning for ucn or ucn escape sequences.
  Used for identifiers with no ucn in any form and literals without escaped ucn.
  \pre len <= length()
  \param len The length of the removed sequence.
  \return The representation of the items' values.
*/
ptr<token_value> ucn_token_buffer::extract_ordinary(ucn_token_buffer::size_type len)
{
        lassert(len <= length());
        
        // reserve space
        ucn_string us(len,0xbeef);
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        for (ucn_token_buffer::size_type i = 0; i < len; i++, ++bit, ++sit) {
                *sit = (*bit)->value_get();
        }
        
        // erase removed items
        buffer->erase(buffer->begin(),bit);

        return token_value::create(us);
}

/*!
  Removes items from the beginning of the buffer until the stop value is encountered. 
  That value is not removed.
  Returns token value representing the removed items.
  Performs no scanning for ucn or ucn escape sequences.
  Used for hchar and qchar sequences.
  \pre The stop value is present in the sequence. 
  \param stop  The value to stop scanning the sequence.
  \return The representation of the items' values.
*/
ptr<token_value> ucn_token_buffer::extract_until(ucn stop)
{
        ucn_token_buffer::size_type len = length();
        // reserve space
        ucn_string us(len,0xbeef);
        ucn u;
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        for (ucn_token_buffer::size_type i = 0; i < len; i++, ++bit, ++sit) {
                u = (*bit)->value_get();
                if (u == stop) {
                        // erase removed items
                        buffer->erase(buffer->begin(),bit);
                        // return only the valid part of the string
                        return token_value::create(ucn_string(us.begin(),sit));
                }
                *sit = u;
        }

        lassert2(false,"The stop value was not found");
        return NULL; 
}

/*!
  Removes \a len items from the beginning of the buffer.
  Returns token value representing the removed items.
  Performs scanning for ucn escape sequences, issues errors for invalid ucn and
  ucn invalid in identifiers.
  Used for identifiers and preprocessing numbers containing translated or
  (possibly unterminated) escaped ucn characters.
  \pre len <= length()
  \pre Only contains backslash in ucn escape sequences.
  \pre Backslash is always followed by `U' or `u'.
  \param len  The length of the removed sequence.
  \param identifier  Flag set to true if extracting identifier.
  \return The representation of the items' values, with converted ucn escape sequences.
*/
ptr<token_value> ucn_token_buffer::extract_invalid_ucn(ucn_token_buffer::size_type len, bool identifier)
{
        lassert(len <= length());
        
        // state of the function
        enum {
                BEGIN,
                BACK,
                UCN
        } fstate = BEGIN;

        ulint count = 0xbad, value = 0xbad;
        ptr<ucn_token> t;
        ptr<simple_location> loc;
        ucn_token_type utt;
        ucn u;
        
        // reserve space, the representation can only be shorter
        ucn_string us(len,0xbeef);
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        for (ucn_token_buffer::size_type i = 0; i < len; i++, ++bit) {
                t = *bit;
                utt = t->type_get();
                u = t->value_get();
                loc = t->location_get();
                switch (fstate) {
                        case BACK:
                                // expecting only ucn escape sequences
                                lassert(utt == ucn_token::TOK_BASIC && 
                                                (u == character::ascii_lower_u || u == character::ascii_upper_u));
                                count = u == character::ascii_lower_u ? 4 : 8;
                                value = 0;
                                fstate = UCN;
                        break;
                        case UCN:
                                if (utt == ucn_token::TOK_BASIC && character::is_xdigit(u)) {
                                        value = (value << 4) | (character::extract_xdigit(u) & 0x0f);
                                        if (--count == 0) {
                                                u = character::create_internal(value);
                                          
                                                if (!character::is_translated(u)) {
                                                        // ucn escape sequence value out of range
                                                        report << ucn_escape_value_invalid << lines->translate_location(loc);
                                                        // substitute for harmless
                                                        u = character::ascii_underscore;
                                                } else if (identifier && !character::is_translated_identifier(u)) {
                                                        // value out of range for identifier
                                                        report << ucn_escape_value_invalid_in_identifier << lines->translate_location(loc);
                                                        // keep the character, even if out of range
                                                }

                                                *sit = u;
                                                ++sit;
                                                fstate = BEGIN;
                                        }
                                        break;
                                } 
                                
                                // malformed ucn escape sequence
                                report << ucn_escape_insufficient_digits << lines->translate_location(loc);
                                
                                // substitute the character and terminate the sequence
                                *sit = character::ascii_underscore;
                                ++sit;
                                fstate = BEGIN;

                                // fall through
                        case BEGIN:
                                if (utt == ucn_token::TOK_BASIC) {
                                        if (u == character::ascii_backslash) {
                                                fstate = BACK;
                                                break;
                                        }
                                } else {
                                        lassert(utt == ucn_token::TOK_TRANSLATED);
                                        if (identifier && !character::is_translated_identifier(u)) {
                                                // value out of range for identifier
                                                report << ucn_escape_value_invalid_in_identifier << lines->translate_location(loc);
                                        }
                                }
                                *sit = u;
                                ++sit;
                                break;
                }
        }

        // shall not end with backslash
        lassert(fstate != BACK);

        if (fstate == UCN) {
                // malformed ucn escape sequence
                report << ucn_escape_insufficient_digits << lines->translate_location(loc);
                // substitute the character and terminate the sequence
                *sit = character::ascii_underscore;
                ++sit;
        }
        
        // erase removed items
        buffer->erase(buffer->begin(),bit);

        // use only filled part of the string
        return token_value::create(ucn_string(us.begin(),sit));
}

/*!
  Removes \a len items from the beginning of the buffer.
  Returns token value representing the removed items.
  Performs scanning for ucn escape sequences, issues errors for ucn invalid in identifiers.
  Useful for identifiers and preprocessing numbers containing escaped or translated ucn characters.
  \pre len <= length()
  \pre Only contains backslash in ucn escape sequences.
  \pre The contained ucn escape sequences are well-formed.
  \param len The length of the removed sequence.
  \param identifier  Flag set to true if extracting identifier.
  \return The representation of the items' values, with converted ucn escape sequences.
*/
ptr<token_value> ucn_token_buffer::extract_simple_ucn(ucn_token_buffer::size_type len, bool identifier)
{
        lassert(len <= length());
        
        // state of the function
        enum {
                BEGIN,
                BACK,
                UCN
        } fstate = BEGIN;

        ulint count = 0xbad, value = 0xbad;
        ptr<ucn_token> t;
        ptr<simple_location> loc;
        ucn_token_type utt;
        ucn u;
        
        // reserve space, the representation can only be shorter
        ucn_string us(len,0xbeef);
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        for (ucn_token_buffer::size_type i = 0; i < len; i++, ++bit) {
                t = *bit;
                loc = t->location_get();
                utt = t->type_get();
                u = t->value_get();

                switch (fstate) {
                        case BEGIN:
                                if (utt == ucn_token::TOK_BASIC) {
                                        if (u == character::ascii_backslash) {
                                                fstate = BACK;
                                                break;
                                        }
                                } else {
                                        lassert(utt == ucn_token::TOK_TRANSLATED);
                                        if (identifier && !character::is_translated_identifier(u)) {
                                                // value out of range for identifier
                                                report << ucn_escape_value_invalid_in_identifier << lines->translate_location(loc);
                                        }
                                }
                                *sit = u;
                                ++sit;
                                break;
                        case BACK:
                                // expecting only ucn escape sequences
                                lassert(utt == ucn_token::TOK_BASIC && 
                                                (u == character::ascii_lower_u || u == character::ascii_upper_u));
                                count = u == character::ascii_lower_u ? 4 : 8;
                                value = 0;
                                fstate = UCN;
                                break;
                        case UCN:
                                // expecting only hexa digits in the ucn escape sequence
                                lassert(utt == ucn_token::TOK_BASIC && character::is_xdigit(u));
                                value = (value << 4) | (character::extract_xdigit(u) & 0x0f);
                                if (--count == 0) {
                                        u = character::create_internal(value);
                                  
                                        if (!character::is_translated(u)) {
                                                // ucn escape sequence value out of range
                                                report << ucn_escape_value_invalid << lines->translate_location(loc);
                                                // substitute for harmless
                                                u = character::ascii_underscore;
                                        } else if (identifier && !character::is_translated_identifier(u)) {
                                                // value out of range for identifier
                                                report << ucn_escape_value_invalid_in_identifier << lines->translate_location(loc);
                                                // keep the character, even if out of range
                                        }
                                        *sit = u;
                                        ++sit;
                                        fstate = BEGIN;
                                }
                                break;
                }
        }

        // shall not be partial ucn escape sequence
        lassert(fstate == BEGIN);
        
        // erase removed items
        buffer->erase(buffer->begin(),bit);

        // use only filled part of the string
        return token_value::create(ucn_string(us.begin(),sit));
}

/*!
  Removes \a len items from the beginning of the buffer.
  Returns token value representing the removed items.
  Performs scanning for ucn escape sequences, issues errors for invalid ucn.
  Useful for well-formed string and character literals.
  \pre len <= length()
  \pre Only contains well-formed escape sequences.
  \pre The contained ucn escape sequences are well-formed.
  \param len The length of the removed sequence.
  \return The representation of the items' values, with converted ucn escape sequences.
*/
ptr<token_value> ucn_token_buffer::extract_ucn_literal(ucn_token_buffer::size_type len)
{
        lassert(len <= length());
        
        // state of the function
        enum {
                BEGIN,
                BACK,
                UCN
        } fstate = BEGIN;
        ulint count = 0xbad, value = 0xbad;
        ptr<ucn_token> t;
        ucn_token_type utt;
        ptr<simple_location> loc;
        ucn u;
        
        // reserve space, the representation can only be shorter
        ucn_string us(len,0xbeef);
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        for (ucn_token_buffer::size_type i = 0; i < len; i++, ++bit) {
                 t = *bit;
                 loc = t->location_get();
                 utt = t->type_get();
                 u = t->value_get();
                 
                 switch (fstate) {
                         case BEGIN:
                                 if (utt == ucn_token::TOK_BASIC && u == character::ascii_backslash) {
                                         fstate = BACK;
                                 } else {
                                        *sit = u;
                                        ++sit;
                                 }
                                 break;
                         case BACK:
                                 // expecting only basic characters after backslash
                                 lassert(utt == ucn_token::TOK_BASIC);
                                 if (u == character::ascii_lower_u || u == character::ascii_upper_u) {
                                        count = u == character::ascii_lower_u ? 4 : 8;
                                        value = 0;
                                        fstate = UCN;
                                 } else fstate = BEGIN;
                         break;
                         case UCN:
                                 // expecting only hexa digits in the ucn escape sequence
                                 lassert(utt == ucn_token::TOK_BASIC && character::is_xdigit(u));
                                 value = (value << 4) | (character::extract_xdigit(u) & 0x0f);
                                 if (--count == 0) {
                                         u = character::create_internal(value);
                                        
                                         if (!character::is_translated(u)) {
                                                 // ucn escape sequence value out of range
                                                 report << ucn_escape_value_invalid << lines->translate_location(loc);
                                                 // substitute for harmless
                                                 u = character::ascii_underscore;
                                         }
                                         
                                         *sit = u;
                                         ++sit;
                                         fstate = BEGIN;
                                 }
                                 break;
                 }
        }

        // shall not be partial escape sequence
        lassert(fstate == BEGIN);
        
        // erase removed items
        buffer->erase(buffer->begin(),bit);

        // use only filled part of the string
        return token_value::create(ucn_string(us.begin(),sit));
}

/*!
  Removes \a len items from the beginning of the buffer.
  Returns token value representing the removed items.
  Performs scanning for escape sequences, issues errors for invalid.
  Used for character and string literals with ill-formed escape sequences.
  \pre len <= length()
  \pre The sequence does not end with backslash.
  \param len The length of the removed sequence.
  \return The representation of the items' values, with converted ucn escape sequences.
*/
ptr<token_value> ucn_token_buffer::extract_bad_literal(ucn_token_buffer::size_type len)
{
        lassert(len <= length());
        
        // state of the function
        enum {
                BEGIN,
                PASS,
                BACK,
                UCN,
                OCT,
                HEX
        } fstate = BEGIN;
        
        ulint count = 0xbad, value = 0xbad;
        ptr<ucn_token> t;
        ptr<simple_location> loc;
        ucn_token_type utt = 0xbad;
        ucn u = 0xbad;
        
        // reserve space
        ucn_string us(len,0xbeef);
        
        buffer_type::iterator bit = buffer->begin();
        ucn_string::iterator sit = us.begin();
        ucn_token_buffer::size_type i = 0;
        while (true) {

                if (fstate == PASS) {
                        fstate = BEGIN;
                } else if (i < len) {
                        t = *bit;
                        ++bit;
                        ++i;
                        loc = t->location_get();
                        utt = t->type_get();
                        u = t->value_get();
                } else break;

                switch (fstate) {
                        case BEGIN:
                                if (utt == ucn_token::TOK_BASIC && u == character::ascii_backslash) {
                                        fstate = BACK;
                                } else {
                                  *sit = u;
                                  ++sit;
                                }
                                break;
                        case BACK:
                                if (utt == ucn_token::TOK_BASIC) {
                                        switch (u) {
                                                case character::ascii_lower_u:
                                                        count = 4;
                                                        fstate = UCN;
                                                        break;
                                                case character::ascii_upper_u:
                                                        count = 8;
                                                        fstate = UCN;
                                                        break;
                                                case character::ascii_lower_x:
                                                        count = 1;
                                                        fstate = HEX;
                                                        break;
                                                case character::ascii_quote:
                                                case character::ascii_dquote:
                                                case character::ascii_qmark:
                                                case character::ascii_backslash:
                                                case character::ascii_lower_a:
                                                case character::ascii_lower_b:
                                                case character::ascii_lower_f:
                                                case character::ascii_lower_n:
                                                case character::ascii_lower_r:
                                                case character::ascii_lower_t:
                                                case character::ascii_lower_v:
                                                        *sit = character::ascii_backslash;
                                                        ++sit;
                                                        *sit = u;
                                                        ++sit;
                                                        fstate = BEGIN;
                                                        break;
                                                default:
                                                  if (character::is_odigit(u)) {
                                                          *sit = character::ascii_backslash;
                                                          ++sit;
                                                          *sit = u;
                                                          ++sit;
                                                          count = 2;
                                                          fstate = OCT;
                                                  } else {
                                                          // unknown escape sequence
                                                          report << invalid_escape_sequence << lines->translate_location(loc);
                                                          *sit = character::ascii_underscore;
                                                          ++sit;
                                                          fstate = BEGIN;
                                                  }
                                        }
                                } else {
                                        // bad character after backslash
                                        report << invalid_escape_sequence << lines->translate_location(loc);
                                        *sit = character::ascii_underscore;
                                        ++sit;
                                        fstate = BEGIN;
                                }
                                break;
                        case UCN:
                                // expecting only hexa digits in the ucn escape sequence
                                if (utt != ucn_token::TOK_BASIC || !character::is_xdigit(u)) {
                                        // bad ucn escape sequence
                                        report << ucn_escape_insufficient_digits << lines->translate_location(loc);
                                        *sit = character::ascii_underscore;
                                        ++sit;
                                        // keep the character for parsing
                                        fstate = PASS;
                                        break;
                                }

                                value = (value << 4) | (character::extract_xdigit(u) & 0x0f);
                                if (--count == 0) {
                                        u = character::create_internal(value);
                                        if (!character::is_translated(u)) {
                                                // disallowed ucn range
                                                report << ucn_escape_value_invalid << lines->translate_location(loc);
                                                *sit = character::ascii_underscore;
                                                ++sit;
                                        } else {
                                                *sit = u;
                                                ++sit;
                                        }
                                        fstate = BEGIN;
                                }
                                break;
                        case OCT:
                                if (utt == ucn_token::TOK_BASIC && character::is_odigit(u)) {
                                        *sit = u;
                                        ++sit;
                                        if (--count == 0) fstate = BEGIN;
                                } else fstate = PASS;
                                break;
                        case HEX:
                                if (utt == ucn_token::TOK_BASIC && character::is_xdigit(u)) {
                                        if (count) {
                                                count = 0;
                                                *sit = character::ascii_backslash;
                                                ++sit;
                                                *sit = character::ascii_lower_x;
                                                ++sit;
                                        }
                                        *sit = u;
                                        ++sit;
                                } else {
                                        if (count) {
                                                // \x with no xdigits
                                                report << missing_hexadecimal_digits << lines->translate_location(loc);
                                                *sit = character::ascii_underscore;
                                                ++sit;
                                        }
                                        // keep the character for parsing
                                        fstate = PASS;
                                }
                                break;
                        default:
                                lassert2(false,"You should never get here");
                }
        }

        switch (fstate) {
                case BEGIN:
                case OCT:
                        break;
                case HEX:
                        if (count) {
                                // \x with no xdigits
                                report << missing_hexadecimal_digits << lines->translate_location(loc);
                                *sit = character::ascii_underscore;
                                ++sit;
                        }
                        break;
                case UCN:
                        // unterminated \uU
                        report << ucn_escape_insufficient_digits << lines->translate_location(loc);
                        *sit = character::ascii_underscore;
                        ++sit;
                        break;
                default:
                        lassert2(false,"You should never get here");
        }
        
        // erase removed items
        buffer->erase(buffer->begin(),bit);

        // use only filled part of the string
        return token_value::create(ucn_string(us.begin(),sit));
}

/*!
  Returns length of the buffer.
  \return  The current length of the buffer.
*/
ucn_token_buffer::size_type ucn_token_buffer::length(void) const
{
        return buffer->size();
}

/*!
  Marks the object.
*/
void ucn_token_buffer::gc_mark(void)
{
        buffer.gc_mark();
        lines.gc_mark();
}

/*!
  Retruns new empty ucn token buffer.
  \post length() == 0
  \param a_lines  The associated line control.
  \return The new ucn token buffer.
*/
ptr<ucn_token_buffer> ucn_token_buffer::create(const ptr<line_control> &a_lines)
{
        return new ucn_token_buffer(a_lines);
}

end_package(lex);
end_package(cplus);
end_package(lang);
end_package(lestes);
/* vim: set ft=lestes : */

---