encoder_utf8.cc

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
/*! \file
  \brief UTF-8 encoder.

  Definition of encoder_utf8 class performing UTF-8 character set encoding.
  \author pt
*/
#include <lestes/common.hh>
#include <lestes/lang/cplus/lex/encoder_utf8.hh>
#include <lestes/lang/cplus/lex/encoder_utf8.m.hh>
#include <lestes/lang/cplus/lex/ucn_token.hh>
#include <lestes/lang/cplus/lex/encoder_utf8.m.hh>
#include <lestes/msg/message.hh>

package(lestes);
package(lang);
package(cplus);
package(lex);

using namespace ::std;

/*!
  Creates the object.
  \post bad == NULL
*/
encoder_utf8::encoder_utf8(void)
{
}

/*!
  Reads next token. Performs encoding from UTF-8 character set.
  Checks whether the input contains valid characters.
  If error is encountered, it returns token with type ucn_token::TOK_ERROR.
  \pre  Input into the filter is set.
  \return  The next token encoded from 7 bit ASCII.
  \return  Token with type ucn_token::TOK_ERROR if the source character is invalid.
  \return  Token with type ucn_token::TOK_EOF in case of previous error.
*/
ptr<ucn_token> encoder_utf8::read(void)
{
        // state of the reader
        typedef enum { 
                START, END, ERR, STORE, TWO_LAST, THREE_NEXT, THREE_LAST, FOUR_FIRST, FOUR_NEXT, FOUR_LAST 
        } state_type;

        ptr<ucn_token> t;
        ucn_token_type utt;
        state_type state;
        ulint v = 0xbad, u = 0xbad, x = 0xbad, y = 0xbad, z = 0xbad;

        state = START;

        do {
                t = input_read();
                utt = t->type_get();

                // for inherited errors
                if (utt == ucn_token::TOK_ERROR) {
                        // pass the error through
                        state = END;
                } else if (utt == ucn_token::TOK_EOF) {
                        if (state == START)
                                state = END;
                        else
                                state = ERR;
                } else {              
                        v = character::extract_value(t->value_get());
                        if (v > 0xFF) {
                                // TODO pt report error
                                state = ERR;
                        }
                        switch (state) {
                                case START:
                                        if ((v & 0x80) == 0) {
                                                // short path, value is already in t
                                                state = END;
                                        } else {
                                                if ((v & 0xE0) == 0xC0) {
                                                        // v <= 0xDF holds, implied
                                                        if (v >= 0xC2 ) {
                                                                u = 0;
                                                                z = 0;
                                                                y = v & 0x1F;
                                                                state = TWO_LAST;
                                                        } else {
                                                                // TODO error report
                                                                state = ERR;
                                                        }
                                                } else if ((v & 0xF0) == 0xE0) {
                                                        u = 0;
                                                        z = v & 0x0F;                     
                                                        state = THREE_NEXT;
                                                } else if ((v & 0xF8) == 0xF0) {
                                                        // v >= 0xF0 holds, implied
                                                        if (v <= 0xF4) {
                                                                u = v & 0x0F;
                                                                state = FOUR_FIRST;
                                                        } else {
                                                                // TODO error report
                                                                state = ERR;
                                                        }
                                                } else {
                                                                // TODO error report
                                                                state = ERR;
                                                }
                                        }
                                        break;
                                case TWO_LAST:
                                        if (v >= 0x80 && v <= 0xBF) {
                                                x = v & 0x3F;
                                                state = STORE;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                case THREE_NEXT:
                                        if ((z == 0x00 && v >= 0xA0 && v <= 0xBF) ||
                                                 (z >= 0x01 && z <= 0x0C && v >= 0x80 && v <= 0xBF) ||
                                                 (z == 0x0D && v >= 0x80 && v <= 0x9F) ||
                                                 (z >= 0x0E && z <= 0x0F && v >= 0x80 && v <= 0xBF)) {
                                                y = v & 0x3F;
                                                state = THREE_LAST;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                case THREE_LAST:
                                        if (v >= 0x80 && v <= 0xBF) {
                                                x = v & 0x3F;
                                                state = STORE;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                case FOUR_FIRST:
                                        if ((u == 0x00 && v >= 0x90 && v <= 0xBF) ||
                                                 (u >= 0x01 && u <= 0x03 && v >= 0x80 && v <= 0xBF) ||
                                                 (u == 0x04 && v >= 0x80 && v <= 0x8F)) {
                                                u = (u << 2) | ((v >> 4) & 0x03);
                                                z = v & 0x0F;
                                                state = FOUR_NEXT;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                case FOUR_NEXT:
                                        if (v >= 0x80 && v <= 0xBF) {
                                                y = v & 0x3F;
                                                state = FOUR_LAST;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                case FOUR_LAST:
                                        if (v >= 0x80 && v <= 0xBF) {
                                                x = v & 0x3F;
                                                state = STORE;
                                        } else {
                                                // TODO error report
                                                state = ERR;
                                        }
                                        break;
                                default:
                                        lassert2(false,"You should never get here");
                        }         
                }
                                
                if (state == STORE) {
                        t = t->clone_value((u << 16) | (z << 12) | (y << 6) | x);
                        state = END;
                } else if (state == ERR) {
                        t = ucn_token::create_error(invalid_utf_character->format());
                        state = END;
                }
        } while (state != END);
        // return the token (possibly error)
        return t;
}

/*!
  Marks the object.
*/
void encoder_utf8::gc_mark(void)
{
        encoder::gc_mark();
}

/*!
  Returns new instance of the encoder.
  \return  The new instance.
*/
ptr<encoder_utf8> encoder_utf8::create(void)
{
        return new encoder_utf8();
}

end_package(lex);
end_package(cplus);
end_package(lang);
end_package(lestes);

/* vim: set ft=lestes : */

---