evaluator.cc

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
/*! \file
  \brief Directives evaluator.

  Definition of evaluator class representing evaluator of preprocessing directives.
  \author pt
*/
#include <lestes/common.hh>
#include <lestes/lang/cplus/lex/evaluator.hh>
#include <lestes/lang/cplus/lex/evaluator.m.hh>
#include <lestes/lang/cplus/lex/pp_token.hh>
#include <lestes/lang/cplus/lex/expander.hh>
#include <lestes/lang/cplus/lex/token_sequence.hh>
#include <lestes/lang/cplus/lex/unit_part.hh>
#include <lestes/lang/cplus/lex/condition_stack.hh>
#include <lestes/lang/cplus/lex/macro_storage.hh>
#include <lestes/lang/cplus/lex/macro.hh>
#include <lestes/lang/cplus/lex/token_value.hh>
#include <lestes/lang/cplus/lex/file_system.hh>
#include <lestes/lang/cplus/lex/named_istream.hh>
#include <lestes/lang/cplus/lex/evaluator.m.hh>
#include <lestes/lang/cplus/lex/stream_source.hh>
#include <lestes/lang/cplus/lex/encoder_ascii7.hh>
#include <lestes/lang/cplus/lex/line_control.hh>
#include <lestes/std/istream_wrapper.hh>
#include <lestes/std/source_location.hh>
#include <lestes/std/file_info.hh>

package(lestes);
package(lang);
package(cplus);
package(lex);

using namespace ::std;

/*!
  Creates new evaluator, try to start processing given file.
  The state is set to EMPTY when the file could not be opened, or START.
  \pre a_fs != NULL
  \param a_fs  The file system binding.
  \param file_name  The name of the file to process.
*/
evaluator::evaluator(const ptr<file_system> &a_fs, const lstring &file_name):
        fs(checked(a_fs)),
        state(EMPTY),
        active(false),
        macros(macro_storage::create()),
        includes(includes_type::create()),
        input(NULL),
        part(NULL),
        buffer(NULL),
        pragma_flag(false),
        nesting(0)
{
        ptr<named_istream> ns = fs->open_file(file_name);
        if (ns) {
                push_stream(ns,NULL);
        } else {
                report << unable_to_open_file << file_name << 
                        source_location::create(file_info::create("<command line>",NULL),1,1);
        }
}

/*!
  Returns the pragma flag.
  \return The pragma flag.
*/
bool evaluator::pragma_flag_get(void) const
{
        return pragma_flag;
}

/*!
  Pushes a new stream to be processed as a part of a translation unit.
  \pre ns != NULL
  \param ns  The named stream to process.
  \param loc  The location of inclusion or NULL for toplevel stream.
*/
void evaluator::push_stream(const ptr<named_istream> &ns, const ptr<source_location> &loc)
{
        lassert(ns);

        ptr<file_info> fi = file_info::create(ns->print_name_get(),loc);
        ptr<stream_source> ss = stream_source::create(ns->stream_get());
        // TODO pt config
        ptr<encoder> enc = encoder_ascii7::create();

        part = unit_part::create(fi,ss,enc,macros);
        state = START;
        active = true;
        input = part->expander_get();
        includes->push_back(part);
        ++nesting;
}

/*!
  Pops a processed stream, returning to the following in the stack.
  \pre state != EMPTY
*/
void evaluator::pop_stream(void)
{
        lassert(state != EMPTY);

        includes->pop_back();
        --nesting;

        if (includes->size() != 0) {
                part = includes->back();
                input = part->expander_get();
                state = START;
        } else {
                input = NULL;
                part = NULL;
                buffer = NULL;
                state = EMPTY;
                macros = NULL;
        }
}

/*!
  Checks for extra tokens in buffer before the end of line.
  Used for issuing error for trailing garbage in directives.
  \pre  The newline is still in buffer.
  \param name  The name of the checked directive.
*/
void evaluator::check_extra_tokens(const lstring &name)
{
        ptr<pp_token> tok = buffer->read_front_skip_ws();
        lassert(tok->type_get() != pp_token::TOK_TERMINATOR);
        if (tok->type_get() != pp_token::TOK_LINE_END) {
                // extra tokens at the end of directive
                report << trailing_tokens_in_directive << name << tok->location_get();
        }
}

/*!
  Parses buffer as file name part of an include directive.
  Returns the appropriate token, or NULL in case of failure.
  \return  The token representing the include directive or NULL.
*/
ptr<pp_token> evaluator::parse_include(void)
{
        buffer = buffer->expand_all(macros);

        ucn_string name;
        
        ptr<pp_token> t = buffer->read_front_skip_ws();
        pp_token_type ptt = t->type_get();
        // save the start location
        ptr<source_location> loc = t->location_get();
        if (ptt == pp_token::TOK_LT) {
                // get the first part
                t = buffer->read_front();
                ptt = t->type_get();
                while (ptt != pp_token::TOK_GT && ptt != pp_token::TOK_LINE_END) {
                        // take the name as literal concatenation
                        name += t->spelling_get();
                        t = buffer->read_front();
                        ptt = t->type_get();
                }
                // unterminated <> sequence
                if (ptt != pp_token::TOK_GT) return NULL;
                // can check here, because the newline was not read yet
                check_extra_tokens("#include");
                return pp_token::create(loc,pp_token::TOK_INCL_HCHAR,token_value::create(name));
                // this is the only possibility to put double quote into the third type include
        } else if (ptt == pp_token::TOK_STRING_LIT) {
                check_extra_tokens("#include");
                return pp_token::create(loc,pp_token::TOK_INCL_QCHAR,t->value_get());
        } 
        
        // else it has to be invalid
        return NULL;
}

/*!
  Processes preprocessor directive in buffer.
  \return  The next token for read().
*/
ptr<pp_token> evaluator::process_directive(void)
{
        // throw away hash
        buffer->read_front();

        ptr<pp_token> newline = buffer->peek_back();
        // get the directive
        ptr<pp_token> tok = buffer->read_front_skip_ws();
        ptr<source_location> loc = tok->location_get();
        pp_token_type directive = tok->type_get();
        bool old_active = active;
        bool done = true;
        bool first = false;
        bool result;

        ptr<condition_stack> conditions = part->conditions_get();

        switch (directive) {
                case pp_token::TOK_ELIF:
                        first = true;
                        // fall through
                case pp_token::TOK_IF:
                        lassert2(false,"#if and #elif are not supported");
                        // TODO pt call differently
                        result = true;
                        //result = expression::instance()->evaluate(buffer);
                        // result = expression::create(buffer)->eval();
                        if (conditions->process(first ? condition_stack::DIR_ELIF : condition_stack::DIR_IF,result,loc)) {
                                active = conditions->active_get();
                        }
                        break;
                case pp_token::TOK_IFNDEF:
                        first = true;
                        // fall through
                case pp_token::TOK_IFDEF:
                        tok = buffer->read_front_skip_ws();
                        if (tok->type_get() == pp_token::TOK_LINE_END) {
                                // expected macro name
                                report << directive_expects_macro_name << lstring(first ? "#ifndef" : "#ifdef") << loc;
                                break;
                        }

                        result = macros->defined(tok);
                        result = result != first;

                        check_extra_tokens(first ? "#ifndef" : "#ifdef");
                        
                        if (conditions->process(first ? condition_stack::DIR_IFNDEF : condition_stack::DIR_IFDEF,result,loc)) {
                                active = conditions->active_get();
                        }
                        break;
                case pp_token::TOK_ELSE:
                        first = true;
                        // fall through
                case pp_token::TOK_ENDIF:
                        check_extra_tokens(first ? "#else" : "#endif");
                        if (conditions->process(first ? condition_stack::DIR_ELSE : condition_stack::DIR_ENDIF,false,loc)) {
                                active = conditions->active_get();
                        }
                        break;
                default:
                        done = false;
                        break;
        }
        
        // either already processed, or ignored
        if (done || !old_active) return newline;
        
        switch (directive) {
                case pp_token::TOK_DEFINE:
                {
                        ptr<macro> mac = macro::create();
                        if (!mac->parse(buffer)) break;
                        macros->define(mac);
                        break;
                }
                case pp_token::TOK_UNDEF:
                        tok = buffer->read_front_skip_ws();

                        if (tok->type_get() == pp_token::TOK_LINE_END) {
                                // expected macro name
                                report << directive_expects_macro_name << lstring("#undef") << loc;
                                break;
                        }

                        macros->undef(tok);
                                
                        check_extra_tokens("#undef");

                        break;
                case pp_token::TOK_INCLUDE:
                {
                        if (nesting == NESTING_LIMIT) {
                                report << nesting_too_deep << nesting << loc;
                                break;
                        }

                        tok = buffer->read_front_skip_ws();
                        pp_token_type ptt = tok->type_get();

                        if (ptt == pp_token::TOK_INCL_SIG) {
                                // parse the include in buffer
                                tok = parse_include();
                                if (!tok) {
                                        ptt = pp_token::TOK_OTHER;
                                } else {
                                        ptt = tok->type_get();
                                }
                        }
                        
                        bool system = false;
                        
                        switch (ptt) {
                                case pp_token::TOK_INCL_HCHAR:
                                        system = true;
                                        // fall through
                                case pp_token::TOK_INCL_QCHAR:
                                {
                                        // TODO pt get the path from unit part
                                        ucn_string us(tok->value_get()->content_get());
                                        lstring ls;
                                        bool invalid = false;

                                        // convert the string to host character set
                                        for (ucn_string::iterator it = us.begin(), end = us.end();
                                                        it != end; ++it) {
                                                ucn u = *it;
                                                // only certain characters are supported
                                                if (!character::is_basic(u)) {
                                                        // error: unsupported character in include name
                                                        report << invalid_character_in_filename << tok->location_get();
                                                        invalid = true;
                                                        break;
                                                }
                                                ls += character::to_host(u);
                                        }

                                        if (invalid) break;
                                        
                                        // TODO pt path
                                        ptr<named_istream> ns = fs->find_file("",ls,system);

                                        if (ns) {
                                                push_stream(ns,loc);
                                        } else {
                                                // file not found or not accessible
                                                report << unable_to_open_file << ls << loc;
                                        }
                                        break;
                                }
                                default:
                                        // malformed include
                                        report << malformed_include << loc;
                                        break;
                        }
                        break;
                }
                case pp_token::TOK_LINE:
                {
                        // in any case, do the expansion, it can not spoil anything
                        buffer = buffer->expand_all(macros);

                        tok = buffer->read_front_skip_ws();
                        pp_token_type ptt = tok->type_get();

                        if (ptt != pp_token::TOK_NUMBER_LIT) {
                                report << line_expects_number << tok->location_get();
                                break;
                        }

                        // check digit sequence
                        ucn_string us(tok->value_get()->content_get());
                        ulint x = 0;
                        bool invalid = false;

                        // convert the string to host character set
                        for (ucn_string::iterator it = us.begin(), end = us.end();
                                        it != end; ++it) {
                                ucn u = *it;
                                // only decimal digit sequence less than 32768 is allowed
                                if (!character::is_digit(u) || (x = 10*x + character::extract_digit(u)) > 32767) {
                                        invalid = true;
                                        break;
                                }
                        }

                        if (invalid || x == 0) {
                                report << line_expects_number << tok->location_get();
                                break;
                        }

                        tok = buffer->read_front_skip_ws();
                        ptt = tok->type_get();

                        ptr<line_control> lic = part->line_control_get();

                        if (ptt == pp_token::TOK_STRING_LIT) {
                                // TODO make this a method, together with include
                                ucn_string us(tok->value_get()->content_get());
                                lstring ls;

                                // convert the string to host character set
                                for (ucn_string::iterator it = us.begin(), end = us.end();
                                                it != end; ++it) {
                                        ucn u = *it;
                                        // only certain characters are supported
                                        if (!character::is_basic(u)) {
                                                invalid = true;
                                                break;
                                        }
                                        ls += character::to_host(u);
                                }

                                if (invalid) {
                                        report << invalid_character_in_filename << tok->location_get();
                                        break;
                                }

                                lic->change_file(ls);

                                // in other cases this error is suppressed by other errors
                                check_extra_tokens("#line");
                        } else if (ptt != pp_token::TOK_LINE_END) {
                                // file name string literal expected
                                report << line_expects_string << tok->location_get();
                                break;
                        }

                        // change the next line number to x
                        lic->change_line(newline->location_get(),x);
                        break;
                }
                case pp_token::TOK_ERROR:
                        // TODO pt add content
                        report << user_error << loc;
                        break;
                case pp_token::TOK_PRAGMA:
                        // no pragmas supported except #pragma lestes
                        tok = buffer->read_front_skip_ws();

                        if (tok->type_get() == pp_token::TOK_IDENT) {
                                if (tok->value_get() == token_value::create("lestes")) {
                                        pragma_flag = true;
                                }
                        }
                                
                        break;
                case pp_token::TOK_LINE_END:
                        // empty directive
                        break;
                default:
                        // invalid directive
                        report << invalid_directve << tok->spelling_get() << loc;
                        break;
        }

        return newline;
}

/*!
  Reads next token from current unit part after evaluation of directives.
  \return The next token.
*/
ptr<pp_token> evaluator::read(void)
{
        if (state == EMPTY) 
                return pp_token::create(source_location::zero(),pp_token::TOK_FILE_END);
        
        ptr<pp_token> tok;
        
        do {
                switch (state) {
                        case START:
                                part->start_of_line();

                                switch (input->mode_get()) {
                                        case expander::DIRECTIVE:
                                                buffer = input->read_line();
                                                return process_directive();
                                        case expander::FILE_END:
                                                buffer = input->read_line();
                                                state = END;
                                                break;
                                        case expander::NORMAL:
                                                if (active) {
                                                        buffer = input->read_expanded();
                                                        state = BUFFER;
                                                } else {
                                                        buffer = input->read_line();
                                                        tok = buffer->peek_back();
                                                        lassert(tok->type_get() == pp_token::TOK_LINE_END);
                                                        // return the newline
                                                        return tok;
                                                }
                                                break;
                                        default:
                                                lassert2(false,"You should never get here");
                                }
                                
                                break;
                  case BUFFER:
                                tok = buffer->read_front();
                                if (buffer->length() == 0) state = START;
                                return tok;
                  case END:
                                {
                                        tok = buffer->read_front();
                                        lassert(tok->type_get() == pp_token::TOK_FILE_END);
                                        
                                        ptr<condition_stack> conditions = part->conditions_get();
                                        // check whether all conditions were closed
                                        conditions->process(condition_stack::DIR_EOF,false,tok->location_get());

                                        // sets state internally
                                        pop_stream();
                                }
                                break;
                  default:
                                lassert2(false,"You should never get here");
                                break;
                }
        } while (state != EMPTY);
        
        // return the last EOF token
        return tok;
}
/*!
  Marks the object.
*/
void evaluator::gc_mark(void)
{
        fs.gc_mark();
        macros.gc_mark();
        includes.gc_mark();
        input.gc_mark();
        part.gc_mark();
        buffer.gc_mark();
        pp_filter::gc_mark();
}


/*!
  Creates new evaluator, try to start processing given file.
  The state is set to EMPTY when the file could not be opened, or START.
  \pre a_fs != NULL
  \param a_fs  The file system binding.
  \param file_name  The name of the file to process.
  \return The evaluator.
*/
ptr<evaluator> evaluator::create(const ptr<file_system> &a_fs, const lstring &file_name)
{
        return new evaluator(a_fs,file_name);
}

end_package(lex);
end_package(cplus);
end_package(lang);
end_package(lestes);

/* vim: set ft=lestes : */

---