token_stream.cc

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
/*! \file
  \brief Stream of tokens.

        Definition of token_stream class representing stream of tokens.
        \author pt
*/
#include <lestes/common.hh>
#include <lestes/std/source_location.hh>
#include <lestes/lang/cplus/lex/token_stream.hh>
#include <lestes/lang/cplus/lex/token_sequence.hh>
#include <lestes/lang/cplus/lex/pp_token.hh>
#include <lestes/lang/cplus/lex/pp_filter.hh>
#include <lestes/lang/cplus/lex/taboo_macros.hh>
#include <lestes/lang/cplus/lex/macro.hh>
#include <lestes/lang/cplus/lex/macro_storage.hh>
#include <lestes/lang/cplus/lex/token_value.hh>

#include <iterator>
#include <iostream>

package(lestes);
package(lang);
package(cplus);
package(lex);

using namespace ::std;

/*!
  Constructs new stream.
  \pre a_input != NULL
  \param a_input  The input to read from.
*/
token_stream::token_stream(const ptr<pp_filter> &a_input):
        token_input(checked(a_input)),
        file_end(NULL),
        sequence(sequence_type::create())
{
}

/*!
  Loads next line delimited by pp_token::TOK_LINE_END from input into internal buffer.
  Expects pp_token::TOK_FILE_END to be returned after the last pp_token::TOK_LINE_END.
  \return true  If before end of stream.
*/
bool token_stream::load_input(void)
{
        // the TOK_FILE_END was read
        if (file_end) return false;

        // still something in buffer
        if (sequence->size() != 0) return true;
        
        ptr<pp_token> tok = input_read();
        
        // TOK_FILE_END can come only at the beginning of line
        if (tok->type_get() == pp_token::TOK_FILE_END) {
                file_end = tok;
                return false;
        }
        
        sequence->push_back(tok);
        
        // fill the line buffer
        while (tok->type_get() != pp_token::TOK_LINE_END) {
                tok = input_read();

                sequence->push_back(tok);
        }
        return true;
}

/*!
  Reads first token, does no transformation.
  \return  The first token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_stream::read(void)
{
        if (!load_input()) return pp_token::terminator();
        ptr<pp_token> t = sequence->front();
        sequence->pop_front();
        return t;
}
        
/*!
  Returns front token, which is kept in the sequence.
  \return  The first token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_stream::peek_front(void)
{
        if (!load_input()) return pp_token::terminator();
        return sequence->front();
}
        
/*!
  Reads front token, squeezing multiple blanks into one.
  \return  The first token in the sequence.
*/
ptr<pp_token> token_stream::read_front(void)
{
        if (!load_input()) return pp_token::terminator();
        
        ptr<pp_token> t = sequence->front();
        sequence->pop_front();

        sequence_type::size_type len = sequence->size();
        
        if (t->type_get() == pp_token::TOK_BLANK) {
                ptr<pp_token> u;
                while (len--) {
                        u = sequence->front();
                        if (u->type_get() != pp_token::TOK_BLANK) break;
                        sequence->pop_front();
                }
        }
        
        return t;
}

/*!
  Reads front token, skipping front whitespace, but not newline.
  \return  The first non-whitespace token in the sequence.
*/
ptr<pp_token> token_stream::read_front_skip_ws(void)
{
        if (!load_input()) return pp_token::terminator();
        
        ptr<pp_token> t;
        
        do {   
                t = sequence->front();
                sequence->pop_front();
        } while (t->type_get() == pp_token::TOK_BLANK);
        
        return t;
}

/*!
  Skips front whitespace.
  \return true  If there are some tokens left in the sequence.
*/
bool token_stream::skip_front_ws(void)
{
        if (!load_input()) return false;

        ptr<pp_token> t;
        
        while (sequence->front()->type_get() == pp_token::TOK_BLANK) {
                sequence->pop_front();
        }
        
        // sequence contains at least TOK_LINE_END
        return true;
}

/*!
  Reads line from the stream until TOK_LINE_END. At the end of stream returns the TOK_FILE_END.
  \return  The nonexpanded tokens including the TOK_LINE_END, or sequence containing TOK_FILE_END.
*/
ptr<token_sequence> token_stream::read_line(void)
{
        ptr<token_sequence> result = token_sequence::create();
        if (!load_input()) {
                result->add_back(file_end);
                return result;
        }
        // TODO pt this is ugly, uses friend, could be done better
        // append the buffer to the result
        result->sequence->splice(result->sequence->end(),*sequence);
        return result;
}

/*!
  Expands macros within sequence till newline. The first line, i.e. the front part
  ending with TOK_LINE_END, not counting TOK_LINE_END inside macro
  calls, is removed from the sequence.
  \pre macros != NULL
  \param macros  The macros for expansion.
  \return The expanded tokens including the TOK_LINE_END, or empty sequence.
*/
ptr<token_sequence> token_stream::expand_line(const ptr<macro_storage> &macros)
{
        ptr<token_sequence> result = token_sequence::create();

        // return empty sequence
        if (!load_input()) return result;

        ptr<pp_token> t;
        // TODO pt copy to token_sequence
        ptr<macro> mac;
        ptr<token_sequence> expanded;
        
        do {
                t = read_front();

                // check whether the token is defined as macro
                // whether the macro is allowed in the context of the token
                // and whether a funlike macro is called with '('
                if (t->is_name() &&
                         (mac = macros->lookup(t->value_get())) &&
                         !t->taboo_get()->contains(mac) &&
                         (!mac->funlike_get() || peek_front()->type_get() == pp_token::TOK_LEFT_PAR) &&
                         (expanded = mac->expand(t,this,macros))) {
                        
                        // TODO pt move into expand
                        expanded->taboo_extend(t->taboo_get()->extend(mac),macros);

                        // prepend in O(1)
                        prepend(expanded);
                } else {
                        result->add_back(t);
                }
        } while (t->type_get() != pp_token::TOK_LINE_END);
        return result;
}

/*!
  Inserts another token sequence before the beginning of the sequence.
  \pre inserted != NULL
  \post inserted->length() == 0
  \param inserted  The sequence to insert, the content is moved into new place.
*/
void token_stream::prepend(const ptr<token_sequence> &inserted)
{
        lassert(inserted);
        sequence->splice(sequence->begin(),*(inserted->sequence));
}

/*!
  Marks the object.
*/
void token_stream::gc_mark(void)
{
        file_end.gc_mark();
        sequence.gc_mark();
        token_input::gc_mark();
}

/*!
  Returns a token stream connected to the \a a_input.
  \pre a_input != NULL
  \param a_input  The input to read from.
  \return  New token stream.
*/
ptr<token_stream> token_stream::create(const ptr<pp_filter> &a_input)
{
        return new token_stream(a_input);
}

// TODO pt remove
/*!
  Prints debug dump of content to stream.
  \param o  The stream to print to.
*/
void token_stream::debug_print(ostream &o)
{
        bool next = false;

        for (sequence_type::iterator it = sequence->begin(), end = sequence->end();
                  it != end; ++it) {
                if (next) o << ' '; else next = true;
                o << (*it)->spelling_get();
        }
}

/*!
  Prints debug dump of token sequence to stream.
  \param o  The stream to print to.
  \param ts  The token sequence to print.
*/
::std::ostream &operator<<(::std::ostream &o, const ptr<token_stream> &ts) {
        ts->debug_print(o);
        return o;
}

end_package(lex);
end_package(cplus);
end_package(lang);
end_package(lestes);
/* vim: set ft=lestes : */

---