token_sequence.cc

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
/*! \file
  \brief Token sequence.

        Definition of token_sequence class representing sequence of tokens.
        \author pt
*/
#include <lestes/common.hh>
#include <lestes/std/source_location.hh>
#include <lestes/std/file_info.hh>
#include <lestes/lang/cplus/lex/token_sequence.hh>
#include <lestes/lang/cplus/lex/pp_token.hh>
#include <lestes/lang/cplus/lex/pp_token.hh>
#include <lestes/lang/cplus/lex/taboo_macros.hh>
#include <lestes/lang/cplus/lex/macro.hh>
#include <lestes/lang/cplus/lex/macro_storage.hh>
#include <lestes/lang/cplus/lex/token_value.hh>

#include <iterator>
#include <iostream>

package(lestes);
package(lang);
package(cplus);
package(lex);

using namespace ::std;

/*!
  Constructs new empty sequence.
  \post length() == 0
*/
token_sequence::token_sequence(void):
        sequence(sequence_type::create())
{
}

/*!
  Reads front token, does no transformation.
  \return  The first token in the sequence.
*/
ptr<pp_token> token_sequence::read(void)
{
        if (length() == 0) return pp_token::terminator();
        ptr<pp_token> t = sequence->front();
        sequence->pop_front();
        return t;
}

/*!
  Returns front token, which is kept in the sequence.
  \return  The first token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_sequence::peek_front(void)
{
        if (length() != 0) return sequence->front();
        return pp_token::terminator();
}
        
/*!
  Returns back token, which is kept in the sequence.
  \return  The last token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_sequence::peek_back(void)
{
        if (length() == 0) return pp_token::terminator();
        return sequence->back();
}

/*!
  Reads front token, squeezing multiple blanks into one.
  \return  The first token in the sequence.
*/
ptr<pp_token> token_sequence::read_front(void)
{
        ulint len = length();
        if (!len) return pp_token::terminator();
        
        ptr<pp_token> t = sequence->front();
        sequence->pop_front();
        len--;
        
        if (t->type_get() == pp_token::TOK_BLANK) {
                ptr<pp_token> u;
                while (len--) {
                        u = sequence->front();
                        if (u->type_get() != pp_token::TOK_BLANK) break;
                        sequence->pop_front();
                }
        }
        
        return t;
}

/*!
  Reads front token, skipping front whitespace, but not newline.
  \return  The first non-whitespace token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_sequence::read_front_skip_ws(void)
{
        ptr<pp_token> t;
        ulint len = length();
        
        while (len--) {
                t = sequence->front();
                sequence->pop_front();
                if (t->type_get() != pp_token::TOK_BLANK)
                        return t;
        }
        
        return pp_token::terminator();
}

/*!
  Reads back token, squeezing blanks.
  \return The last token in the sequence, or TOK_TERMINATOR.
*/
ptr<pp_token> token_sequence::read_back(void)
{
        ulint len = length();
        
        if (!len) return pp_token::terminator();
        
        ptr<pp_token> t = sequence->back();
        sequence->pop_back();
        len--;
        
        if (t->type_get() == pp_token::TOK_BLANK) {
                ptr<pp_token> u;
                while (len--) {
                        u = sequence->back();
                        if (u->type_get() != pp_token::TOK_BLANK) break;
                        sequence->pop_back();
                }
        }
        
        return t;
}

/*!
  Reads back token, skipping back whitespace.
  \pre The sequence contains non-whitespace tokens.
  \return The last non-whitespace token in the sequence.
*/
ptr<pp_token> token_sequence::read_back_skip_ws(void)
{
        ptr<pp_token> t;
        ulint len = length();
        
        while (len--) {
                t = sequence->back();
                sequence->pop_back();
                if (t->type_get() != pp_token::TOK_BLANK)
                        return t;
        }
        
        return pp_token::terminator();
}

/*!
  Skips front whitespace.
  \return true  If there are some tokens left in the sequence.
*/
bool token_sequence::skip_front_ws(void)
{
        sequence_type::size_type len = sequence->size();
        while (len && sequence->front()->type_get() == pp_token::TOK_BLANK) {
                sequence->pop_front();
                len--;
        }
        return len;
}

/*!
  Skips back whitespace.
  \return true  If there are some tokens left in the sequence.
*/
bool token_sequence::skip_back_ws(void)
{
        sequence_type::size_type len = sequence->size();
        while (len && sequence->back()->type_get() == pp_token::TOK_BLANK) {
                sequence->pop_back();
                len--;
        }
        return len;
}

/*!
  Adds token to back of the sequence.
  \pre a_token != NULL
  \param a_token  The token to add.
*/
void token_sequence::add_back(const ptr<pp_token> &a_token)
{
        lassert(a_token);
        sequence->push_back(a_token);
}

/*!
  Expands macros within sequence. The sequence is discarded.
  \pre macros != NULL
  \post length() == 0
  \param macros  The macros for expansion.
  \return The expanded tokens in a new sequence.
*/
ptr<token_sequence> token_sequence::expand_all(const ptr<macro_storage> &macros)
{
        lassert(macros);

        ptr<token_sequence> result = token_sequence::create();

        if (!length()) return result;
        
        ptr<pp_token> t = sequence->front();
        pp_token_type ptt = t->type_get();

        ptr<macro> mac;
        ptr<token_sequence> expanded;
        
        while (length()) {
                t = sequence->front();
                sequence->pop_front();
                
                ptt = t->type_get();
                
                // check whether the token is defined as macro
                // whether the macro is allowed in the context of the token
                // and whether a funlike macro is called with '('
                if (t->is_name() &&
                         (mac = macros->lookup(t->value_get())) &&
                         !t->taboo_get()->contains(mac) &&
                         (!mac->funlike_get() || peek_front()->type_get() == pp_token::TOK_LEFT_PAR)) {

                        ptr<token_sequence> expanded = mac->expand(t,this,macros);

                        expanded->taboo_extend(t->taboo_get()->extend(mac),macros);

                        // prepend in O(1) to allow rescanning
                        prepend(expanded);
                } else {
                        result->add_back(t);
                }
        }
        return result;
}

/*!
  Extends taboo set for possibly expandable tokens in sequence by a macro.
  \pre a_macro != NULL
  \pre macros != NULL
  \param a_macro  The macro to add to the taboo set.
  \param macros  The defined macros.
*/
void token_sequence::taboo_extend(const ptr<taboo_macros> &tm, const ptr<macro_storage> &macros)
{
        lassert(tm);
        lassert(macros);

        ptr<pp_token> t;
        
        for (sequence_type::iterator it = sequence->begin(),
                  end = sequence->end(); it != end; ++it) {
                t = *it;
                
                // extend only currently defined macros
                if (t->is_name() &&
                         macros->lookup(t->value_get())) {
                        // extend taboo for token by mac
                        
                        *it = t->clone(tm);
                }
        }
}

/*!
  Returns copy of the sequence.
  \post is_equal(returned,this)
  \return New sequence with equal values.
*/
ptr<token_sequence> token_sequence::clone(void) const
{
        ptr<token_sequence> nju = token_sequence::create();
        copy(sequence->begin(),sequence->end(),back_inserter(*(nju->sequence)));
        return nju;
}

/*!
  Returns a copy of the sequence with new location.
  \pre a_location != NULL
  \param a_location  The location for the copied tokens.
  \return The copy of the sequence.
*/
ptr<token_sequence> token_sequence::clone(const ptr<source_location> &a_location) const
{
        lassert(a_location);
        
        ptr<pp_token> t;
        ptr<token_sequence> nju = token_sequence::create();

        sequence_type::iterator it = sequence->begin();
        sequence_type::iterator end = sequence->end();
        
        for (; it != end; ++it) {
                t = *it;
                nju->add_back(t->clone(a_location));
        }

        return nju;
}

/*!
  Inserts another token sequence before the beginning of the sequence.
  \pre inserted != NULL
  \post inserted->length() == 0
  \param inserted  The sequence to insert, the content is moved into new place.
*/
void token_sequence::prepend(const ptr<token_sequence> &inserted)
{
        lassert(inserted);
        sequence->splice(sequence->begin(),*(inserted->sequence));
}

/*!
  Inserts another token sequence after the end of the sequence.
  \pre inserted != NULL
  \post inserted->length() == 0
  \param inserted  The sequence to insert, the content is moved into new place.
*/
void token_sequence::append(const ptr<token_sequence> &inserted)
{
        lassert(inserted);
        sequence->splice(sequence->end(),*(inserted->sequence));
}

/*!
  Returns length of the sequence.
  \return The length of the sequence.
*/
token_sequence::size_type token_sequence::length(void) const
{
        return sequence->size();
}

/*!
  Returns empty token sequence.
  \return  Empty token sequence.
*/
ptr<token_sequence> token_sequence::create(void)
{
        return new token_sequence();
}

/*!
  Tests equality to other token sequence.
  \param other  The token sequence to compare with.
  \return true  If both sequences contain equal tokens.
*/
bool token_sequence::equals(const ptr<token_sequence> &other) const
{
        if (!other || length() != other->length()) return false;
        
        sequence_type::iterator it = sequence->begin();
        sequence_type::iterator end = sequence->end();

        sequence_type::iterator ot = other->sequence->begin();
        
        for (; it != end; ++it, ++ot) {
                ptr<pp_token> pt = *it;
                ptr<pp_token> qt = *ot;
                
                if (!is_equal(pt,qt)) return false;
        }
        
        return true;
}

/*!
  Tests whether sequences have the same type and spelling of tokens.
  \param other  The token sequence to compare with.
  \return true  If both sequences contain interchangeable tokens.
*/
bool token_sequence::congruent(const ptr<token_sequence> &other) const
{
        // TODO pt ::std::cerr << "token_sequence::resembles\n";
        // TODO pt debug_print( ::std::cerr); ::std::cerr << '\n';
        // TODO pt ::std::cerr << other << '\n';
        
        if (!other || length() != other->length()) return false;
        
        // TODO pt ::std::cerr << "length ok\n";
        
        sequence_type::iterator it = sequence->begin();
        sequence_type::iterator end = sequence->end();

        sequence_type::iterator ot = other->sequence->begin();
        
        for (; it != end; ++it, ++ot) {
                ptr<pp_token> pt = *it;
                ptr<pp_token> qt = *ot;
                
                if (!pt->congruent(qt)) return false;
                // TODO pt ::std::cerr << pt->description_get() << " is congruent to " << qt->description_get() << '\n';
        }
        
        return true;
}

/*!
  Marks the object.
*/
void token_sequence::gc_mark(void)
{
        sequence.gc_mark();
        token_input::gc_mark();
}

/*!
  Prints debug dump of content to stream.
  \param o  The stream to print to.
*/
void token_sequence::debug_print(ostream &o) const
{
        bool next = false;

        for (sequence_type::iterator it = sequence->begin(), end = sequence->end();
                  it != end; ++it) {
                if (next) o << ' '; else next = true;
                o << (*it)->spelling_get();
        }
}

/*!
  Prints debug dump of token sequence to stream.
  \param o  The stream to print to.
  \param ts  The token sequence to print.
*/
::std::ostream &operator<<(::std::ostream &o, const ptr<token_sequence> &ts) {
        ts->debug_print(o);
        return o;
}

end_package(lex);
end_package(cplus);
end_package(lang);
end_package(lestes);
/* vim: set ft=lestes : */

---