lestes: lestes::lang::cplus::lex::token

typedef ::lestes::std::list< srp<pp_token> > lestes::lang::cplus::lex::token_stream::sequence_type [private]

The type of sequence container.

Definition at line 97 of file token_stream.hh.

lestes::lang::cplus::lex::token_stream::token_stream ( const ptr< pp_filter > & a_input ) [protected]

Creates stream.

Constructs new stream.

Precondition:: a_input != NULL

Parameters:

a_input

The input to read from.

Definition at line 60 of file token_stream.cc.

Referenced by create().

00060                                                        :
00061         token_input(checked(a_input)),
00062         file_end(NULL),
00063         sequence(sequence_type::create())
00064 {
00065 }

ptr< pp_token > lestes::lang::cplus::lex::token_stream::read ( void ) [virtual]

Reads first token.

Reads first token, does no transformation.

Returns:: The first token in the sequence, or TOK_TERMINATOR.

Implements lestes::lang::cplus::lex::pp_filter.

Definition at line 103 of file token_stream.cc.

References load_input(), sequence, and lestes::lang::cplus::lex::pp_token::terminator().

00104 {
00105         if (!load_input()) return pp_token::terminator();
00106         ptr<pp_token> t = sequence->front();
00107         sequence->pop_front();
00108         return t;
00109 }

ptr< pp_token > lestes::lang::cplus::lex::token_stream::peek_front ( void ) [virtual]

Returns front token.

Returns front token, which is kept in the sequence.

Returns:: The first token in the sequence, or TOK_TERMINATOR.

Implements lestes::lang::cplus::lex::token_input.

Definition at line 115 of file token_stream.cc.

References load_input(), sequence, and lestes::lang::cplus::lex::pp_token::terminator().

Referenced by expand_line().

00116 {
00117         if (!load_input()) return pp_token::terminator();
00118         return sequence->front();
00119 }

ptr< pp_token > lestes::lang::cplus::lex::token_stream::read_front ( void ) [virtual]

Reads front token, squeezing whitespace.

Reads front token, squeezing multiple blanks into one.

Returns:: The first token in the sequence.

Implements lestes::lang::cplus::lex::token_input.

Definition at line 125 of file token_stream.cc.

References load_input(), sequence, lestes::lang::cplus::lex::pp_token::terminator(), lestes::lang::cplus::lex::pp_token::TOK_BLANK, and u.

Referenced by expand_line().

00126 {
00127         if (!load_input()) return pp_token::terminator();
00128         
00129         ptr<pp_token> t = sequence->front();
00130         sequence->pop_front();
00131 
00132         sequence_type::size_type len = sequence->size();
00133         
00134         if (t->type_get() == pp_token::TOK_BLANK) {
00135                 ptr<pp_token> u;
00136                 while (len--) {
00137                         u = sequence->front();
00138                         if (u->type_get() != pp_token::TOK_BLANK) break;
00139                         sequence->pop_front();
00140                 }
00141         }
00142         
00143         return t;
00144 }

ptr< pp_token > lestes::lang::cplus::lex::token_stream::read_front_skip_ws ( void ) [virtual]

Reads front token, skipping front whitespace, but not newline.

Returns:: The first non-whitespace token in the sequence.

Implements lestes::lang::cplus::lex::token_input.

Definition at line 150 of file token_stream.cc.

References load_input(), sequence, lestes::lang::cplus::lex::pp_token::terminator(), and lestes::lang::cplus::lex::pp_token::TOK_BLANK.

00151 {
00152         if (!load_input()) return pp_token::terminator();
00153         
00154         ptr<pp_token> t;
00155         
00156         do {   
00157                 t = sequence->front();
00158                 sequence->pop_front();
00159         } while (t->type_get() == pp_token::TOK_BLANK);
00160         
00161         return t;
00162 }

bool lestes::lang::cplus::lex::token_stream::skip_front_ws ( void ) [virtual]

Skips front whitespace.

Returns:: true If there are some tokens left in the sequence.

Implements lestes::lang::cplus::lex::token_input.

Definition at line 168 of file token_stream.cc.

References load_input(), sequence, and lestes::lang::cplus::lex::pp_token::TOK_BLANK.

00169 {
00170         if (!load_input()) return false;
00171 
00172         ptr<pp_token> t;
00173         
00174         while (sequence->front()->type_get() == pp_token::TOK_BLANK) {
00175                 sequence->pop_front();
00176         }
00177         
00178         // sequence contains at least TOK_LINE_END
00179         return true;
00180 }

ptr< token_sequence > lestes::lang::cplus::lex::token_stream::read_line ( void )

Reads next line in input.

Reads line from the stream until TOK_LINE_END. At the end of stream returns the TOK_FILE_END.

Returns:: The nonexpanded tokens including the TOK_LINE_END, or sequence containing TOK_FILE_END.

Definition at line 186 of file token_stream.cc.

References lestes::lang::cplus::lex::token_sequence::create(), file_end, load_input(), and sequence.

00187 {
00188         ptr<token_sequence> result = token_sequence::create();
00189         if (!load_input()) {
00190                 result->add_back(file_end);
00191                 return result;
00192         }
00193         // TODO pt this is ugly, uses friend, could be done better
00194         // append the buffer to the result
00195         result->sequence->splice(result->sequence->end(),*sequence);
00196         return result;
00197 }

ptr< token_sequence > lestes::lang::cplus::lex::token_stream::expand_line ( const ptr< macro_storage > & macros )

Expands next line in input.

Expands macros within sequence till newline. The first line, i.e. the front part ending with TOK_LINE_END, not counting TOK_LINE_END inside macro calls, is removed from the sequence.

Precondition:: macros != NULL

Parameters:

macros

The macros for expansion.

Returns:: The expanded tokens including the TOK_LINE_END, or empty sequence.

Definition at line 207 of file token_stream.cc.

References lestes::lang::cplus::lex::token_sequence::create(), load_input(), peek_front(), prepend(), read_front(), lestes::lang::cplus::lex::pp_token::TOK_LEFT_PAR, and lestes::lang::cplus::lex::pp_token::TOK_LINE_END.

00208 {
00209         ptr<token_sequence> result = token_sequence::create();
00210 
00211         // return empty sequence
00212         if (!load_input()) return result;
00213 
00214         ptr<pp_token> t;
00215         // TODO pt copy to token_sequence
00216         ptr<macro> mac;
00217         ptr<token_sequence> expanded;
00218         
00219         do {
00220                 t = read_front();
00221 
00222                 // check whether the token is defined as macro
00223                 // whether the macro is allowed in the context of the token
00224                 // and whether a funlike macro is called with '('
00225                 if (t->is_name() &&
00226                          (mac = macros->lookup(t->value_get())) &&
00227                          !t->taboo_get()->contains(mac) &&
00228                          (!mac->funlike_get() || peek_front()->type_get() == pp_token::TOK_LEFT_PAR) &&
00229                          (expanded = mac->expand(t,this,macros))) {
00230                         
00231                         // TODO pt move into expand
00232                         expanded->taboo_extend(t->taboo_get()->extend(mac),macros);
00233 
00234                         // prepend in O(1)
00235                         prepend(expanded);
00236                 } else {
00237                         result->add_back(t);
00238                 }
00239         } while (t->type_get() != pp_token::TOK_LINE_END);
00240         return result;
00241 }

void lestes::lang::cplus::lex::token_stream::debug_print ( ::std::ostream & o )

Prints debug dump.

ptr< token_stream > lestes::lang::cplus::lex::token_stream::create ( const ptr< pp_filter > & a_input ) [static]

Returns stream with input.

Returns a token stream connected to the a_input.

Precondition:: a_input != NULL

Parameters:

a_input

The input to read from.

Returns:: New token stream.

Definition at line 271 of file token_stream.cc.

References token_stream().

00272 {
00273         return new token_stream(a_input);
00274 }

void lestes::lang::cplus::lex::token_stream::prepend ( const ptr< token_sequence > & inserted ) [protected]

Prepends a sequence.

Inserts another token sequence before the beginning of the sequence.

Precondition:: inserted != NULL

Postcondition:: inserted->length() == 0

Parameters:

inserted

The sequence to insert, the content is moved into new place.

Definition at line 249 of file token_stream.cc.

References lassert, and sequence.

Referenced by expand_line().

00250 {
00251         lassert(inserted);
00252         sequence->splice(sequence->begin(),*(inserted->sequence));
00253 }

void lestes::lang::cplus::lex::token_stream::gc_mark ( void ) [protected, virtual]

Marks the object.

Reimplemented from lestes::lang::cplus::lex::pp_filter.

Definition at line 258 of file token_stream.cc.

References file_end, lestes::lang::cplus::lex::pp_filter::gc_mark(), and sequence.

00259 {
00260         file_end.gc_mark();
00261         sequence.gc_mark();
00262         token_input::gc_mark();
00263 }

bool lestes::lang::cplus::lex::token_stream::load_input ( void ) [private]

Loads next line.

Loads next line delimited by pp_token::TOK_LINE_END from input into internal buffer. Expects pp_token::TOK_FILE_END to be returned after the last pp_token::TOK_LINE_END.

Returns:: true If before end of stream.

Definition at line 72 of file token_stream.cc.

References file_end, lestes::lang::cplus::lex::pp_filter::input_read(), sequence, lestes::lang::cplus::lex::pp_token::TOK_FILE_END, and lestes::lang::cplus::lex::pp_token::TOK_LINE_END.

Referenced by expand_line(), peek_front(), read(), read_front(), read_front_skip_ws(), read_line(), and skip_front_ws().

00073 {
00074         // the TOK_FILE_END was read
00075         if (file_end) return false;
00076 
00077         // still something in buffer
00078         if (sequence->size() != 0) return true;
00079         
00080         ptr<pp_token> tok = input_read();
00081         
00082         // TOK_FILE_END can come only at the beginning of line
00083         if (tok->type_get() == pp_token::TOK_FILE_END) {
00084                 file_end = tok;
00085                 return false;
00086         }
00087         
00088         sequence->push_back(tok);
00089         
00090         // fill the line buffer
00091         while (tok->type_get() != pp_token::TOK_LINE_END) {
00092                 tok = input_read();
00093 
00094                 sequence->push_back(tok);
00095         }
00096         return true;
00097 }

srp<pp_token> lestes::lang::cplus::lex::token_stream::file_end [private]

The eof token.

Definition at line 105 of file token_stream.hh.

Referenced by gc_mark(), load_input(), and read_line().

srp<sequence_type> lestes::lang::cplus::lex::token_stream::sequence [private]

The sequence container.

Definition at line 107 of file token_stream.hh.

Referenced by gc_mark(), load_input(), peek_front(), prepend(), read(), read_front(), read_front_skip_ws(), read_line(), and skip_front_ws().

lestes::lang::cplus::lex::token_stream Class Reference

Public Member Functions

Static Public Member Functions

Protected Member Functions

Private Types

Private Member Functions

Private Attributes

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation


Public Member Functions
ptr< pp_token >	read (void)
	Reads first token.
ptr< pp_token >	peek_front (void)
	Returns front token.
ptr< pp_token >	read_front (void)
	Reads front token, squeezing whitespace.
ptr< pp_token >	read_front_skip_ws (void)
	Reads front token, skipping front whitespace, but not newline.
bool	skip_front_ws (void)
	Skips front whitespace.
ptr< token_sequence >	read_line (void)
	Reads next line in input.
ptr< token_sequence >	expand_line (const ptr< macro_storage > &macros)
	Expands next line in input.
void	debug_print (::std::ostream &o)
	Prints debug dump.
Static Public Member Functions
static ptr< token_stream >	create (const ptr< pp_filter > &a_input)
	Returns stream with input.
Protected Member Functions
	token_stream (const ptr< pp_filter > &a_input)
	Creates stream.
void	prepend (const ptr< token_sequence > &ts)
	Prepends a sequence.
virtual void	gc_mark (void)
	Marks the object.
Private Types
typedef ::lestes::std::list< srp< pp_token > >	sequence_type
	The type of sequence container.
Private Member Functions
bool	load_input (void)
	Loads next line.
Private Attributes
srp< pp_token >	file_end
	The eof token.
srp< sequence_type >	sequence
	The sequence container.