lestes: pp_lex.test.cc Source File

00001 /*
00002    The lestes compiler suite
00003    Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
00004    Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
00005    Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
00006    Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
00007    Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
00008    Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
00009    Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas
00010 
00011    This program is free software; you can redistribute it and/or modify
00012    it under the terms of the GNU General Public License as published by
00013    the Free Software Foundation; version 2 of the License.
00014 
00015    This program is distributed in the hope that it will be useful,
00016    but WITHOUT ANY WARRANTY; without even the implied warranty of
00017    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00018    GNU General Public License for more details.
00019 
00020    See the full text of the GNU General Public License version 2, and
00021    the limitations in the file doc/LICENSE.
00022 
00023    By accepting the license the licensee waives any and all claims
00024    against the copyright holder(s) related in whole or in part to the
00025    work, its use, and/or the inability to use it.
00026  
00027  */
00028 /*! \file
00029   Unit test for class pp_lex.
00030   \author pt
00031 */
00032 #include <lestes/common.hh>
00033 #include <lestes/std/file_info.hh>
00034 #include <lestes/std/source_location.hh>
00035 #include <lestes/lang/cplus/lex/pp_lex.hh>
00036 #include <lestes/lang/cplus/lex/pp_token.hh>
00037 #include <lestes/lang/cplus/lex/string_source.hh>
00038 #include <lestes/lang/cplus/lex/encoder_ascii7.hh>
00039 
00040 #include <string>
00041 #include <iostream>
00042 
00043 package(lestes);
00044 package(lang);
00045 package(cplus);
00046 package(lex);
00047 
00048 using namespace ::std;
00049 
00050 void pp_lex_test(void)
00051 {
00052         char inp[] = 
00053                 " [<:]:>}%>{<%#%:##%:%:)(;:?.+*%/^xor&bitand|bitor"
00054                 "~compl!not=<>,-::.*+=-=*=/=%=^=xor_eq&=and_eq|=or_eq<<>>"
00055                 ">>=<<===!=not_eq<=>=&&and||or++--->*->..."
00056                 "asm auto bool break case catch char class const_cast continue default delete do "
00057                 "double dynamic_cast else enum explicit export extern float for friend goto if "
00058                 "inline int long mutable namespace new operator private protected public register "
00059                 "reinterpret_cast return short signed sizeof static static_cast struct switch "
00060                 "template this throw try typedef typeid typename union unsigned using virtual "
00061                 "void volatile wchar_t while false true define undef pragma error "
00062                 "defined ifdef ifndef elif endif "
00063                 "\n#include\n#line\n"
00064                 "123 007 0xF00 1.23e+45 "
00065                 "1eet 0range 0xygen 1.2.3e+e-45 "
00066                 "'a''bcd''\\\'\\\"\\?\\\\\\a\\b\\r\\f\\n\\r\\t\\v''\\123\\xFaCe''\\udEaD\\U1234cdef'\n"
00067                 "'''\\c\\d\\e\\g\\h\\i''\\xg\\x''\\u\\uabcx\\U\\Uabcdeef\\uf0''unterminated\n"
00068                 "\"\"\"a\"\"bcd\"\"\\\'\\\"\\?\\\\\\a\\b\\r\\f\\n\\r\\t\\v\"\"\\123\\xFaCe\"\"\\udEaD\\U1234cdef\"\n"
00069                 "\"\\c\\d\\e\\g\\h\\i\"\"\\xg\\x\"\"\\u\\uabcx\\U\\Uabcdef\\uf0\"\"unterminated\n"
00070                 "\n";
00071 
00072         pp_token_type out[] = {
00073                 pp_token::TOK_BLANK,
00074                 pp_token::TOK_LEFT_BRACKET,
00075                 pp_token::TOK_LEFT_BRACKET,
00076                 pp_token::TOK_RIGHT_BRACKET,
00077                 pp_token::TOK_RIGHT_BRACKET,
00078                 pp_token::TOK_RIGHT_BRACE,
00079                 pp_token::TOK_RIGHT_BRACE,
00080                 pp_token::TOK_LEFT_BRACE,
00081                 pp_token::TOK_LEFT_BRACE,
00082                 pp_token::TOK_HASH,
00083                 pp_token::TOK_HASH,
00084                 pp_token::TOK_HASH_HASH,
00085                 pp_token::TOK_HASH_HASH,
00086                 pp_token::TOK_RIGHT_PAR,
00087                 pp_token::TOK_LEFT_PAR,
00088                 pp_token::TOK_SEMICOLON,
00089                 pp_token::TOK_COLON,
00090                 pp_token::TOK_QMARK,
00091                 pp_token::TOK_DOT,
00092                 pp_token::TOK_PLUS,
00093                 pp_token::TOK_STAR,
00094                 pp_token::TOK_PERCENT,
00095                 pp_token::TOK_SLASH,
00096                 pp_token::TOK_HAT,
00097                 pp_token::TOK_HAT,
00098                 pp_token::TOK_AMP,
00099                 pp_token::TOK_AMP,
00100                 pp_token::TOK_VBAR,
00101                 pp_token::TOK_VBAR,
00102                 pp_token::TOK_TILDE,
00103                 pp_token::TOK_TILDE,
00104                 pp_token::TOK_EMARK,
00105                 pp_token::TOK_EMARK,
00106                 pp_token::TOK_EQ,
00107                 pp_token::TOK_LT,
00108                 pp_token::TOK_GT,
00109                 pp_token::TOK_COMMA,
00110                 pp_token::TOK_MINUS,
00111                 pp_token::TOK_COLON_COLON,
00112                 pp_token::TOK_DOT_STAR,
00113                 pp_token::TOK_PLUS_EQ,
00114                 pp_token::TOK_MINUS_EQ,
00115                 pp_token::TOK_STAR_EQ,
00116                 pp_token::TOK_SLASH_EQ,
00117                 pp_token::TOK_PERCENT_EQ,
00118                 pp_token::TOK_HAT_EQ,
00119                 pp_token::TOK_HAT_EQ,
00120                 pp_token::TOK_AMP_EQ,
00121                 pp_token::TOK_AMP_EQ,
00122                 pp_token::TOK_VBAR_EQ,
00123                 pp_token::TOK_VBAR_EQ,
00124                 pp_token::TOK_LT_LT,
00125                 pp_token::TOK_GT_GT,
00126                 pp_token::TOK_GT_GT_EQ,
00127                 pp_token::TOK_LT_LT_EQ,
00128                 pp_token::TOK_EQ_EQ,
00129                 pp_token::TOK_EMARK_EQ,
00130                 pp_token::TOK_EMARK_EQ,
00131                 pp_token::TOK_LT_EQ,
00132                 pp_token::TOK_GT_EQ,
00133                 pp_token::TOK_AMP_AMP,
00134                 pp_token::TOK_AMP_AMP,
00135                 pp_token::TOK_VBAR_VBAR,
00136                 pp_token::TOK_VBAR_VBAR,
00137                 pp_token::TOK_PLUS_PLUS,
00138                 pp_token::TOK_MINUS_MINUS,
00139                 pp_token::TOK_MINUS_GT_STAR,
00140                 pp_token::TOK_MINUS_GT,
00141                 pp_token::TOK_DOT_DOT_DOT,
00142 
00143                 pp_token::TOK_ASM,
00144                 pp_token::TOK_BLANK,
00145                 pp_token::TOK_AUTO,
00146                 pp_token::TOK_BLANK,
00147                 pp_token::TOK_BOOL,
00148                 pp_token::TOK_BLANK,
00149                 pp_token::TOK_BREAK,
00150                 pp_token::TOK_BLANK,
00151                 pp_token::TOK_CASE,
00152                 pp_token::TOK_BLANK,
00153                 pp_token::TOK_CATCH,
00154                 pp_token::TOK_BLANK,
00155                 pp_token::TOK_CHAR,
00156                 pp_token::TOK_BLANK,
00157                 pp_token::TOK_CLASS,
00158                 pp_token::TOK_BLANK,
00159                 pp_token::TOK_CONST_CAST,
00160                 pp_token::TOK_BLANK,
00161                 pp_token::TOK_CONTINUE,
00162                 pp_token::TOK_BLANK,
00163                 pp_token::TOK_DEFAULT,
00164                 pp_token::TOK_BLANK,
00165                 pp_token::TOK_DELETE,
00166                 pp_token::TOK_BLANK,
00167                 pp_token::TOK_DO,
00168                 pp_token::TOK_BLANK,
00169                 pp_token::TOK_DOUBLE,
00170                 pp_token::TOK_BLANK,
00171                 pp_token::TOK_DYNAMIC_CAST,
00172                 pp_token::TOK_BLANK,
00173                 pp_token::TOK_ELSE,
00174                 pp_token::TOK_BLANK,
00175                 pp_token::TOK_ENUM,
00176                 pp_token::TOK_BLANK,
00177                 pp_token::TOK_EXPLICIT,
00178                 pp_token::TOK_BLANK,
00179                 pp_token::TOK_EXPORT,
00180                 pp_token::TOK_BLANK,
00181                 pp_token::TOK_EXTERN,
00182                 pp_token::TOK_BLANK,
00183                 pp_token::TOK_FLOAT,
00184                 pp_token::TOK_BLANK,
00185                 pp_token::TOK_FOR,
00186                 pp_token::TOK_BLANK,
00187                 pp_token::TOK_FRIEND,
00188                 pp_token::TOK_BLANK,
00189                 pp_token::TOK_GOTO,
00190                 pp_token::TOK_BLANK,
00191                 pp_token::TOK_IF,
00192                 pp_token::TOK_BLANK,
00193                 pp_token::TOK_INLINE,
00194                 pp_token::TOK_BLANK,
00195                 pp_token::TOK_INT,
00196                 pp_token::TOK_BLANK,
00197                 pp_token::TOK_LONG,
00198                 pp_token::TOK_BLANK,
00199                 pp_token::TOK_MUTABLE,
00200                 pp_token::TOK_BLANK,
00201                 pp_token::TOK_NAMESPACE,
00202                 pp_token::TOK_BLANK,
00203                 pp_token::TOK_NEW,
00204                 pp_token::TOK_BLANK,
00205                 pp_token::TOK_OPERATOR,
00206                 pp_token::TOK_BLANK,
00207                 pp_token::TOK_PRIVATE,
00208                 pp_token::TOK_BLANK,
00209                 pp_token::TOK_PROTECTED,
00210                 pp_token::TOK_BLANK,
00211                 pp_token::TOK_PUBLIC,
00212                 pp_token::TOK_BLANK,
00213                 pp_token::TOK_REGISTER,
00214                 pp_token::TOK_BLANK,
00215                 pp_token::TOK_REINTERPRET_CAST,
00216                 pp_token::TOK_BLANK,
00217                 pp_token::TOK_RETURN,
00218                 pp_token::TOK_BLANK,
00219                 pp_token::TOK_SHORT,
00220                 pp_token::TOK_BLANK,
00221                 pp_token::TOK_SIGNED,
00222                 pp_token::TOK_BLANK,
00223                 pp_token::TOK_SIZEOF,
00224                 pp_token::TOK_BLANK,
00225                 pp_token::TOK_STATIC,
00226                 pp_token::TOK_BLANK,
00227                 pp_token::TOK_STATIC_CAST,
00228                 pp_token::TOK_BLANK,
00229                 pp_token::TOK_STRUCT,
00230                 pp_token::TOK_BLANK,
00231                 pp_token::TOK_SWITCH,
00232                 pp_token::TOK_BLANK,
00233                 pp_token::TOK_TEMPLATE,
00234                 pp_token::TOK_BLANK,
00235                 pp_token::TOK_THIS,
00236                 pp_token::TOK_BLANK,
00237                 pp_token::TOK_THROW,
00238                 pp_token::TOK_BLANK,
00239                 pp_token::TOK_TRY,
00240                 pp_token::TOK_BLANK,
00241                 pp_token::TOK_TYPEDEF,
00242                 pp_token::TOK_BLANK,
00243                 pp_token::TOK_TYPEID,
00244                 pp_token::TOK_BLANK,
00245                 pp_token::TOK_TYPENAME,
00246                 pp_token::TOK_BLANK,
00247                 pp_token::TOK_UNION,
00248                 pp_token::TOK_BLANK,
00249                 pp_token::TOK_UNSIGNED,
00250                 pp_token::TOK_BLANK,
00251                 pp_token::TOK_USING,
00252                 pp_token::TOK_BLANK,
00253                 pp_token::TOK_VIRTUAL,
00254                 pp_token::TOK_BLANK,
00255                 pp_token::TOK_VOID,
00256                 pp_token::TOK_BLANK,
00257                 pp_token::TOK_VOLATILE,
00258                 pp_token::TOK_BLANK,
00259                 pp_token::TOK_WCHAR,
00260                 pp_token::TOK_BLANK,
00261                 pp_token::TOK_WHILE,
00262                 pp_token::TOK_BLANK,
00263                 pp_token::TOK_BOOL_LIT,
00264                 pp_token::TOK_BLANK,
00265                 pp_token::TOK_BOOL_LIT,
00266                 pp_token::TOK_BLANK,
00267                 pp_token::TOK_DEFINE,
00268                 pp_token::TOK_BLANK,
00269                 pp_token::TOK_UNDEF,
00270                 pp_token::TOK_BLANK,
00271                 pp_token::TOK_PRAGMA,
00272                 pp_token::TOK_BLANK,
00273                 pp_token::TOK_ERROR,
00274                 pp_token::TOK_BLANK,
00275                 pp_token::TOK_DEFINED,
00276                 pp_token::TOK_BLANK,
00277                 pp_token::TOK_IFDEF,
00278                 pp_token::TOK_BLANK,
00279                 pp_token::TOK_IFNDEF,
00280                 pp_token::TOK_BLANK,
00281                 pp_token::TOK_ELIF,
00282                 pp_token::TOK_BLANK,
00283                 pp_token::TOK_ENDIF,
00284                 pp_token::TOK_BLANK,
00285                 pp_token::TOK_LINE_END,
00286                 pp_token::TOK_HASH,
00287                 pp_token::TOK_INCLUDE,
00288                 pp_token::TOK_INCL_SIG,
00289                 pp_token::TOK_LINE_END,
00290                 pp_token::TOK_HASH,
00291                 pp_token::TOK_LINE,
00292                 pp_token::TOK_LINE_SIG,
00293                 pp_token::TOK_LINE_END,
00294 
00295                 pp_token::TOK_NUMBER_LIT,
00296                 pp_token::TOK_BLANK,
00297                 pp_token::TOK_NUMBER_LIT,
00298                 pp_token::TOK_BLANK,
00299                 pp_token::TOK_NUMBER_LIT,
00300                 pp_token::TOK_BLANK,
00301                 pp_token::TOK_NUMBER_LIT,
00302                 pp_token::TOK_BLANK,
00303 
00304                 pp_token::TOK_NUMBER_LIT,
00305                 pp_token::TOK_BLANK,
00306                 pp_token::TOK_NUMBER_LIT,
00307                 pp_token::TOK_BLANK,
00308                 pp_token::TOK_NUMBER_LIT,
00309                 pp_token::TOK_BLANK,
00310                 pp_token::TOK_NUMBER_LIT,
00311                 pp_token::TOK_BLANK,
00312 
00313                 pp_token::TOK_CHAR_LIT,
00314                 pp_token::TOK_CHAR_LIT,
00315                 pp_token::TOK_CHAR_LIT,
00316                 pp_token::TOK_CHAR_LIT,
00317                 pp_token::TOK_CHAR_LIT,
00318                 pp_token::TOK_LINE_END,
00319                 
00320                 pp_token::TOK_CHAR_LIT,
00321                 pp_token::TOK_CHAR_LIT,
00322                 pp_token::TOK_CHAR_LIT,
00323                 pp_token::TOK_CHAR_LIT,
00324                 pp_token::TOK_CHAR_LIT,
00325                 pp_token::TOK_LINE_END,
00326                 
00327                 pp_token::TOK_STRING_LIT,
00328                 pp_token::TOK_STRING_LIT,
00329                 pp_token::TOK_STRING_LIT,
00330                 pp_token::TOK_STRING_LIT,
00331                 pp_token::TOK_STRING_LIT,
00332                 pp_token::TOK_STRING_LIT,
00333                 pp_token::TOK_LINE_END,
00334                 
00335                 pp_token::TOK_STRING_LIT,
00336                 pp_token::TOK_STRING_LIT,
00337                 pp_token::TOK_STRING_LIT,
00338                 pp_token::TOK_STRING_LIT,
00339                 pp_token::TOK_LINE_END,
00340                 
00341                 pp_token::TOK_LINE_END,
00342                 pp_token::TOK_FILE_END
00343         };
00344 
00345         ptr<file_info> fi = file_info::create("abcd",NULL);
00346 
00347         ptr<pre_lex> plx = pre_lex::create(string_source::create(inp),encoder_ascii7::create());
00348         ptr<pp_lex> pp = pp_lex::create(plx,fi);
00349         ptr<pp_token> tok;
00350 
00351         ulint i = 0;
00352         do {
00353                 
00354                 tok = pp->read(tok && tok->type_get() == pp_token::TOK_LINE_END);
00355 #if 0
00356                 ::std::cerr << " test " << i << " : " <<
00357                         tok->name_get() << " " <<
00358                         tok->type_get() << " ?== " << (int)out[i] << ""
00359                         << ::std::endl;
00360 #if 0         
00361                         (int)tok->value_get() << " ?== " << (int)ouv[i] << " offset " <<
00362                         (int)tok->location_get()->offset_get() << " ?== " << loc[i]->offset_get() << " line " <<
00363                         (int)tok->location_get()->line_get() << " ?== " << loc[i]->line_get() << " column " <<
00364                         (int)tok->location_get()->column_get() << " ?== " << loc[i]->column_get() << " order " <<
00365                         (int)tok->location_get()->order_get() << " ?== " << loc[i]->order_get() << ::std::endl;
00366 #endif
00367 #endif
00368                 lassert(is_equal(tok->type_get(),out[i]));
00369 
00370                 if (tok->type_get() == pp_token::TOK_FILE_END) break;
00371                 i++;
00372         } while (true);
00373 }
00374 
00375 end_package(lex);
00376 end_package(cplus);
00377 end_package(lang);
00378 end_package(lestes);
00379 
00380 int main(void)
00381 {
00382 	::lestes::lang::cplus::lex::pp_lex_test();
00383         return 0;
00384 }
00385 /* vim: set ft=lestes : */
00386