character.hh

Go to the documentation of this file.

/*
   The lestes compiler suite
   Copyright (C) 2002, 2003, 2004, 2005 Miroslav Tichy
   Copyright (C) 2002, 2003, 2004, 2005 Petr Zika
   Copyright (C) 2002, 2003, 2004, 2005 Vojtech Hala
   Copyright (C) 2002, 2003, 2004, 2005 Jiri Kosina
   Copyright (C) 2002, 2003, 2004, 2005 Pavel Sanda
   Copyright (C) 2002, 2003, 2004, 2005 Jan Zouhar
   Copyright (C) 2002, 2003, 2004, 2005 Rudolf Thomas

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   See the full text of the GNU General Public License version 2, and
   the limitations in the file doc/LICENSE.

   By accepting the license the licensee waives any and all claims
   against the copyright holder(s) related in whole or in part to the
   work, its use, and/or the inability to use it.
 
 */
#ifndef lestes__std___character_hh___included
#define lestes__std___character_hh___included

/*! \file
  \brief Representation of characters in source encoding.

  Defines type ucn for storing characters in source encoding.
  Defines type hchar for storing host characters.
  Declares helper class character containing constants and manipulators.
*/
#include <lestes/common.hh>
#include <limits>

package(lestes);
package(std);

/*!
  \brief Type for storing characters in source encoding.
  
  The source character is represented by an at least 32-bit unsigned integer.
  When bit 31 is set, a value stored in the lower bits is in external character set, 
  either input file character set, or execution (target) character set, depending on the context.
  The exception is the eof value, represented as 32 one bits.
  Thus execution character with value 0x7FFFFFFF cannot be represented in this scheme.
  When bit 31 is not set, Unicode UTF-32 value is stored in the lower bits.
  When the type is longer than 32 bits, the higher bits shall not be used and it is entirely
  in the responsibility of the programmer to keep these zero, otherwise the whole
  mechanism would break. To keep sane, use character class static methods.
*/
typedef lc_host_uint_least32 ucn;

/*!
  \brief Host character type.

  The native host character type, used in i/o operations.
*/
typedef char hchar;

/*!
  \brief Character constants and manipulators.
  
  Contains symbolic constants for Unicode representation of basic source characters
  as well as creators, predicates, transformers and extracters.
*/
class character {
public:
        //! ASCII bell character.
        static const ucn ascii_bell = static_cast<ucn>(0x07);
        //! ASCII backspace character.
        static const ucn ascii_backspace = static_cast<ucn>(0x08);
        //! ASCII tabulator character.
        static const ucn ascii_tab = static_cast<ucn>(0x09);
        //! ASCII vertical tabulator character.
        static const ucn ascii_vtab = static_cast<ucn>(0x0B);
        //! ASCII form feed character.
        static const ucn ascii_form_feed = static_cast<ucn>(0x0C);
        //! ASCII new line character.
        static const ucn ascii_new_line = static_cast<ucn>(0x0A);
        //! ASCII carriage return character.
        static const ucn ascii_carriage_return = static_cast<ucn>(0x0D);
        //! ASCII space character.
        static const ucn ascii_space = static_cast<ucn>(0x20);
        //! ASCII exclamation mark character.
        static const ucn ascii_emark = static_cast<ucn>(0x21);
        //! ASCII double quote character.
        static const ucn ascii_dquote = static_cast<ucn>(0x22);
        //! ASCII hash character.
        static const ucn ascii_hash = static_cast<ucn>(0x23);
        //! ASCII dollar character.
        static const ucn ascii_dollar = static_cast<ucn>(0x24);
        //! ASCII percent character.
        static const ucn ascii_percent = static_cast<ucn>(0x25);
        //! ASCII ampersand character.
        static const ucn ascii_amp = static_cast<ucn>(0x26);
        //! ASCII quote character.
        static const ucn ascii_quote = static_cast<ucn>(0x27);
        //! ASCII left parenthesis character.
        static const ucn ascii_left_par = static_cast<ucn>(0x28);
        //! ASCII right parenthesis character.
        static const ucn ascii_right_par = static_cast<ucn>(0x29);
        //! ASCII star character.
        static const ucn ascii_star = static_cast<ucn>(0x2A);
        //! ASCII plus character.
        static const ucn ascii_plus = static_cast<ucn>(0x2B);
        //! ASCII comma character.
        static const ucn ascii_comma = static_cast<ucn>(0x2C);
        //! ASCII minus character.
        static const ucn ascii_minus = static_cast<ucn>(0x2D);
        //! ASCII dot character.
        static const ucn ascii_dot = static_cast<ucn>(0x2E);
        //! ASCII slash character.
        static const ucn ascii_slash = static_cast<ucn>(0x2F);
        //! ASCII digit 0 character.
        static const ucn ascii_digit_0 = static_cast<ucn>(0x30);
        //! ASCII digit 1 character.
        static const ucn ascii_digit_1 = static_cast<ucn>(0x31);
        //! ASCII digit 2 character.
        static const ucn ascii_digit_2 = static_cast<ucn>(0x32);
        //! ASCII digit 3 character.
        static const ucn ascii_digit_3 = static_cast<ucn>(0x33);
        //! ASCII digit 4 character.
        static const ucn ascii_digit_4 = static_cast<ucn>(0x34);
        //! ASCII digit 5 character.
        static const ucn ascii_digit_5 = static_cast<ucn>(0x35);
        //! ASCII digit 6 character.
        static const ucn ascii_digit_6 = static_cast<ucn>(0x36);
        //! ASCII digit 7 character.
        static const ucn ascii_digit_7 = static_cast<ucn>(0x37);
        //! ASCII digit 8 character.
        static const ucn ascii_digit_8 = static_cast<ucn>(0x38);
        //! ASCII digit 9 character.
        static const ucn ascii_digit_9 = static_cast<ucn>(0x39);
        //! ASCII colon character.
        static const ucn ascii_colon = static_cast<ucn>(0x3A);
        //! ASCII semicolon character.
        static const ucn ascii_semicolon = static_cast<ucn>(0x3B);
        //! ASCII less than character.
        static const ucn ascii_lt = static_cast<ucn>(0x3C);
        //! ASCII equals character.
        static const ucn ascii_eq = static_cast<ucn>(0x3D);
        //! ASCII greater than character.
        static const ucn ascii_gt = static_cast<ucn>(0x3E);
        //! ASCII question mark character.
        static const ucn ascii_qmark = static_cast<ucn>(0x3F);
        //! ASCII commercial at character (zavinac).
        static const ucn ascii_at = static_cast<ucn>(0x40);
        //! ASCII upper a character.
        static const ucn ascii_upper_a = static_cast<ucn>(0x41);
        //! ASCII upper b character.
        static const ucn ascii_upper_b = static_cast<ucn>(0x42);
        //! ASCII upper c character.
        static const ucn ascii_upper_c = static_cast<ucn>(0x43);
        //! ASCII upper d character.
        static const ucn ascii_upper_d = static_cast<ucn>(0x44);
        //! ASCII upper e character.
        static const ucn ascii_upper_e = static_cast<ucn>(0x45);
        //! ASCII upper f character.
        static const ucn ascii_upper_f = static_cast<ucn>(0x46);
        //! ASCII upper g character.
        static const ucn ascii_upper_g = static_cast<ucn>(0x47);
        //! ASCII upper h character.
        static const ucn ascii_upper_h = static_cast<ucn>(0x48);
        //! ASCII upper i character.
        static const ucn ascii_upper_i = static_cast<ucn>(0x49);
        //! ASCII upper j character.
        static const ucn ascii_upper_j = static_cast<ucn>(0x4A);
        //! ASCII upper k character.
        static const ucn ascii_upper_k = static_cast<ucn>(0x4B);
        //! ASCII upper l character.
        static const ucn ascii_upper_l = static_cast<ucn>(0x4C);
        //! ASCII upper m character.
        static const ucn ascii_upper_m = static_cast<ucn>(0x4D);
        //! ASCII upper n character.
        static const ucn ascii_upper_n = static_cast<ucn>(0x4E);
        //! ASCII upper o character.
        static const ucn ascii_upper_o = static_cast<ucn>(0x4F);
        //! ASCII upper p character.
        static const ucn ascii_upper_p = static_cast<ucn>(0x50);
        //! ASCII upper q character.
        static const ucn ascii_upper_q = static_cast<ucn>(0x51);
        //! ASCII upper r character.
        static const ucn ascii_upper_r = static_cast<ucn>(0x52);
        //! ASCII upper s character.
        static const ucn ascii_upper_s = static_cast<ucn>(0x53);
        //! ASCII upper t character.
        static const ucn ascii_upper_t = static_cast<ucn>(0x54);
        //! ASCII upper u character.
        static const ucn ascii_upper_u = static_cast<ucn>(0x55);
        //! ASCII upper v character.
        static const ucn ascii_upper_v = static_cast<ucn>(0x56);
        //! ASCII upper w character.
        static const ucn ascii_upper_w = static_cast<ucn>(0x57);
        //! ASCII upper x character.
        static const ucn ascii_upper_x = static_cast<ucn>(0x58);
        //! ASCII upper y character.
        static const ucn ascii_upper_y = static_cast<ucn>(0x59);
        //! ASCII upper z character.
        static const ucn ascii_upper_z = static_cast<ucn>(0x5A);
        //! ASCII left bracket character.
        static const ucn ascii_left_bracket = static_cast<ucn>(0x5B);
        //! ASCII backslash character.
        static const ucn ascii_backslash = static_cast<ucn>(0x5C);
        //! ASCII right bracket character.
        static const ucn ascii_right_bracket = static_cast<ucn>(0x5D);
        //! ASCII hat character.
        static const ucn ascii_hat = static_cast<ucn>(0x5E);
        //! ASCII underscore character.
        static const ucn ascii_underscore = static_cast<ucn>(0x5F);
        //! ASCII back quote character.
        static const ucn ascii_bquote = static_cast<ucn>(0x60);
        //! ASCII lower a character.
        static const ucn ascii_lower_a = static_cast<ucn>(0x61);
        //! ASCII lower b character.
        static const ucn ascii_lower_b = static_cast<ucn>(0x62);
        //! ASCII lower c character.
        static const ucn ascii_lower_c = static_cast<ucn>(0x63);
        //! ASCII lower d character.
        static const ucn ascii_lower_d = static_cast<ucn>(0x64);
        //! ASCII lower e character.
        static const ucn ascii_lower_e = static_cast<ucn>(0x65);
        //! ASCII lower f character.
        static const ucn ascii_lower_f = static_cast<ucn>(0x66);
        //! ASCII lower g character.
        static const ucn ascii_lower_g = static_cast<ucn>(0x67);
        //! ASCII lower h character.
        static const ucn ascii_lower_h = static_cast<ucn>(0x68);
        //! ASCII lower i character.
        static const ucn ascii_lower_i = static_cast<ucn>(0x69);
        //! ASCII lower j character.
        static const ucn ascii_lower_j = static_cast<ucn>(0x6A);
        //! ASCII lower k character.
        static const ucn ascii_lower_k = static_cast<ucn>(0x6B);
        //! ASCII lower l character.
        static const ucn ascii_lower_l = static_cast<ucn>(0x6C);
        //! ASCII lower m character.
        static const ucn ascii_lower_m = static_cast<ucn>(0x6D);
        //! ASCII lower n character.
        static const ucn ascii_lower_n = static_cast<ucn>(0x6E);
        //! ASCII lower o character.
        static const ucn ascii_lower_o = static_cast<ucn>(0x6F);
        //! ASCII lower p character.
        static const ucn ascii_lower_p = static_cast<ucn>(0x70);
        //! ASCII lower q character.
        static const ucn ascii_lower_q = static_cast<ucn>(0x71);
        //! ASCII lower r character.
        static const ucn ascii_lower_r = static_cast<ucn>(0x72);
        //! ASCII lower s character.
        static const ucn ascii_lower_s = static_cast<ucn>(0x73);
        //! ASCII lower t character.
        static const ucn ascii_lower_t = static_cast<ucn>(0x74);
        //! ASCII lower u character.
        static const ucn ascii_lower_u = static_cast<ucn>(0x75);
        //! ASCII lower v character.
        static const ucn ascii_lower_v = static_cast<ucn>(0x76);
        //! ASCII lower w character.
        static const ucn ascii_lower_w = static_cast<ucn>(0x77);
        //! ASCII lower x character.
        static const ucn ascii_lower_x = static_cast<ucn>(0x78);
        //! ASCII lower y character.
        static const ucn ascii_lower_y = static_cast<ucn>(0x79);
        //! ASCII lower z character.
        static const ucn ascii_lower_z = static_cast<ucn>(0x7A);
        //! ASCII left brace character.
        static const ucn ascii_left_brace = static_cast<ucn>(0x7B);
        //! ASCII vertical bar character.
        static const ucn ascii_vbar = static_cast<ucn>(0x7C);
        //! ASCII right brace character.
        static const ucn ascii_right_brace = static_cast<ucn>(0x7D);
        //! ASCII tilde character.
        static const ucn ascii_tilde = static_cast<ucn>(0x7E);
        //! Initializes internal tables.
        character(void);
        //! Destructor of the initializer.
        ~character(void);
        //! Returns internal character set ucn.
        static inline ucn create_internal(ulint code);
        //! Returns internal ucn from host character.
        static inline ucn create_from_host(hchar c);
        //! Returns external character set ucn.
        static inline ucn create_external(ulint code);
        //! Returns internal character set ucn from hexadecimal digit.
        static inline ucn create_xdigit(ulint digit);
        //! Tests if value is internal character.
        static inline bool is_internal(ucn u);
        //! Tests if value is external character.
        static inline bool is_external(ucn u);
        //! Tests if value is basic source character.
        static inline bool is_basic(ucn u);
        //! Tests if value is translated source character.
        static inline bool is_translated(ucn u);
        //! Tests if value is 7 bit ASCII.
        static inline bool is_ascii7(ucn u);
        //! Tests if value is ASCII letter.
        static inline bool is_alpha(ucn u);
        //! Tests if value is ASCII uppercase letter.
        static inline bool is_upper(ucn u);
        //! Tests if value is ASCII lowercase letter.
        static inline bool is_lower(ucn u);
        //! Tests if value is ASCII digit.
        static inline bool is_digit(ucn u);
        //! Tests if value is ASCII octal digit.
        static inline bool is_odigit(ucn u);
        //! Tests if value is ASCII hexadecimal digit .
        static inline bool is_xdigit(ucn u);
        //! Tests if value is ASCII space character .
        static inline bool is_space(ucn u);
        //! Tests if value is translated C++ identifier character.
        static bool is_translated_identifier(ucn u);
        //! Tests if value is host character encodable in ucn.
        static inline bool is_encodable_host(ucn u);
        //! Returns uppercase of ASCII letter.
        static inline ucn to_upper(ucn u);
        //! Returns lowercase of ASCII letter.
        static inline ucn to_lower(ucn u);
        //! Returns host character.
        static inline hchar to_host(ucn u);
        //! Returns digit value.
        static inline ulint extract_digit(ucn u);
        //! Returns hexadecimal digit value.
        static inline ulint extract_xdigit(ucn u);
        //! Returns hexadecimal digit value.
        static inline ulint extract_odigit(ucn u);
        //! Returns character value.
        static inline ulint extract_value(ucn u);
private:
        //! Number of significant ucn_bits.
        static const ulint ucn_bits = 32;
        //! Mask for character value.
        static const ucn value_mask = (static_cast<ucn>(1) << (ucn_bits - 1)) - 1;
        //! Mask for external characters.
        static const ucn external_mask = (static_cast<ucn>(1) << (ucn_bits - 1));
        //! Internal eof constant, only for ucn_traits, must not be used elsewhere.
        static const ucn eof = ((static_cast<ucn>(1) << (ucn_bits - 1) - 1) << 1) | 1;
        //! Host character value presumably not representing encodable character.
        static const hchar hchar_unknown = '\0';
        //! Length of 7 bit ASCII table.
        static const ulint ascii_length = 128;
        //! Length of table of host characters .
        static const ulint host_length = 1 << (::std::numeric_limits<hchar>::digits);
        //! Values of ASCII flags.
        enum ascii_flags_values {
                FLG_NONE = 0x00,
                FLG_BASIC = 0x01,
                FLG_UPPER = 0x02,
                FLG_LOWER = 0x04,
                FLG_ALPHA = 0x06,
                FLG_DIGIT = 0x08,
                FLG_ODIGIT = 0x10,
                FLG_XDIGIT = 0x20,
                FLG_SPACE = 0x40
        };
        //! Type of ASCII flags.
        typedef lc_host_uint_least8 ascii_flags_type;
        //! Flags of ASCII characters.
        static ascii_flags_type ascii_flags[ascii_length];
        //! Range of 16 bit codes.
        typedef struct {
                lc_host_uint_least16 low;
                lc_host_uint_least16 high;
        } range_type;
        //! List of unicode character ranges for C++ identifiers.
        static range_type identifier_ranges[];
        //! Internal to host encoding of basic characters translation table.
        static hchar internal_to_host[ascii_length];
        //! Host to internal value encoding translation table.
        static ulint host_to_internal[host_length];
        //! Hides copy constructor.
        character(const character &copy);
        //! Hides assignment operator.
        character &operator=(const character &rhs);
        //! Multiple initialization guard.
        static bool initialized;
};

/*!
  Returns ucn for internal character of given code.
  \pre code <= 0x7FFFFFFF
  \param code  The code of the character.
  \return The internal character encoded in ucn.
*/
inline ucn character::create_internal(ulint code)
{
        lassert(code <= 0x7FFFFFFF);
        return static_cast<ucn>(code);
}

/*!
  Returns ucn for basic host character.
  \pre The host character is basic.
  \param c  The host character.
  \return The host character encoded in ucn.
*/
inline ucn character::create_from_host(hchar c)
{
        ulint i = static_cast<unsigned char>(c);
        ulint x = host_to_internal[i];
        lassert(x != 0);
        return create_internal(x);
}

/*!
  Returns ucn for external character of given code.
  \pre code < 0x7FFFFFFF
  \param code  The code of the character.
  \return The external character encoded in ucn.
*/
inline ucn character::create_external(ulint code)
{
        lassert(code < 0x7FFFFFFF);
        return static_cast<ucn>(code) | external_mask;  
}

/*!
  Returns ucn for internal character representing given lower case hexadecimal number.
  \pre digit < 16
  \param digit  The hexadecimal digit.
  \return The digit as internal ucn.
*/
inline ucn character::create_xdigit(ulint digit)
{
        lassert(digit < 16);
        return static_cast<ucn>(digit < 10 ? ascii_digit_0 + digit : ascii_lower_a + digit - 10);
}

/*!
  Tests if value is internal character.
  \param u  The value to test.
  \return true  If the value represents internal character.
*/
inline bool character::is_internal(ucn u)
{
        return (u & external_mask) == 0;
}

/*!
  Tests if value is external host character with known encoding into ucn.
  \param u  The value to test.
  \return true  If the value represents encodable host character.
*/
inline bool character::is_encodable_host(ucn u)
{
        if (!is_external(u)) return false;
        ulint x = extract_value(u);
        return x < host_length && host_to_internal[x] != 0;
}

/*!
  Tests if value is external character.
  \param u The value to test.
  \return true  If the value represents external character.
*/
inline bool character::is_external(ucn u)
{
        return (u & external_mask) != 0; 
}

/*!
  Tests if value is basic source character.
  \param u The value to test.
  \return true  If the value represents basic source character.
*/
inline bool character::is_basic(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_BASIC);
}

/*!
  Tests if value is translated source character.
  Certain ranges are disallowed for translated characters.
  \param u The value to test.
  \return true  If the value represents translated source character.
*/
inline bool character::is_translated(ucn u)
{
        return is_internal(u) && !is_basic(u) && !(u < 0x20 || (0x7F <= u && u <= 0x9F));
}

/*!
  Tests if value is 7 bit ASCII.
  \param u The value to test.
  \return true  If the value represents 7 bit ASCII character.
*/
inline bool character::is_ascii7(ucn u)
{
        // implicitly is_internal(u) check
        return u < 0x80;
}

/*!
  Tests if value is ASCII letter.
  \param u The value to test.
  \return true  If the value represents ASCII letter.
*/
inline bool character::is_alpha(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_ALPHA == FLG_ALPHA);
}

/*!
  Tests if value is ASCII uppercase letter.
  \param u The value to test.
  \return true  If the value represents ASCII uppercase letter.
*/
inline bool character::is_upper(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_UPPER);
}

/*!
  Tests if value is ASCII lowercase letter.
  \param u The value to test.
  \return true  If the value represents ASCII lowercase letter.
*/
inline bool character::is_lower(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_LOWER);
}

/*!
  Tests if value is ASCII digit.
  \param u The value to test.
  \return true  If the value represents ASCII digit.
*/
inline bool character::is_digit(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_DIGIT);
}

/*!
  Tests if value is ASCII octal digit.
  \param u The value to test.
  \return true  If the value represents ASCII octal digit.
*/
inline bool character::is_odigit(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_ODIGIT);
}

/*!
  Tests if value is ASCII hexadecimal digit.
  \param u The value to test.
  \return true  If the value represents ASCII hexadecimal digit.
*/
inline bool character::is_xdigit(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_XDIGIT);
}

/*!
  Tests if value is ASCII space character (space, tab, vtab, ff, cr, lf) 
  \param u The value to test.
  \return true  If the value represents ASCII space character.
*/
inline bool character::is_space(ucn u)
{
        return is_ascii7(u) && (ascii_flags[u] & FLG_SPACE);
}

/*!
  Returns uppercase of ASCII lowercase letter, other values intact.
  \param u The value to transform.
  \return Uppercase of the given lowercase letter or the original ucn.
*/
inline ucn character::to_upper(ucn u)
{
        if (is_lower(u)) return u & 0xDF;
        return u;
}

/*!
  Returns lowercase of ASCII uppercase letter, other values intact.
  \param u The value to transform.
  \return Uppercase of the given uppercase letter or the original ucn.
*/
inline ucn character::to_lower(ucn u)
{
        if (is_upper(u)) return u | 0x20;
        return u;
}

/*!
  Returns host character for subset of ASCII for which the encoding is known.
  It contains basic ASCII characters, '$', '@' and '`'.
  Other values are not supported.
  \pre The value is internal and contains only character with known encoding.
  \param u The value to transform.
  \return The given character in host encoding.
*/
inline hchar character::to_host(ucn u)
{
        lassert(is_ascii7(u));
        hchar c = internal_to_host[extract_value(u)];
        lassert(c != hchar_unknown);
        return c;
}

/*!
  Returns digit value of the ucn.
  \pre is_digit(u)
  \param u  The digit to extract.
  \return The value of the digit.
*/
inline ulint character::extract_digit(ucn u)
{
        lassert(is_digit(u));
        return static_cast<ulint>(u) - ascii_digit_0;
}

/*!
  Returns hexadecimal digit value of the ucn.
  \pre is_xdigit(u)
  \param u  The digit to extract.
  \return The value of the hexadecimal digit.
*/
inline ulint character::extract_xdigit(ucn u)
{
        if (is_digit(u)) return extract_digit(u);
        lassert(is_xdigit(u));
        return static_cast<ulint>(to_upper(u)) - ascii_upper_a + 10;
}

/*!
  Returns octal digit value of the ucn.
  \pre is_odigit(u)
  \param u  The digit to extract.
  \return The value of the octal digit.
*/
inline ulint character::extract_odigit(ucn u)
{
        lassert(is_odigit(u));
        return extract_digit(u); 
}

/*!
  Returns character value of the ucn for both unicode and external characters.
  \param u  The character to extract.
  \return The value of the stored character.
*/
inline ulint character::extract_value(ucn u)
{
        return u & value_mask; 
}

/*!
  Initializer of static fields of character class.
*/
static character character_initializer;

end_package(std);
end_package(lestes);

#endif
/* vim: set ft=lestes : */

---