Token.def

(*!m2r10*) (* Copyright (c) 2015 B.Kowarsch. All rights reserved. *)

DEFINITION MODULE Token;

(* Token Definitions for Modula-2 R10 Core Compiler *)


(* Semantic Tokens *)

TYPE Semantic = [Invalid..EOF] OF Token;


(* Semantic and Non-Semantic Tokens *)

TYPE Token =
  ( Invalid,         (* 0 *)
  
  (* Reserved Words *)
    Alias,           (* 1 *)
    And,             (* 2 *)
    Arglist,         (* 3 *)
    Array,           (* 4 *)
    Bare,            (* 5 *)
    Begin,           (* 6 *)
    Blueprint,       (* 7 *)
    By,              (* 8 *)
    Case,            (* 9 *)
    Const,          (* 10 *)
    Copy,           (* 11 *)
    Definition,     (* 12 *)
    Div,            (* 13 *)
    Do,             (* 14 *)
    Else,           (* 15 *)
    Elsif,          (* 16 *)
    End,            (* 17 *)
    Exit,           (* 18 *)
    For,            (* 19 *)
    From,           (* 20 *)
    Genlib,         (* 21 *)
    If,             (* 22 *)
    Implementation, (* 23 *)
    Import,         (* 24 *)
    In,             (* 25 *)
    Loop,           (* 26 *)
    Mod,            (* 27 *)
    Module,         (* 28 *)
    New,            (* 29 *)
    None,           (* 30 *)
    Not,            (* 31 *)
    Of,             (* 32 *)
    Opaque,         (* 33 *)
    Or,             (* 34 *)
    Pointer,        (* 35 *)
    Procedure,      (* 36 *)
    Record,         (* 37 *)
    Referential,    (* 38 *)
    Release,        (* 39 *)
    Repeat,         (* 40 *)
    Retain,         (* 41 *)
    Return,         (* 42 *)
    Set,            (* 43 *)
    Then,           (* 44 *)
    To,             (* 45 *)
    Type,           (* 46 *)
    Until,          (* 47 *)
    Var,            (* 48 *)
    While,          (* 49 *)
    Yield,          (* 50 *)
    
  (* Dual-Use RW-Identifiers *)
  
  (* Constant Bindable Identifiers *)
    Tflags,         (* 51 *)
    Tdyn,           (* 52 *)
    Trefc,          (* 53 *)
    Tordered,       (* 54 *)
    Tsorted,        (* 55 *)
    Tlimit,         (* 56 *)
    Tscalar,        (* 57 *)
    Tmax,           (* 58 *)
    Tmin,           (* 59 *)
    
  (* Procedure Bindable Identifiers *)
    Abs,            (* 60 *)
    Length,         (* 61 *)
    Exists,         (* 62 *)
    Seek,           (* 63 *)
    Subset,         (* 64 *)
    Read,           (* 65 *)
    Readnew,        (* 66 *)
    Write,          (* 67 *)
    Writef,         (* 68 *)
    Sxf,            (* 69 *)
    Val,            (* 70 *)
    Count,          (* 71 *)
    Value,          (* 72 *)
    Store,          (* 73 *)
    Insert,         (* 74 *)
    Remove,         (* 75 *)
    Append,         (* 76 *)
    
  (* Other Dual-Use Identifiers *)
    Address,        (* 77 *)
    Cast,           (* 78 *)
    Coroutine,      (* 79 *)
    Octet,          (* 80 *)
    Tliteral,       (* 81 *)
    Unsafe,         (* 82 *)
    
  (* Any Other Identifiers *)
    OtherIdent,     (* 83 *)
  
  (* Literals *)
    WholeNumber,    (* 84 *)
    RealNumber,     (* 85 *)
    Character,      (* 86 *)
    QuotedString,   (* 87 *)
    ChevronText,    (* 88 *)
    
  (* Punctuation *)
    Dot,            (* 89 *)
    Comma,          (* 90 *)
    Colon,          (* 91 *)
    Semicolon,      (* 92 *)
    VerticalBar,    (* 93 *)
    Deref,          (* 94 *)
    DotDot,         (* 95 *)
    Assign,         (* 96 *)
    PlusPlus,       (* 97 *)
    MinusMinus,     (* 98 *)
    OneWayDep,      (* 99 *)
    MutualDep,     (* 100 *)
    MutualExcl,    (* 101 *)
    PlusMinus,     (* 102 *)
    
  (* Paired Delimiters *)
    LParen,        (* 103 *)
    RParen,        (* 104 *)
    LBracket,      (* 105 *)
    RBracket,      (* 106 *)
    LBrace,        (* 107 *)
    RBrace,        (* 108 *)
    
  (* Operators *)
  
  (* Non-Resword Level-1 Operators *)
    Equal,         (* 109 *)
    NotEqual,      (* 110 *)
    Greater,       (* 111 *)
    GreaterOrEq,   (* 112 *)
    Less,          (* 113 *)
    LessOrEq,      (* 114 *)
    Identity,      (* 115 *)
    
  (* Non-Resword Level-2 Operators *)
    Plus,          (* 116 *)   (* also used as punctuation *)
    Minus,         (* 117 *)   (* also used as punctuation *)
    Concat,        (* 118 *)
    SetDiff,       (* 119 *)
    
  (* Non-Resword Level-3 Operators *)
    Asterisk,      (* 120 *)   (* also used as punctuation *)
    RealDiv,       (* 121 *)
    DotProd,       (* 122 *)   (* reserved for future use *)
    
  (* Non-Resword Level-4 Operators *)
    Power,         (* 123 *)   (* reserved for future use *)
    
  (* Non-Resword Level-5 Operators *)
    TypeConv,      (* 124 *)
    
  (* End Of File Marker *)
    EOF,           (* 125 *)
    
  (* Non-Semantic Tokens *)
  
  (* Pragmas *)
    Pragma,        (* 126 *)
    
  (* Comments *)
    LineComment,   (* 127 *)
    BlockComment); (* 128 *)


(* Functions To Determine Token Classification *)

PROCEDURE isResWord ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a reserved word, otherwise FALSE. *)

PROCEDURE isIdentifier ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is an identifier, otherwise FALSE. *)

PROCEDURE isConstBindableIdent ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a constant bindable identifier, otherwise FALSE. *)

PROCEDURE isProcBindableIdent ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a procedure bindable identifier, otherwise FALSE. *)

PROCEDURE isNumber ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a number literal, otherwise FALSE. *)

PROCEDURE isCharOrString ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a character or string, otherwise FALSE. *)

PROCEDURE isOperL1 ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a level-1 operator, otherwise FALSE. *)

PROCEDURE isOperL2 ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a level-2 operator, otherwise FALSE. *)

PROCEDURE isOperL3 ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a level-3 operator, otherwise FALSE. *)

PROCEDURE isComment ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a comment, otherwise FALSE. *)

PROCEDURE isPragma ( t : Token ) : BOOLEAN;
 (* Returns TRUE if t is a pragma, otherwise FALSE. *)


END Token.