sti/parser/c_tokens.txt

# character sets
[identifier_schars a-zA-Z_
[identifier_chars a-zA-Z0-9_
[num_chars_start 0-9
[num_chars 0-9
[octal_chars 0-7
[hex_chars 0-9a-fA-F
[float_chars_start 0-9.\-\+
[float_chars 0-9.eE
[float_chars_start 0-9.\-\+
[ws \ \t\r\n
[endl \r\n
# single-character transitions can't be inverted yet, but csets can 
[slash \\
[star *
[quote "
[squote '


# keywords
# : is extra user data to be carried through
# | is a failover on a character set on discontinuity in the word 
{alignas :cat=keyword |identifier_chars>LST_identifier
{alignof :cat=keyword |identifier_chars>LST_identifier
{_Alignas :cat=keyword |identifier_chars>LST_identifier
{_Alignof :cat=keyword |identifier_chars>LST_identifier
{_Atomic :cat=keyword |identifier_chars>LST_identifier
{break :cat=keyword |identifier_chars>LST_identifier
{case :cat=keyword |identifier_chars>LST_identifier
{catch :cat=keyword |identifier_chars>LST_identifier
{continue :cat=keyword |identifier_chars>LST_identifier
{const :cat=keyword |identifier_chars>LST_identifier
{default :cat=keyword |identifier_chars>LST_identifier
{do :cat=keyword |identifier_chars>LST_identifier
{else :cat=keyword |identifier_chars>LST_identifier
{enum :cat=keyword |identifier_chars>LST_identifier
{extern :cat=keyword |identifier_chars>LST_identifier
{for :cat=keyword |identifier_chars>LST_identifier
{_Generic :cat=keyword |identifier_chars>LST_identifier
{goto :cat=keyword |identifier_chars>LST_identifier
{if :cat=keyword |identifier_chars>LST_identifier
{inline :cat=keyword |identifier_chars>LST_identifier
{new :cat=keyword |identifier_chars>LST_identifier
{restrict :cat=keyword |identifier_chars>LST_identifier
{return :cat=keyword |identifier_chars>LST_identifier
{_Noreturn :cat=keyword |identifier_chars>LST_identifier
{sizeof :cat=keyword |identifier_chars>LST_identifier
{_Static_assert :cat=keyword |identifier_chars>LST_identifier
{static :cat=keyword |identifier_chars>LST_identifier
{struct :cat=keyword |identifier_chars>LST_identifier
{switch :cat=keyword |identifier_chars>LST_identifier
{cat=typedef :cat=keyword |identifier_chars>LST_identifier
{union :cat=keyword |identifier_chars>LST_identifier
{while :cat=keyword |identifier_chars>LST_identifier

# operators and punctuation
{+ :cat=operator
{& :cat=operator
{+= :cat=operator
{&= :cat=operator
{&& :cat=operator
{== :cat=operator
{!= :cat=operator
{( :cat=punct :block=open
{) :cat=punct :block=close
{- :cat=operator
{| :cat=operator
{-= :cat=operator
{|= :cat=operator
{|| :cat=operator
{< :cat=operator
{<= :cat=operator
{[ :cat=punct :block=open
{] :cat=punct :block=close
{* :cat=operator
{^ :cat=operator
{*= :cat=operator
{^= :cat=operator
{<- :cat=operator
{> :cat=operator
{>= :cat=operator
{{ :cat=punct :block=open
{} :cat=punct :block=close
{/ :cat=operator
:LST_NULL___slash @/>LST_sl_comment
:LST_NULL___slash @*>LST_ml_comment
{<< :cat=operator
{/= :cat=operator
{<<= :cat=operator
{++ :cat=operator
{= :cat=operator
{, :cat=operator
{; :cat=punct
{% :cat=operator
{>> :cat=operator
{%= :cat=operator
{>>= :cat=operator
{-- :cat=operator
{! :cat=operator
{? :cat=operator
{... :cat=operator
{. :cat=operator +num_chars>LST_float
{: :cat=operator

# types
{_Bool :cat=typespec |identifier_chars>LST_identifier
{_Imaginary :cat=typespec |identifier_chars>LST_identifier
{_Thread_local :cat=storespec |identifier_chars>LST_identifier
{_Complex :cat=typespec |identifier_chars>LST_identifier
{register :cat=storespec |identifier_chars>LST_identifier
{thread_local :cat=storespec |identifier_chars>LST_identifier
{volatile :cat=storespec |identifier_chars>LST_identifier
{const :cat=typespec |identifier_chars>LST_identifier
{mutable :cat=typespec |identifier_chars>LST_identifier
{signed :cat=typespec |identifier_chars>LST_identifier
{unsigned :cat=typespec |identifier_chars>LST_identifier
{auto :cat=type |identifier_chars>LST_identifier
{bool :cat=type |identifier_chars>LST_identifier
{uint8_t :cat=type |identifier_chars>LST_identifier
{uint16_t :cat=type |identifier_chars>LST_identifier
{uint32_t :cat=type |identifier_chars>LST_identifier
{uint64_t :cat=type |identifier_chars>LST_identifier
{int8_t :cat=type |identifier_chars>LST_identifier
{int16_t :cat=type |identifier_chars>LST_identifier
{int32_t :cat=type |identifier_chars>LST_identifier
{int64_t :cat=type |identifier_chars>LST_identifier
{char16_t :cat=type |identifier_chars>LST_identifier
{char32_t :cat=type |identifier_chars>LST_identifier
{float :cat=type |identifier_chars>LST_identifier
{double :cat=type |identifier_chars>LST_identifier
{char :cat=type |identifier_chars>LST_identifier
{short :cat=type |identifier_chars>LST_identifier
{int :cat=type |identifier_chars>LST_identifier
{long :cat=type |identifier_chars>LST_identifier
# no support for spaces yet {"long double" :cat=type |identifier_chars>LST_identifier
# no support for spaces yet {"long long" :cat=type |identifier_chars>LST_identifier
{intptr_t :cat=type |identifier_chars>LST_identifier
{ptrdiff_t :cat=type |identifier_chars>LST_identifier
{size_t :cat=type |identifier_chars>LST_identifier
{void :cat=type |identifier_chars>LST_identifier
{wchar_t :cat=type |identifier_chars>LST_identifier
{__int64 :cat=type |identifier_chars>LST_identifier
{__m64 :cat=type |identifier_chars>LST_identifier
{__m128 :cat=type |identifier_chars>LST_identifier
{__m128i :cat=type |identifier_chars>LST_identifier
{__m128d :cat=type |identifier_chars>LST_identifier
{__m256 :cat=type |identifier_chars>LST_identifier
{__m256i :cat=type |identifier_chars>LST_identifier
{__m256d :cat=type |identifier_chars>LST_identifier
{__m512 :cat=type |identifier_chars>LST_identifier
{__m512i :cat=type |identifier_chars>LST_identifier
{__m512d :cat=type |identifier_chars>LST_identifier
{__mmask8 :cat=type |identifier_chars>LST_identifier
{__mmask16 :cat=type |identifier_chars>LST_identifier
{__mmask32 :cat=type |identifier_chars>LST_identifier
{__mmask64 :cat=type |identifier_chars>LST_identifier


# mark some terminal states
&LST_float
&LST_probenum
&LST_intnum
&LST_octalnum
&LST_hexnum
&LST_identifier
&LST_float_exp

# all other identifiers
# + is transition on a character set
:LST_identifier :cat=identifier +identifier_chars>LST_identifier
:LST_NULL +identifier_schars>LST_identifier

# ignore whitespace
# ^ discards the char while moving to the next state
:LST_NULL +ws^LST_NULL

# strings
# @ is a single-char transition
:LST_string @"=LST_string
:LST_NULL @">LST_string
&LST_string_end
:LST_string @\\>LST_string_esc
:LST_string !+slash>LST_string
:LST_string_esc !+quote>LST_string

# char literals
:LST_charlit @'=LST_charlit
:LST_NULL @'>LST_charlit
&LST_charlit_end
:LST_charlit @\\>LST_charlit_esc
:LST_charlit !+slash>LST_charlit
:LST_charlit_esc !+squote>LST_charlit

# numbers
:LST_NULL @0>LST_probenum
:LST_NULL @1>LST_intnum
:LST_NULL @2>LST_intnum
:LST_NULL @3>LST_intnum
:LST_NULL @4>LST_intnum
:LST_NULL @5>LST_intnum
:LST_NULL @6>LST_intnum
:LST_NULL @7>LST_intnum
:LST_NULL @8>LST_intnum
:LST_NULL @9>LST_intnum

:LST_probenum @.>LST_float
:LST_probenum @x>LST_hexnum
:LST_probenum @X>LST_hexnum
:LST_probefixed @x>LST_hexnum
:LST_probefixed @X>LST_hexnum
:LST_probenum +num_chars>LST_octalnum
:LST_probefixed +num_chars>LST_octalnum
:LST_intnum @.>LST_float
:LST_intnum +num_chars>LST_intnum
:LST_octalnum +octal_chars>LST_octalnum
:LST_hexnum +hex_chars>LST_hexnum

:LST_float @.>LST_INVALID
:LST_float +num_chars>LST_float
:LST_float @e>LST_float_exp_start
:LST_float @E>LST_float_exp_start
:LST_float @d=LST_float
:LST_float @f=LST_float
:LST_float_exp_start +num_chars>LST_float_exp
:LST_float_exp_start @->LST_float_exp
:LST_float_exp_start @+>LST_float_exp
:LST_float_exp +num_chars>LST_float_exp
:LST_float_exp @d=LST_float
:LST_float_exp @f=LST_float

# comments
# ! inverts a character set
# = finishes a token, including the character tested
# ~ finishes a token but does not consume the character tested
&LST_sl_comment 
:LST_sl_comment !+endl>LST_sl_comment
:LST_sl_comment @\r~LST_sl_comment
:LST_sl_comment @\n~LST_sl_comment
&LST_ml_comment
:LST_ml_comment @*>LST_ml_comment_star
:LST_ml_comment !+star>LST_ml_comment
:LST_ml_comment_star @/=LST_ml_comment
:LST_ml_comment_star !+slash>LST_ml_comment

&LST_preproc
:LST_NULL @#>LST_preproc
:LST_preproc !+endl>LST_preproc
:LST_preproc @\r~LST_preproc
:LST_preproc @\n~LST_preproc