sti/parser/js_tokens.txt

# character sets
[identifier_schars a-zA-Z_
[identifier_chars a-zA-Z0-9_
[num_chars_start 0-9
[num_chars 0-9
[octal_chars 0-7
[hex_chars 0-9a-fA-F
[float_chars_start 0-9.\-\+
[float_chars 0-9.eE
[float_chars_start 0-9.\-\+
[ws \ \t\r\n
[endl \r\n
# single-character transitions can't be inverted yet, but csets can 
[slash \\
[star *

# keywords
{await :cat=keyword |identifier_chars>LST_identifier
{break :cat=keyword |identifier_chars>LST_identifier
{case :cat=keyword |identifier_chars>LST_identifier
{catch :cat=keyword |identifier_chars>LST_identifier
{class :cat=keyword |identifier_chars>LST_identifier
{const :cat=keyword |identifier_chars>LST_identifier
{continue :cat=keyword |identifier_chars>LST_identifier
{debugger :cat=keyword |identifier_chars>LST_identifier
{default :cat=keyword |identifier_chars>LST_identifier
{delete :cat=keyword |identifier_chars>LST_identifier
{do :cat=keyword |identifier_chars>LST_identifier
{else :cat=keyword |identifier_chars>LST_identifier
{export :cat=keyword |identifier_chars>LST_identifier
{extends :cat=keyword |identifier_chars>LST_identifier
{false :cat=keyword |identifier_chars>LST_identifier
{finally :cat=keyword |identifier_chars>LST_identifier
{for :cat=keyword |identifier_chars>LST_identifier
{function :cat=keyword |identifier_chars>LST_identifier
{if :cat=keyword |identifier_chars>LST_identifier
{import :cat=keyword |identifier_chars>LST_identifier
{in :cat=keyword |identifier_chars>LST_identifier
{instanceof :cat=keyword |identifier_chars>LST_identifier
{let :cat=keyword |identifier_chars>LST_identifier
{new :cat=keyword |identifier_chars>LST_identifier
{null :cat=keyword |identifier_chars>LST_identifier
{return :cat=keyword |identifier_chars>LST_identifier
{of :cat=keyword |identifier_chars>LST_identifier
{super :cat=keyword |identifier_chars>LST_identifier
{switch :cat=keyword |identifier_chars>LST_identifier
{this :cat=keyword |identifier_chars>LST_identifier
{throw :cat=keyword |identifier_chars>LST_identifier
{true :cat=keyword |identifier_chars>LST_identifier
{try :cat=keyword |identifier_chars>LST_identifier
{typeof :cat=keyword |identifier_chars>LST_identifier
{var :cat=keyword |identifier_chars>LST_identifier
{void :cat=keyword |identifier_chars>LST_identifier
{while :cat=keyword |identifier_chars>LST_identifier
{with :cat=keyword |identifier_chars>LST_identifier
{yield :cat=keyword |identifier_chars>LST_identifier


# operators and punctuation
{+ :cat=operator
{& :cat=operator
{+= :cat=operator
{&= :cat=operator
{&& :cat=operator
{== :cat=operator
{=== :cat=operator
{!= :cat=operator
{!== :cat=operator
{?? :cat=operator
{( :cat=punct
{) :cat=punct
{- :cat=operator
{| :cat=operator
{-= :cat=operator
{|= :cat=operator
{|| :cat=operator
{< :cat=operator
{<= :cat=operator
{[ :cat=punct
{] :cat=punct
{* :cat=operator
{** :cat=operator
{**- :cat=operator
{^ :cat=operator
{*= :cat=operator
{^= :cat=operator
{<- :cat=operator
{> :cat=operator
{>= :cat=operator
{{ :cat=punct
{} :cat=punct
{/ :cat=operator
:LST_NULL___slash @/>LST_sl_comment
:LST_NULL___slash @*>LST_ml_comment
{<< :cat=operator
{<<< :cat=operator
{/= :cat=operator
{<<= :cat=operator
{<<<= :cat=operator
{++ :cat=operator
{= :cat=operator
{, :cat=operator
{; :cat=punct
{% :cat=operator
{>> :cat=operator
{>>> :cat=operator
{%= :cat=operator
{>>= :cat=operator
{>>>= :cat=operator
{-- :cat=operator
{! :cat=operator
{? :cat=operator
{... :cat=operator
{. :cat=operator +num_chars>LST_float
{: :cat=operator
{:: :cat=operator




# mark some terminal states
&LST_float :cat=number
&LST_probenum :cat=number
&LST_intnum :cat=number
&LST_octalnum :cat=number
&LST_hexnum :cat=number
&LST_identifier :cat=identifier
&LST_float_exp :cat=number

# all other identifiers
# + is transition on a character set
:LST_identifier +identifier_chars>LST_identifier
:LST_NULL +identifier_schars>LST_identifier

# ignore whitespace
# ^ discards the char while moving to the next state
:LST_NULL +ws^LST_NULL

# double-quote strings
# @ is a single-char transition
:LST_string @"=LST_string
:LST_NULL @">LST_string
&LST_string_end :cat=string
:LST_string @\\>LST_string_esc
:LST_string !+slash>LST_string
:LST_string_esc @">LST_string


# single-quote strings
:LST_sq_string @'=LST_sq_string
:LST_NULL @'>LST_sq_string
&LST_sq_string_end :cat=string
:LST_sq_string @\\>LST_sq_string_esc
:LST_sq_string !+slash>LST_sq_string
:LST_sq_string_esc @'>LST_sq_string


# regex literals
#:LST___slash @'=LST_sq_string
#:LST_NULL @/>LST_sq_string
#&LST_sq_string_end :cat=regex
#:LST_sq_string @\\>LST_sq_string_esc
#:LST_sq_string !+slash>LST_sq_string
#:LST_sq_string_esc @'>LST_sq_string

# numbers
:LST_NULL @0>LST_probenum
:LST_NULL @1>LST_intnum
:LST_NULL @2>LST_intnum
:LST_NULL @3>LST_intnum
:LST_NULL @4>LST_intnum
:LST_NULL @5>LST_intnum
:LST_NULL @6>LST_intnum
:LST_NULL @7>LST_intnum
:LST_NULL @8>LST_intnum
:LST_NULL @9>LST_intnum

:LST_probenum @.>LST_float
:LST_probenum @x>LST_hexnum
:LST_probenum @X>LST_hexnum
:LST_probefixed @x>LST_hexnum
:LST_probefixed @X>LST_hexnum
:LST_probenum +num_chars>LST_octalnum
:LST_probefixed +num_chars>LST_octalnum
:LST_intnum @.>LST_float
:LST_intnum +num_chars>LST_intnum
:LST_octalnum +octal_chars>LST_octalnum
:LST_hexnum +hex_chars>LST_hexnum

:LST_float @.>LST_INVALID
:LST_float +num_chars>LST_float
:LST_float @e>LST_float_exp_start
:LST_float @E>LST_float_exp_start
:LST_float @d=LST_float
:LST_float @f=LST_float
:LST_float_exp_start +num_chars>LST_float_exp
:LST_float_exp_start @->LST_float_exp
:LST_float_exp_start @+>LST_float_exp
:LST_float_exp +num_chars>LST_float_exp
:LST_float_exp @d=LST_float
:LST_float_exp @f=LST_float

# comments
# ! inverts a character set
# = finishes a token, including the character tested
# ~ finishes a token but does not consume the character tested
&LST_sl_comment :cat=comment
:LST_sl_comment !+endl>LST_sl_comment
:LST_sl_comment @\r~LST_sl_comment
:LST_sl_comment @\n~LST_sl_comment
&LST_ml_comment :cat=comment
:LST_ml_comment @*>LST_ml_comment_star
:LST_ml_comment !+star>LST_ml_comment
:LST_ml_comment_star @/=LST_ml_comment
:LST_ml_comment_star !+slash>LST_ml_comment