tokenfmt 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. #!/usr/bin/env perl
  2. use strict;
  3. use warnings;
  4. my (%token, @order, @keywords);
  5. {
  6. my $keywords;
  7. my @const;
  8. push @const, <<_END_;
  9. package token
  10. const(
  11. _ Token = iota
  12. _END_
  13. for (split m/\n/, <<_END_) {
  14. ILLEGAL
  15. EOF
  16. COMMENT
  17. KEYWORD
  18. STRING
  19. BOOLEAN
  20. NULL
  21. NUMBER
  22. IDENTIFIER
  23. PLUS +
  24. MINUS -
  25. MULTIPLY *
  26. SLASH /
  27. REMAINDER %
  28. AND &
  29. OR |
  30. EXCLUSIVE_OR ^
  31. SHIFT_LEFT <<
  32. SHIFT_RIGHT >>
  33. UNSIGNED_SHIFT_RIGHT >>>
  34. AND_NOT &^
  35. ADD_ASSIGN +=
  36. SUBTRACT_ASSIGN -=
  37. MULTIPLY_ASSIGN *=
  38. QUOTIENT_ASSIGN /=
  39. REMAINDER_ASSIGN %=
  40. AND_ASSIGN &=
  41. OR_ASSIGN |=
  42. EXCLUSIVE_OR_ASSIGN ^=
  43. SHIFT_LEFT_ASSIGN <<=
  44. SHIFT_RIGHT_ASSIGN >>=
  45. UNSIGNED_SHIFT_RIGHT_ASSIGN >>>=
  46. AND_NOT_ASSIGN &^=
  47. LOGICAL_AND &&
  48. LOGICAL_OR ||
  49. INCREMENT ++
  50. DECREMENT --
  51. EQUAL ==
  52. STRICT_EQUAL ===
  53. LESS <
  54. GREATER >
  55. ASSIGN =
  56. NOT !
  57. BITWISE_NOT ~
  58. NOT_EQUAL !=
  59. STRICT_NOT_EQUAL !==
  60. LESS_OR_EQUAL <=
  61. GREATER_OR_EQUAL <=
  62. LEFT_PARENTHESIS (
  63. LEFT_BRACKET [
  64. LEFT_BRACE {
  65. COMMA ,
  66. PERIOD .
  67. RIGHT_PARENTHESIS )
  68. RIGHT_BRACKET ]
  69. RIGHT_BRACE }
  70. SEMICOLON ;
  71. COLON :
  72. QUESTION_MARK ?
  73. firstKeyword
  74. IF
  75. IN
  76. DO
  77. VAR
  78. FOR
  79. NEW
  80. TRY
  81. THIS
  82. ELSE
  83. CASE
  84. VOID
  85. WITH
  86. WHILE
  87. BREAK
  88. CATCH
  89. THROW
  90. RETURN
  91. TYPEOF
  92. DELETE
  93. SWITCH
  94. DEFAULT
  95. FINALLY
  96. FUNCTION
  97. CONTINUE
  98. DEBUGGER
  99. INSTANCEOF
  100. lastKeyword
  101. _END_
  102. chomp;
  103. next if m/^\s*#/;
  104. my ($name, $symbol) = m/(\w+)\s*(\S+)?/;
  105. if (defined $symbol) {
  106. push @order, $name;
  107. push @const, "$name // $symbol";
  108. $token{$name} = $symbol;
  109. } elsif (defined $name) {
  110. $keywords ||= $name eq 'firstKeyword';
  111. push @const, $name;
  112. #$const[-1] .= " Token = iota" if 2 == @const;
  113. if ($name =~ m/^([A-Z]+)/) {
  114. push @keywords, $name if $keywords;
  115. push @order, $name;
  116. if ($token{SEMICOLON}) {
  117. $token{$name} = lc $1;
  118. } else {
  119. $token{$name} = $name;
  120. }
  121. }
  122. } else {
  123. push @const, "";
  124. }
  125. }
  126. push @const, ")";
  127. print join "\n", @const, "";
  128. }
  129. {
  130. print <<_END_;
  131. var token2string = [...]string{
  132. _END_
  133. for my $name (@order) {
  134. print "$name: \"$token{$name}\",\n";
  135. }
  136. print <<_END_;
  137. }
  138. _END_
  139. print <<_END_;
  140. var keywordTable = map[string]_keyword{
  141. _END_
  142. for my $name (@keywords) {
  143. print <<_END_
  144. "@{[ lc $name ]}": _keyword{
  145. token: $name,
  146. },
  147. _END_
  148. }
  149. for my $name (qw/
  150. const
  151. class
  152. enum
  153. export
  154. extends
  155. import
  156. super
  157. /) {
  158. print <<_END_
  159. "$name": _keyword{
  160. token: KEYWORD,
  161. futureKeyword: true,
  162. },
  163. _END_
  164. }
  165. for my $name (qw/
  166. implements
  167. interface
  168. let
  169. package
  170. private
  171. protected
  172. public
  173. static
  174. /) {
  175. print <<_END_
  176. "$name": _keyword{
  177. token: KEYWORD,
  178. futureKeyword: true,
  179. strict: true,
  180. },
  181. _END_
  182. }
  183. print <<_END_;
  184. }
  185. _END_
  186. }