levovix
/
Nim
同期ミラー https://github.com/nim-lang/Nim


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364
							<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!--  This file is generated by Nim. -->
<html xmlns="https://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-theme="auto">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>packages/docutils/highlite</title>

<!-- Google fonts -->
<link href='https://fonts.googleapis.com/css?family=Lato:400,600,900' rel='stylesheet' type='text/css'/>
<link href='https://fonts.googleapis.com/css?family=Source+Code+Pro:400,500,600' rel='stylesheet' type='text/css'/>

<!-- Favicon -->
<link rel="shortcut icon" href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAUAAAAF////AP///wD///8A////AP///wD///8A////AP///wD///8A////AAAAAAIAAABbAAAAlQAAAKIAAACbAAAAmwAAAKIAAACVAAAAWwAAAAL///8A////AP///wD///8A////AAAAABQAAADAAAAAYwAAAA3///8A////AP///wD///8AAAAADQAAAGMAAADAAAAAFP///wD///8A////AP///wAAAACdAAAAOv///wD///8A////AP///wD///8A////AP///wD///8AAAAAOgAAAJ3///8A////AP///wAAAAAnAAAAcP///wAAAAAoAAAASv///wD///8A////AP///wAAAABKAAAAKP///wAAAABwAAAAJ////wD///8AAAAAgQAAABwAAACIAAAAkAAAAJMAAACtAAAAFQAAABUAAACtAAAAkwAAAJAAAACIAAAAHAAAAIH///8A////AAAAAKQAAACrAAAAaP///wD///8AAAAARQAAANIAAADSAAAARf///wD///8AAAAAaAAAAKsAAACk////AAAAADMAAACcAAAAnQAAABj///8A////AP///wAAAAAYAAAAGP///wD///8A////AAAAABgAAACdAAAAnAAAADMAAAB1AAAAwwAAAP8AAADpAAAAsQAAAE4AAAAb////AP///wAAAAAbAAAATgAAALEAAADpAAAA/wAAAMMAAAB1AAAAtwAAAOkAAAD/AAAA/wAAAP8AAADvAAAA3gAAAN4AAADeAAAA3gAAAO8AAAD/AAAA/wAAAP8AAADpAAAAtwAAAGUAAAA/AAAA3wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAAD/AAAA/wAAAP8AAADfAAAAPwAAAGX///8A////AAAAAEgAAADtAAAAvwAAAL0AAADGAAAA7wAAAO8AAADGAAAAvQAAAL8AAADtAAAASP///wD///8A////AP///wD///8AAAAAO////wD///8A////AAAAAIcAAACH////AP///wD///8AAAAAO////wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A////AP///wD///8A//8AAP//AAD4HwAA7/cAAN/7AAD//wAAoYUAAJ55AACf+QAAh+EAAAAAAADAAwAA4AcAAP5/AAD//wAA//8AAA=="/>
<link rel="icon" type="image/png" sizes="32x32" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAA3XAAAN1wFCKJt4AAAAB3RJTUUH4QQQEwksSS9ZWwAAAk1JREFUWMPtll2ITVEUx39nn/O7Y5qR8f05wtCUUr6ZIS++8pEnkZInPImneaCQ5METNdOkeFBKUhMPRIkHKfEuUZSUlGlKPN2TrgfncpvmnntnmlEyq1Z7t89/rf9a6+y99oZxGZf/XeIq61EdtgKXgdXA0xrYAvBjOIF1AI9zvjcC74BSpndrJPkBWDScTF8Aa4E3wDlgHbASaANmVqlcCnwHvgDvgVfAJ+AikAAvgfVZwLnSVZHZaOuKoQi3ZOMi4NkYkpe1p4J7A8BpYAD49hfIy/oqG0+hLomiKP2L5L+1ubn5115S+3OAn4EnwBlgMzCjyt6ZAnQCJ4A7wOs88iRJHvw50HoujuPBoCKwHWiosy8MdfZnAdcHk8dxXFJ3VQbQlCTJvRBCGdRbD4M6uc5glpY3eAihpN5S5w12diSEcCCEcKUO4ljdr15T76ur1FDDLIQQ3qv71EdDOe3Kxj3leRXyk+pxdWnFWod6Wt2bY3de3aSuUHcPBVimHs7mK9WrmeOF6lR1o9qnzskh2ar2qm1qizpfXaPeVGdlmGN5pb09qMxz1Xb1kLqgzn1RyH7JUXW52lr5e/Kqi9qpto7V1atuUzfnARrV7jEib1T76gG2qxdGmXyiekkt1GswPTtek0aBfJp6YySGBfWg2tPQ0FAYgf1stUfdmdcjarbYJEniKIq6gY/Aw+zWHAC+p2labGpqiorFYgGYCEzN7oQdQClN07O1/EfDyGgC0ALMBdYAi4FyK+4H3gLPsxfR1zRNi+NP7nH5J+QntnXe5B5mpfQAAAAASUVORK5CYII=">

<!-- CSS -->
<link rel="stylesheet" type="text/css" href="nimdoc.out.css?v=2.3.1">

<!-- JS -->
<script type="text/javascript" src="dochack.js?v=2.3.1"></script>
</head>
<body>
  <div class="document" id="documentId">
    <div class="container">
      <h1 class="title">packages/docutils/highlite</h1>
      <div class="row">
  <div class="three columns">
    <div class="theme-select-wrapper">
      <label for="theme-select">Theme:&nbsp;</label>
      <select id="theme-select" onchange="setTheme(this.value)">
        <option value="auto">🌗 Match OS</option>
        <option value="dark">🌑 Dark</option>
        <option value="light">🌕 Light</option>
      </select>
    </div>
    <div id="global-links">
      <ul class="simple-boot">
        <li><a href="manual.html">Manual</a></li>
        <li><a href="lib.html">Standard library</a></li>
        <li> <a id="indexLink" href="theindex.html">Index</a></li>
        <li><a href="compiler/theindex.html">Compiler docs</a></li>
        <li><a href="https://nim-lang.github.io/fusion/theindex.html">Fusion docs</a></li>
        <li><a href="https://nim-lang.github.io/Nim/">devel</a>, <a href="https://nim-lang.org/documentation.html">stable</a></li>
      </ul>
    </div>
    <div id="searchInputDiv">
      Search: <input type="search" id="searchInput"
        oninput="search()" />
    </div>
    
    <ul class="simple simple-toc" id="toc-list">
  <li>
  <a class="reference reference-toplevel" href="#6" id="56">Imports</a>
</li>
<li>
  <details open>
    <summary><a class="reference reference-toplevel" href="#7" id="57">Types</a></summary>
    <ul class="simple simple-toc-section">
      <li><a class="reference" href="#GeneralTokenizer" title="GeneralTokenizer = object of RootObj
  kind*: TokenClass
  start*, length*: int">GeneralTokenizer</a></li>
<li><a class="reference" href="#SourceLanguage" title="SourceLanguage = enum
  langNone, langNim, langCpp, langCsharp, langC, langJava, langYaml, langPython,
  langCmd, langConsole">SourceLanguage</a></li>
<li><a class="reference" href="#TokenClass" title="TokenClass = enum
  gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber,
  gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit,
  gtLongStringLit, gtCharLit, gtEscapeSequence, gtOperator, gtPunctuation,
  gtComment, gtLongComment, gtRegularExpression, gtTagStart, gtTagEnd, gtKey,
  gtValue, gtRawData, gtAssembler, gtPreprocessor, gtDirective, gtCommand,
  gtRule, gtHyperlink, gtLabel, gtReference, gtPrompt, gtProgramOutput,
  gtProgram, gtOption, gtOther">TokenClass</a></li>

    </ul>
  </details>
</li>
<li>
  <details open>
    <summary><a class="reference reference-toplevel" href="#10" id="60">Consts</a></summary>
    <ul class="simple simple-toc-section">
      <li><a class="reference" href="#sourceLanguageToAlpha" title="sourceLanguageToAlpha: array[SourceLanguage, string] = [&quot;none&quot;, &quot;Nim&quot;, &quot;cpp&quot;,
    &quot;csharp&quot;, &quot;C&quot;, &quot;Java&quot;, &quot;Yaml&quot;, &quot;Python&quot;, &quot;Cmd&quot;, &quot;Console&quot;]">sourceLanguageToAlpha</a></li>
<li><a class="reference" href="#sourceLanguageToStr" title="sourceLanguageToStr: array[SourceLanguage, string] = [&quot;none&quot;, &quot;Nim&quot;, &quot;C++&quot;,
    &quot;C#&quot;, &quot;C&quot;, &quot;Java&quot;, &quot;Yaml&quot;, &quot;Python&quot;, &quot;Cmd&quot;, &quot;Console&quot;]">sourceLanguageToStr</a></li>
<li><a class="reference" href="#tokenClassToStr" title="tokenClassToStr: array[TokenClass, string] = [&quot;Eof&quot;, &quot;None&quot;, &quot;Whitespace&quot;,
    &quot;DecNumber&quot;, &quot;BinNumber&quot;, &quot;HexNumber&quot;, &quot;OctNumber&quot;, &quot;FloatNumber&quot;,
    &quot;Identifier&quot;, &quot;Keyword&quot;, &quot;StringLit&quot;, &quot;LongStringLit&quot;, &quot;CharLit&quot;,
    &quot;EscapeSequence&quot;, &quot;Operator&quot;, &quot;Punctuation&quot;, &quot;Comment&quot;, &quot;LongComment&quot;,
    &quot;RegularExpression&quot;, &quot;TagStart&quot;, &quot;TagEnd&quot;, &quot;Key&quot;, &quot;Value&quot;, &quot;RawData&quot;,
    &quot;Assembler&quot;, &quot;Preprocessor&quot;, &quot;Directive&quot;, &quot;Command&quot;, &quot;Rule&quot;, &quot;Hyperlink&quot;,
    &quot;Label&quot;, &quot;Reference&quot;, &quot;Prompt&quot;, &quot;ProgramOutput&quot;, &quot;program&quot;, &quot;option&quot;,
    &quot;Other&quot;]">tokenClassToStr</a></li>

    </ul>
  </details>
</li>
<li>
  <details open>
    <summary><a class="reference reference-toplevel" href="#12" id="62">Procs</a></summary>
    <ul class="simple simple-toc-section">
      <ul class="simple nested-toc-section">deinitGeneralTokenizer
  <li><a class="reference" href="#deinitGeneralTokenizer%2CGeneralTokenizer" title="deinitGeneralTokenizer(g: var GeneralTokenizer)">deinitGeneralTokenizer(g: var GeneralTokenizer)</a></li>

</ul>
<ul class="simple nested-toc-section">getNextToken
  <li><a class="reference" href="#getNextToken%2CGeneralTokenizer%2CSourceLanguage" title="getNextToken(g: var GeneralTokenizer; lang: SourceLanguage)">getNextToken(g: var GeneralTokenizer; lang: SourceLanguage)</a></li>

</ul>
<ul class="simple nested-toc-section">getSourceLanguage
  <li><a class="reference" href="#getSourceLanguage%2Cstring" title="getSourceLanguage(name: string): SourceLanguage">getSourceLanguage(name: string): SourceLanguage</a></li>

</ul>
<ul class="simple nested-toc-section">initGeneralTokenizer
  <li><a class="reference" href="#initGeneralTokenizer%2CGeneralTokenizer%2Ccstring" title="initGeneralTokenizer(g: var GeneralTokenizer; buf: cstring)">initGeneralTokenizer(g: var GeneralTokenizer; buf: cstring)</a></li>
<li><a class="reference" href="#initGeneralTokenizer%2CGeneralTokenizer%2Cstring" title="initGeneralTokenizer(g: var GeneralTokenizer; buf: string)">initGeneralTokenizer(g: var GeneralTokenizer; buf: string)</a></li>

</ul>
<ul class="simple nested-toc-section">tokenize
  <li><a class="reference" href="#tokenize%2Cstring%2CSourceLanguage" title="tokenize(text: string; lang: SourceLanguage): seq[(string, TokenClass)]">tokenize(text: string; lang: SourceLanguage): seq[(string, TokenClass)]</a></li>

</ul>

    </ul>
  </details>
</li>

</ul>

  </div>
  <div class="nine columns" id="content">
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L1" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L1" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

    <div id="tocRoot"></div>
    
    <p class="module-desc"><p>Source highlighter for programming or markup languages. Currently only few languages are supported, other languages may be added. The interface supports one language nested in another.</p>
<p>You can use this to build your own syntax highlighting, check this example:</p>
<p><pre class="listing"><span class="Keyword">let</span> <span class="Identifier">code</span> <span class="Operator">=</span> <span class="LongStringLit">&quot;&quot;&quot;for x in $int.high: echo x.ord mod 2 == 0&quot;&quot;&quot;</span>
<span class="Keyword">var</span> <span class="Identifier">toknizr</span><span class="Punctuation">:</span> <span class="Identifier">GeneralTokenizer</span>
<span class="Identifier">initGeneralTokenizer</span><span class="Punctuation">(</span><span class="Identifier">toknizr</span><span class="Punctuation">,</span> <span class="Identifier">code</span><span class="Punctuation">)</span>
<span class="Keyword">while</span> <span class="Identifier">true</span><span class="Punctuation">:</span>
  <span class="Identifier">getNextToken</span><span class="Punctuation">(</span><span class="Identifier">toknizr</span><span class="Punctuation">,</span> <span class="Identifier">langNim</span><span class="Punctuation">)</span>
  <span class="Keyword">case</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">kind</span>
  <span class="Keyword">of</span> <span class="Identifier">gtEof</span><span class="Punctuation">:</span> <span class="Keyword">break</span>  <span class="Comment"># End Of File (or string)</span>
  <span class="Keyword">of</span> <span class="Identifier">gtWhitespace</span><span class="Punctuation">:</span>
    <span class="Identifier">echo</span> <span class="Identifier">gtWhitespace</span> <span class="Comment"># Maybe you want &quot;visible&quot; whitespaces?.</span>
    <span class="Identifier">echo</span> <span class="Identifier">substr</span><span class="Punctuation">(</span><span class="Identifier">code</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">length</span> <span class="Operator">+</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span> <span class="Operator">-</span> <span class="DecNumber">1</span><span class="Punctuation">)</span>
  <span class="Keyword">of</span> <span class="Identifier">gtOperator</span><span class="Punctuation">:</span>
    <span class="Identifier">echo</span> <span class="Identifier">gtOperator</span> <span class="Comment"># Maybe you want Operators to use a specific color?.</span>
    <span class="Identifier">echo</span> <span class="Identifier">substr</span><span class="Punctuation">(</span><span class="Identifier">code</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">length</span> <span class="Operator">+</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span> <span class="Operator">-</span> <span class="DecNumber">1</span><span class="Punctuation">)</span>
  <span class="Comment"># of gtSomeSymbol: syntaxHighlight(&quot;Comic Sans&quot;, &quot;bold&quot;, &quot;99px&quot;, &quot;pink&quot;)</span>
  <span class="Keyword">else</span><span class="Punctuation">:</span>
    <span class="Identifier">echo</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">kind</span> <span class="Comment"># All the kinds of tokens can be processed here.</span>
    <span class="Identifier">echo</span> <span class="Identifier">substr</span><span class="Punctuation">(</span><span class="Identifier">code</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span><span class="Punctuation">,</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">length</span> <span class="Operator">+</span> <span class="Identifier">toknizr</span><span class="Operator">.</span><span class="Identifier">start</span> <span class="Operator">-</span> <span class="DecNumber">1</span><span class="Punctuation">)</span></pre></p>
<p>The proc <tt class="docutils literal"><span class="pre"><span class="Identifier">getSourceLanguage</span></span></tt> can get the language <tt class="docutils literal"><span class="pre"><span class="Keyword">enum</span></span></tt> from a string:</p>
<p><pre class="listing"><span class="Keyword">for</span> <span class="Identifier">l</span> <span class="Keyword">in</span> <span class="Punctuation">[</span><span class="StringLit">&quot;C&quot;</span><span class="Punctuation">,</span> <span class="StringLit">&quot;c++&quot;</span><span class="Punctuation">,</span> <span class="StringLit">&quot;jAvA&quot;</span><span class="Punctuation">,</span> <span class="StringLit">&quot;Nim&quot;</span><span class="Punctuation">,</span> <span class="StringLit">&quot;c#&quot;</span><span class="Punctuation">]</span><span class="Punctuation">:</span> <span class="Identifier">echo</span> <span class="Identifier">getSourceLanguage</span><span class="Punctuation">(</span><span class="Identifier">l</span><span class="Punctuation">)</span></pre></p>
<p>There is also a <tt class="docutils literal"><span class="pre"><span class="Identifier">Cmd</span></span></tt> pseudo-language supported, which is a simple generic shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command): no escaping, no programming language constructs besides variable definition at the beginning of line. It supports these operators:</p>
<p><pre class="listing"><span class="Operator">&amp;</span>  <span class="Operator">&amp;&amp;</span>  <span class="Operator">|</span>  <span class="Operator">||</span>  <span class="Operator">(</span>  <span class="Operator">)</span>  <span class="option">''</span>  <span class="option">&quot;&quot;</span>  <span class="Operator">;</span>  <span class="Comment"># for comments</span></pre></p>
<p>Instead of escaping always use quotes like here <tt class="docutils literal"><span class="pre"><span class="program">nimgrep</span> <span class="option">--ext:</span><span class="option">'nim|nims'</span> <span class="Identifier">file.name</span></span></tt> shows how to input <tt class="docutils literal"><span class="pre">|</span></tt>. Any argument that contains <tt class="docutils literal"><span class="pre">.</span></tt> or <tt class="docutils literal"><span class="pre">/</span></tt> or <tt class="docutils literal"><span class="pre">\</span></tt> will be treated as a file or directory.</p>
<p>In addition to <tt class="docutils literal"><span class="pre"><span class="Identifier">Cmd</span></span></tt> there is also <tt class="docutils literal"><span class="pre"><span class="Identifier">Console</span></span></tt> language for displaying interactive sessions. Lines with a command should start with <tt class="docutils literal"><span class="pre">$</span></tt>, other lines are considered as program output.</p>
</p>
    <div class="section" id="6">
  <h1><a class="toc-backref" href="#6">Imports</a></h1>
  <dl class="item">
    <a class="reference external" href="strutils.html">strutils</a>, <a class="reference external" href="algorithm.html">algorithm</a>, <a class="reference external" href="assertions.html">assertions</a>, <a class="reference external" href="syncio.html">syncio</a>
  </dl>
</div>
<div class="section" id="7">
  <h1><a class="toc-backref" href="#7">Types</a></h1>
  <dl class="item">
    <div id="GeneralTokenizer">
  <dt><pre><a href="highlite.html#GeneralTokenizer"><span class="Identifier">GeneralTokenizer</span></a> <span class="Other">=</span> <span class="Keyword">object</span> <span class="Keyword">of</span> <a href="system.html#RootObj"><span class="Identifier">RootObj</span></a>
  <span class="Identifier">kind</span><span class="Operator">*</span><span class="Other">:</span> <a href="highlite.html#TokenClass"><span class="Identifier">TokenClass</span></a>
  <span class="Identifier">start</span><span class="Operator">*</span><span class="Other">,</span> <span class="Identifier">length</span><span class="Operator">*</span><span class="Other">:</span> <a href="system.html#int"><span class="Identifier">int</span></a></pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L79" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L79" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>
<div id="SourceLanguage">
  <dt><pre><a href="highlite.html#SourceLanguage"><span class="Identifier">SourceLanguage</span></a> <span class="Other">=</span> <span class="Keyword">enum</span>
  <span class="Identifier">langNone</span><span class="Other">,</span> <span class="Identifier">langNim</span><span class="Other">,</span> <span class="Identifier">langCpp</span><span class="Other">,</span> <span class="Identifier">langCsharp</span><span class="Other">,</span> <span class="Identifier">langC</span><span class="Other">,</span> <span class="Identifier">langJava</span><span class="Other">,</span> <span class="Identifier">langYaml</span><span class="Other">,</span> <span class="Identifier">langPython</span><span class="Other">,</span>
  <span class="Identifier">langCmd</span><span class="Other">,</span> <span class="Identifier">langConsole</span></pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L68" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L68" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>
<div id="TokenClass">
  <dt><pre><a href="highlite.html#TokenClass"><span class="Identifier">TokenClass</span></a> <span class="Other">=</span> <span class="Keyword">enum</span>
  <span class="Identifier">gtEof</span><span class="Other">,</span> <span class="Identifier">gtNone</span><span class="Other">,</span> <span class="Identifier">gtWhitespace</span><span class="Other">,</span> <span class="Identifier">gtDecNumber</span><span class="Other">,</span> <span class="Identifier">gtBinNumber</span><span class="Other">,</span> <span class="Identifier">gtHexNumber</span><span class="Other">,</span>
  <span class="Identifier">gtOctNumber</span><span class="Other">,</span> <span class="Identifier">gtFloatNumber</span><span class="Other">,</span> <span class="Identifier">gtIdentifier</span><span class="Other">,</span> <span class="Identifier">gtKeyword</span><span class="Other">,</span> <span class="Identifier">gtStringLit</span><span class="Other">,</span>
  <span class="Identifier">gtLongStringLit</span><span class="Other">,</span> <span class="Identifier">gtCharLit</span><span class="Other">,</span> <span class="Identifier">gtEscapeSequence</span><span class="Other">,</span> <span class="Identifier">gtOperator</span><span class="Other">,</span> <span class="Identifier">gtPunctuation</span><span class="Other">,</span>
  <span class="Identifier">gtComment</span><span class="Other">,</span> <span class="Identifier">gtLongComment</span><span class="Other">,</span> <span class="Identifier">gtRegularExpression</span><span class="Other">,</span> <span class="Identifier">gtTagStart</span><span class="Other">,</span> <span class="Identifier">gtTagEnd</span><span class="Other">,</span> <span class="Identifier">gtKey</span><span class="Other">,</span>
  <span class="Identifier">gtValue</span><span class="Other">,</span> <span class="Identifier">gtRawData</span><span class="Other">,</span> <span class="Identifier">gtAssembler</span><span class="Other">,</span> <span class="Identifier">gtPreprocessor</span><span class="Other">,</span> <span class="Identifier">gtDirective</span><span class="Other">,</span> <span class="Identifier">gtCommand</span><span class="Other">,</span>
  <span class="Identifier">gtRule</span><span class="Other">,</span> <span class="Identifier">gtHyperlink</span><span class="Other">,</span> <span class="Identifier">gtLabel</span><span class="Other">,</span> <span class="Identifier">gtReference</span><span class="Other">,</span> <span class="Identifier">gtPrompt</span><span class="Other">,</span> <span class="Identifier">gtProgramOutput</span><span class="Other">,</span>
  <span class="Identifier">gtProgram</span><span class="Other">,</span> <span class="Identifier">gtOption</span><span class="Other">,</span> <span class="Identifier">gtOther</span></pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L71" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L71" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

  </dl>
</div>
<div class="section" id="10">
  <h1><a class="toc-backref" href="#10">Consts</a></h1>
  <dl class="item">
    <div id="sourceLanguageToAlpha">
  <dt><pre><a href="highlite.html#sourceLanguageToAlpha"><span class="Identifier">sourceLanguageToAlpha</span></a><span class="Other">:</span> <a href="system.html#array"><span class="Identifier">array</span></a><span class="Other">[</span><span class="Identifier">SourceLanguage</span><span class="Other">,</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">]</span> <span class="Other">=</span> <span class="Other">[</span><span class="StringLit">&quot;none&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Nim&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;cpp&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;csharp&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;C&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Java&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Yaml&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Python&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Cmd&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Console&quot;</span><span class="Other">]</span></pre></dt>
  <dd>
    
    list of languages spelled with alpabetic characters
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L90" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L90" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>
<div id="sourceLanguageToStr">
  <dt><pre><a href="highlite.html#sourceLanguageToStr"><span class="Identifier">sourceLanguageToStr</span></a><span class="Other">:</span> <a href="system.html#array"><span class="Identifier">array</span></a><span class="Other">[</span><span class="Identifier">SourceLanguage</span><span class="Other">,</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">]</span> <span class="Other">=</span> <span class="Other">[</span><span class="StringLit">&quot;none&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Nim&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;C++&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;C#&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;C&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Java&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Yaml&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Python&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Cmd&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Console&quot;</span><span class="Other">]</span></pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L88" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L88" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>
<div id="tokenClassToStr">
  <dt><pre><a href="highlite.html#tokenClassToStr"><span class="Identifier">tokenClassToStr</span></a><span class="Other">:</span> <a href="system.html#array"><span class="Identifier">array</span></a><span class="Other">[</span><span class="Identifier">TokenClass</span><span class="Other">,</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">]</span> <span class="Other">=</span> <span class="Other">[</span><span class="StringLit">&quot;Eof&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;None&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Whitespace&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;DecNumber&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;BinNumber&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;HexNumber&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;OctNumber&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;FloatNumber&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;Identifier&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Keyword&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;StringLit&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;LongStringLit&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;CharLit&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;EscapeSequence&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Operator&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Punctuation&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Comment&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;LongComment&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;RegularExpression&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;TagStart&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;TagEnd&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Key&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Value&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;RawData&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;Assembler&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Preprocessor&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Directive&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Command&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Rule&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Hyperlink&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;Label&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Reference&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;Prompt&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;ProgramOutput&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;program&quot;</span><span class="Other">,</span> <span class="StringLit">&quot;option&quot;</span><span class="Other">,</span>
    <span class="StringLit">&quot;Other&quot;</span><span class="Other">]</span></pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L93" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L93" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

  </dl>
</div>
<div class="section" id="12">
  <h1><a class="toc-backref" href="#12">Procs</a></h1>
  <dl class="item">
    <div id="deinitGeneralTokenizer-procs-all">
  <div id="deinitGeneralTokenizer,GeneralTokenizer">
  <dt><pre><span class="Keyword">proc</span> <a href="#deinitGeneralTokenizer%2CGeneralTokenizer"><span class="Identifier">deinitGeneralTokenizer</span></a><span class="Other">(</span><span class="Identifier">g</span><span class="Other">:</span> <span class="Keyword">var</span> <a href="highlite.html#GeneralTokenizer"><span class="Identifier">GeneralTokenizer</span></a><span class="Other">)</span> {.<span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span>
    <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L138" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L138" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

</div>
<div id="getNextToken-procs-all">
  <div id="getNextToken,GeneralTokenizer,SourceLanguage">
  <dt><pre><span class="Keyword">proc</span> <a href="#getNextToken%2CGeneralTokenizer%2CSourceLanguage"><span class="Identifier">getNextToken</span></a><span class="Other">(</span><span class="Identifier">g</span><span class="Other">:</span> <span class="Keyword">var</span> <a href="highlite.html#GeneralTokenizer"><span class="Identifier">GeneralTokenizer</span></a><span class="Other">;</span> <span class="Identifier">lang</span><span class="Other">:</span> <a href="highlite.html#SourceLanguage"><span class="Identifier">SourceLanguage</span></a><span class="Other">)</span> {.<span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span>
    <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L1002" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L1002" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

</div>
<div id="getSourceLanguage-procs-all">
  <div id="getSourceLanguage,string">
  <dt><pre><span class="Keyword">proc</span> <a href="#getSourceLanguage%2Cstring"><span class="Identifier">getSourceLanguage</span></a><span class="Other">(</span><span class="Identifier">name</span><span class="Other">:</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">)</span><span class="Other">:</span> <a href="highlite.html#SourceLanguage"><span class="Identifier">SourceLanguage</span></a> {.<span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span>
    <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L118" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L118" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

</div>
<div id="initGeneralTokenizer-procs-all">
  <div id="initGeneralTokenizer,GeneralTokenizer,cstring">
  <dt><pre><span class="Keyword">proc</span> <a href="#initGeneralTokenizer%2CGeneralTokenizer%2Ccstring"><span class="Identifier">initGeneralTokenizer</span></a><span class="Other">(</span><span class="Identifier">g</span><span class="Other">:</span> <span class="Keyword">var</span> <a href="highlite.html#GeneralTokenizer"><span class="Identifier">GeneralTokenizer</span></a><span class="Other">;</span> <span class="Identifier">buf</span><span class="Other">:</span> <a href="system.html#cstring"><span class="Identifier">cstring</span></a><span class="Other">)</span> {.<span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span>
    <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L126" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L126" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>
<div id="initGeneralTokenizer,GeneralTokenizer,string">
  <dt><pre><span class="Keyword">proc</span> <a href="#initGeneralTokenizer%2CGeneralTokenizer%2Cstring"><span class="Identifier">initGeneralTokenizer</span></a><span class="Other">(</span><span class="Identifier">g</span><span class="Other">:</span> <span class="Keyword">var</span> <a href="highlite.html#GeneralTokenizer"><span class="Identifier">GeneralTokenizer</span></a><span class="Other">;</span> <span class="Identifier">buf</span><span class="Other">:</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">)</span> {.<span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span>
    <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L135" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L135" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

</div>
<div id="tokenize-procs-all">
  <div id="tokenize,string,SourceLanguage">
  <dt><pre><span class="Keyword">proc</span> <a href="#tokenize%2Cstring%2CSourceLanguage"><span class="Identifier">tokenize</span></a><span class="Other">(</span><span class="Identifier">text</span><span class="Other">:</span> <a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">;</span> <span class="Identifier">lang</span><span class="Other">:</span> <a href="highlite.html#SourceLanguage"><span class="Identifier">SourceLanguage</span></a><span class="Other">)</span><span class="Other">:</span> <a href="system.html#seq"><span class="Identifier">seq</span></a><span class="Other">[</span><span class="Other">(</span><a href="system.html#string"><span class="Identifier">string</span></a><span class="Other">,</span> <a href="highlite.html#TokenClass"><span class="Identifier">TokenClass</span></a><span class="Other">)</span><span class="Other">]</span> {.
    <span><span class="Other pragmadots">...</span></span><span class="pragmawrap"><span class="Identifier">raises</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">tags</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span><span class="Other">,</span> <span class="Identifier">forbids</span><span class="Other">:</span> <span class="Other">[</span><span class="Other">]</span></span>.}</pre></dt>
  <dd>
    
    
    <a href="https://github.com/nim-lang/Nim/tree/devel/lib/packages/docutils/highlite.nim#L1016" class="link-seesrc" target="_blank">Source</a>&nbsp;&nbsp;
<a href="https://github.com/nim-lang/Nim/edit/devel/lib/packages/docutils/highlite.nim#L1016" class="link-seesrc" target="_blank" >Edit</a>&nbsp;&nbsp;

  </dd>
</div>

</div>

  </dl>
</div>

  </div>
</div>

      <div class="twelve-columns footer">
        <span class="nim-sprite"></span>
        <br>
        <small style="color: var(--hint);">Made with Nim. Generated: 2024-12-25 08:35:52 UTC</small>
      </div>
    </div>
  </div>
      <script defer data-domain="nim-lang.org" src="https://plausible.io/js/plausible.js"></script>
    
</body>
</html>