neovim/test/unit/viml/expressions/lexer_spec.lua

local t = require('test.unit.testutil')
local itp = t.gen_itp(it)
local t_viml = require('test.unit.viml.testutil')

local child_call_once = t.child_call_once
local conv_enum = t.conv_enum
local cimport = t.cimport
local ffi = t.ffi
local eq = t.eq
local shallowcopy = t.shallowcopy
local intchar2lua = t.intchar2lua

local conv_ccs = t_viml.conv_ccs
local new_pstate = t_viml.new_pstate
local conv_cmp_type = t_viml.conv_cmp_type
local pstate_set_str = t_viml.pstate_set_str
local conv_expr_asgn_type = t_viml.conv_expr_asgn_type

local lib = cimport('./src/nvim/viml/parser/expressions.h')

local eltkn_type_tab, eltkn_mul_type_tab, eltkn_opt_scope_tab
child_call_once(function()
  eltkn_type_tab = {
    [tonumber(lib.kExprLexInvalid)] = 'Invalid',
    [tonumber(lib.kExprLexMissing)] = 'Missing',
    [tonumber(lib.kExprLexSpacing)] = 'Spacing',
    [tonumber(lib.kExprLexEOC)] = 'EOC',

    [tonumber(lib.kExprLexQuestion)] = 'Question',
    [tonumber(lib.kExprLexColon)] = 'Colon',
    [tonumber(lib.kExprLexOr)] = 'Or',
    [tonumber(lib.kExprLexAnd)] = 'And',
    [tonumber(lib.kExprLexComparison)] = 'Comparison',
    [tonumber(lib.kExprLexPlus)] = 'Plus',
    [tonumber(lib.kExprLexMinus)] = 'Minus',
    [tonumber(lib.kExprLexDot)] = 'Dot',
    [tonumber(lib.kExprLexMultiplication)] = 'Multiplication',

    [tonumber(lib.kExprLexNot)] = 'Not',

    [tonumber(lib.kExprLexNumber)] = 'Number',
    [tonumber(lib.kExprLexSingleQuotedString)] = 'SingleQuotedString',
    [tonumber(lib.kExprLexDoubleQuotedString)] = 'DoubleQuotedString',
    [tonumber(lib.kExprLexOption)] = 'Option',
    [tonumber(lib.kExprLexRegister)] = 'Register',
    [tonumber(lib.kExprLexEnv)] = 'Env',
    [tonumber(lib.kExprLexPlainIdentifier)] = 'PlainIdentifier',

    [tonumber(lib.kExprLexBracket)] = 'Bracket',
    [tonumber(lib.kExprLexFigureBrace)] = 'FigureBrace',
    [tonumber(lib.kExprLexParenthesis)] = 'Parenthesis',
    [tonumber(lib.kExprLexComma)] = 'Comma',
    [tonumber(lib.kExprLexArrow)] = 'Arrow',

    [tonumber(lib.kExprLexAssignment)] = 'Assignment',
  }

  eltkn_mul_type_tab = {
    [tonumber(lib.kExprLexMulMul)] = 'Mul',
    [tonumber(lib.kExprLexMulDiv)] = 'Div',
    [tonumber(lib.kExprLexMulMod)] = 'Mod',
  }

  eltkn_opt_scope_tab = {
    [tonumber(lib.kExprOptScopeUnspecified)] = 'Unspecified',
    [tonumber(lib.kExprOptScopeGlobal)] = 'Global',
    [tonumber(lib.kExprOptScopeLocal)] = 'Local',
  }
end)

local function conv_eltkn_type(typ)
  return conv_enum(eltkn_type_tab, typ)
end

local bracket_types = {
  Bracket = true,
  FigureBrace = true,
  Parenthesis = true,
}

local function eltkn2lua(pstate, tkn)
  local ret = {
    type = conv_eltkn_type(tkn.type),
  }
  pstate_set_str(pstate, tkn.start, tkn.len, ret)
  if not ret.error and (#ret.str ~= ret.len) then
    ret.error = '#str /= len'
  end
  if ret.type == 'Comparison' then
    ret.data = {
      type = conv_cmp_type(tkn.data.cmp.type),
      ccs = conv_ccs(tkn.data.cmp.ccs),
      inv = not not tkn.data.cmp.inv,
    }
  elseif ret.type == 'Multiplication' then
    ret.data = { type = conv_enum(eltkn_mul_type_tab, tkn.data.mul.type) }
  elseif bracket_types[ret.type] then
    ret.data = { closing = not not tkn.data.brc.closing }
  elseif ret.type == 'Register' then
    ret.data = { name = intchar2lua(tkn.data.reg.name) }
  elseif ret.type == 'SingleQuotedString' or ret.type == 'DoubleQuotedString' then
    ret.data = { closed = not not tkn.data.str.closed }
  elseif ret.type == 'Option' then
    ret.data = {
      scope = conv_enum(eltkn_opt_scope_tab, tkn.data.opt.scope),
      name = ffi.string(tkn.data.opt.name, tkn.data.opt.len),
    }
  elseif ret.type == 'PlainIdentifier' then
    ret.data = {
      scope = intchar2lua(tkn.data.var.scope),
      autoload = not not tkn.data.var.autoload,
    }
  elseif ret.type == 'Number' then
    ret.data = {
      is_float = not not tkn.data.num.is_float,
      base = tonumber(tkn.data.num.base),
    }
    ret.data.val =
      tonumber(tkn.data.num.is_float and tkn.data.num.val.floating or tkn.data.num.val.integer)
  elseif ret.type == 'Assignment' then
    ret.data = { type = conv_expr_asgn_type(tkn.data.ass.type) }
  elseif ret.type == 'Invalid' then
    ret.data = { error = ffi.string(tkn.data.err.msg) }
  end
  return ret, tkn
end

local function next_eltkn(pstate, flags)
  return eltkn2lua(pstate, lib.viml_pexpr_next_token(pstate, flags))
end

describe('Expressions lexer', function()
  local flags = 0
  local should_advance = true
  local function check_advance(pstate, bytes_to_advance, initial_col)
    local tgt = initial_col + bytes_to_advance
    if should_advance then
      if pstate.reader.lines.items[0].size == tgt then
        eq(1, pstate.pos.line)
        eq(0, pstate.pos.col)
      else
        eq(0, pstate.pos.line)
        eq(tgt, pstate.pos.col)
      end
    else
      eq(0, pstate.pos.line)
      eq(initial_col, pstate.pos.col)
    end
  end
  local function singl_eltkn_test(typ, str, data)
    local pstate = new_pstate({ str })
    eq(
      { data = data, len = #str, start = { col = 0, line = 0 }, str = str, type = typ },
      next_eltkn(pstate, flags)
    )
    check_advance(pstate, #str, 0)
    if
      not (
        typ == 'Spacing'
        or (typ == 'Register' and str == '@')
        or ((typ == 'SingleQuotedString' or typ == 'DoubleQuotedString') and not data.closed)
      )
    then
      pstate = new_pstate({ str .. ' ' })
      eq(
        { data = data, len = #str, start = { col = 0, line = 0 }, str = str, type = typ },
        next_eltkn(pstate, flags)
      )
      check_advance(pstate, #str, 0)
    end
    pstate = new_pstate({ 'x' .. str })
    pstate.pos.col = 1
    eq(
      { data = data, len = #str, start = { col = 1, line = 0 }, str = str, type = typ },
      next_eltkn(pstate, flags)
    )
    check_advance(pstate, #str, 1)
  end
  local function scope_test(scope)
    singl_eltkn_test('PlainIdentifier', scope .. ':test#var', { autoload = true, scope = scope })
    singl_eltkn_test('PlainIdentifier', scope .. ':', { autoload = false, scope = scope })
  end
  local function comparison_test(op, inv_op, cmp_type)
    singl_eltkn_test('Comparison', op, { type = cmp_type, inv = false, ccs = 'UseOption' })
    singl_eltkn_test('Comparison', inv_op, { type = cmp_type, inv = true, ccs = 'UseOption' })
    singl_eltkn_test('Comparison', op .. '#', { type = cmp_type, inv = false, ccs = 'MatchCase' })
    singl_eltkn_test(
      'Comparison',
      inv_op .. '#',
      { type = cmp_type, inv = true, ccs = 'MatchCase' }
    )
    singl_eltkn_test('Comparison', op .. '?', { type = cmp_type, inv = false, ccs = 'IgnoreCase' })
    singl_eltkn_test(
      'Comparison',
      inv_op .. '?',
      { type = cmp_type, inv = true, ccs = 'IgnoreCase' }
    )
  end
  local function simple_test(pstate_arg, exp_type, exp_len, exp)
    local pstate = new_pstate(pstate_arg)
    exp = shallowcopy(exp)
    exp.type = exp_type
    exp.len = exp_len or #pstate_arg[0]
    exp.start = { col = 0, line = 0 }
    eq(exp, next_eltkn(pstate, flags))
  end
  local function stable_tests()
    singl_eltkn_test('Parenthesis', '(', { closing = false })
    singl_eltkn_test('Parenthesis', ')', { closing = true })
    singl_eltkn_test('Bracket', '[', { closing = false })
    singl_eltkn_test('Bracket', ']', { closing = true })
    singl_eltkn_test('FigureBrace', '{', { closing = false })
    singl_eltkn_test('FigureBrace', '}', { closing = true })
    singl_eltkn_test('Question', '?')
    singl_eltkn_test('Colon', ':')
    singl_eltkn_test('Dot', '.')
    singl_eltkn_test('Assignment', '.=', { type = 'Concat' })
    singl_eltkn_test('Plus', '+')
    singl_eltkn_test('Assignment', '+=', { type = 'Add' })
    singl_eltkn_test('Comma', ',')
    singl_eltkn_test('Multiplication', '*', { type = 'Mul' })
    singl_eltkn_test('Multiplication', '/', { type = 'Div' })
    singl_eltkn_test('Multiplication', '%', { type = 'Mod' })
    singl_eltkn_test('Spacing', '  \t\t  \t\t')
    singl_eltkn_test('Spacing', ' ')
    singl_eltkn_test('Spacing', '\t')
    singl_eltkn_test(
      'Invalid',
      '\x01\x02\x03',
      { error = 'E15: Invalid control character present in input: %.*s' }
    )
    singl_eltkn_test('Number', '0123', { is_float = false, base = 8, val = 83 })
    singl_eltkn_test('Number', '01234567', { is_float = false, base = 8, val = 342391 })
    singl_eltkn_test('Number', '012345678', { is_float = false, base = 10, val = 12345678 })
    singl_eltkn_test('Number', '0x123', { is_float = false, base = 16, val = 291 })
    singl_eltkn_test('Number', '0x56FF', { is_float = false, base = 16, val = 22271 })
    singl_eltkn_test('Number', '0xabcdef', { is_float = false, base = 16, val = 11259375 })
    singl_eltkn_test('Number', '0xABCDEF', { is_float = false, base = 16, val = 11259375 })
    singl_eltkn_test('Number', '0x0', { is_float = false, base = 16, val = 0 })
    singl_eltkn_test('Number', '00', { is_float = false, base = 8, val = 0 })
    singl_eltkn_test('Number', '0b0', { is_float = false, base = 2, val = 0 })
    singl_eltkn_test('Number', '0b010111', { is_float = false, base = 2, val = 23 })
    singl_eltkn_test('Number', '0b100111', { is_float = false, base = 2, val = 39 })
    singl_eltkn_test('Number', '0', { is_float = false, base = 10, val = 0 })
    singl_eltkn_test('Number', '9', { is_float = false, base = 10, val = 9 })
    singl_eltkn_test('Env', '$abc')
    singl_eltkn_test('Env', '$')
    singl_eltkn_test('PlainIdentifier', 'test', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', '_test', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', '_test_foo', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 't', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 'test5', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 't0', { autoload = false, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 'test#var', { autoload = true, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 'test#var#val###', { autoload = true, scope = 0 })
    singl_eltkn_test('PlainIdentifier', 't#####', { autoload = true, scope = 0 })
    singl_eltkn_test('And', '&&')
    singl_eltkn_test('Or', '||')
    singl_eltkn_test('Invalid', '&', { error = 'E112: Option name missing: %.*s' })
    singl_eltkn_test('Option', '&opt', { scope = 'Unspecified', name = 'opt' })
    singl_eltkn_test('Option', '&t_xx', { scope = 'Unspecified', name = 't_xx' })
    singl_eltkn_test('Option', '&t_\r\r', { scope = 'Unspecified', name = 't_\r\r' })
    singl_eltkn_test('Option', '&t_\t\t', { scope = 'Unspecified', name = 't_\t\t' })
    singl_eltkn_test('Option', '&t_  ', { scope = 'Unspecified', name = 't_  ' })
    singl_eltkn_test('Option', '&g:opt', { scope = 'Global', name = 'opt' })
    singl_eltkn_test('Option', '&l:opt', { scope = 'Local', name = 'opt' })
    singl_eltkn_test('Invalid', '&l:', { error = 'E112: Option name missing: %.*s' })
    singl_eltkn_test('Invalid', '&g:', { error = 'E112: Option name missing: %.*s' })
    singl_eltkn_test('Register', '@', { name = -1 })
    singl_eltkn_test('Register', '@a', { name = 'a' })
    singl_eltkn_test('Register', '@\r', { name = 13 })
    singl_eltkn_test('Register', '@ ', { name = ' ' })
    singl_eltkn_test('Register', '@\t', { name = 9 })
    singl_eltkn_test('SingleQuotedString', "'test", { closed = false })
    singl_eltkn_test('SingleQuotedString', "'test'", { closed = true })
    singl_eltkn_test('SingleQuotedString', "''''", { closed = true })
    singl_eltkn_test('SingleQuotedString', "'x'''", { closed = true })
    singl_eltkn_test('SingleQuotedString', "'''x'", { closed = true })
    singl_eltkn_test('SingleQuotedString', "'''", { closed = false })
    singl_eltkn_test('SingleQuotedString', "'x''", { closed = false })
    singl_eltkn_test('SingleQuotedString', "'''x", { closed = false })
    singl_eltkn_test('DoubleQuotedString', '"test', { closed = false })
    singl_eltkn_test('DoubleQuotedString', '"test"', { closed = true })
    singl_eltkn_test('DoubleQuotedString', '"\\""', { closed = true })
    singl_eltkn_test('DoubleQuotedString', '"x\\""', { closed = true })
    singl_eltkn_test('DoubleQuotedString', '"\\"x"', { closed = true })
    singl_eltkn_test('DoubleQuotedString', '"\\"', { closed = false })
    singl_eltkn_test('DoubleQuotedString', '"x\\"', { closed = false })
    singl_eltkn_test('DoubleQuotedString', '"\\"x', { closed = false })
    singl_eltkn_test('Not', '!')
    singl_eltkn_test('Assignment', '=', { type = 'Plain' })
    comparison_test('==', '!=', 'Equal')
    comparison_test('=~', '!~', 'Matches')
    comparison_test('>', '<=', 'Greater')
    comparison_test('>=', '<', 'GreaterOrEqual')
    singl_eltkn_test('Minus', '-')
    singl_eltkn_test('Assignment', '-=', { type = 'Subtract' })
    singl_eltkn_test('Arrow', '->')
    singl_eltkn_test('Invalid', '~', { error = 'E15: Unidentified character: %.*s' })
    simple_test({ { data = nil, size = 0 } }, 'EOC', 0, { error = 'start.col >= #pstr' })
    simple_test({ '' }, 'EOC', 0, { error = 'start.col >= #pstr' })
    simple_test(
      { '2.' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2e5' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.x' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.2.' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0x' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e+' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e-' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e+x' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e-x' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e+1a' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e-1a' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '0b102' },
      'Number',
      4,
      { data = { is_float = false, base = 2, val = 2 }, str = '0b10' }
    )
    simple_test(
      { '10F' },
      'Number',
      2,
      { data = { is_float = false, base = 10, val = 10 }, str = '10' }
    )
    simple_test({ '0x0123456789ABCDEFG' }, 'Number', 18, {
      data = { is_float = false, base = 16, val = 81985529216486895 },
      str = '0x0123456789ABCDEF',
    })
    simple_test(
      { { data = '00', size = 2 } },
      'Number',
      2,
      { data = { is_float = false, base = 8, val = 0 }, str = '00' }
    )
    simple_test(
      { { data = '009', size = 2 } },
      'Number',
      2,
      { data = { is_float = false, base = 8, val = 0 }, str = '00' }
    )
    simple_test(
      { { data = '01', size = 1 } },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 0 }, str = '0' }
    )
  end

  local function regular_scope_tests()
    scope_test('s')
    scope_test('g')
    scope_test('v')
    scope_test('b')
    scope_test('w')
    scope_test('t')
    scope_test('l')
    scope_test('a')

    simple_test(
      { 'g:' },
      'PlainIdentifier',
      2,
      { data = { scope = 'g', autoload = false }, str = 'g:' }
    )
    simple_test(
      { 'g:is#foo' },
      'PlainIdentifier',
      8,
      { data = { scope = 'g', autoload = true }, str = 'g:is#foo' }
    )
    simple_test(
      { 'g:isnot#foo' },
      'PlainIdentifier',
      11,
      { data = { scope = 'g', autoload = true }, str = 'g:isnot#foo' }
    )
  end

  local function regular_is_tests()
    comparison_test('is', 'isnot', 'Identical')

    simple_test(
      { 'is' },
      'Comparison',
      2,
      { data = { type = 'Identical', inv = false, ccs = 'UseOption' }, str = 'is' }
    )
    simple_test(
      { 'isnot' },
      'Comparison',
      5,
      { data = { type = 'Identical', inv = true, ccs = 'UseOption' }, str = 'isnot' }
    )
    simple_test(
      { 'is?' },
      'Comparison',
      3,
      { data = { type = 'Identical', inv = false, ccs = 'IgnoreCase' }, str = 'is?' }
    )
    simple_test(
      { 'isnot?' },
      'Comparison',
      6,
      { data = { type = 'Identical', inv = true, ccs = 'IgnoreCase' }, str = 'isnot?' }
    )
    simple_test(
      { 'is#' },
      'Comparison',
      3,
      { data = { type = 'Identical', inv = false, ccs = 'MatchCase' }, str = 'is#' }
    )
    simple_test(
      { 'isnot#' },
      'Comparison',
      6,
      { data = { type = 'Identical', inv = true, ccs = 'MatchCase' }, str = 'isnot#' }
    )
    simple_test(
      { 'is#foo' },
      'Comparison',
      3,
      { data = { type = 'Identical', inv = false, ccs = 'MatchCase' }, str = 'is#' }
    )
    simple_test(
      { 'isnot#foo' },
      'Comparison',
      6,
      { data = { type = 'Identical', inv = true, ccs = 'MatchCase' }, str = 'isnot#' }
    )
  end

  local function regular_number_tests()
    simple_test(
      { '2.0' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e5' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e+5' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { '2.0e-5' },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
  end

  local function regular_eoc_tests()
    singl_eltkn_test('EOC', '|')
    singl_eltkn_test('EOC', '\0')
    singl_eltkn_test('EOC', '\n')
  end

  itp('works (single tokens, zero flags)', function()
    stable_tests()

    regular_eoc_tests()
    regular_scope_tests()
    regular_is_tests()
    regular_number_tests()
  end)
  itp('peeks', function()
    flags = tonumber(lib.kELFlagPeek)
    should_advance = false
    stable_tests()

    regular_eoc_tests()
    regular_scope_tests()
    regular_is_tests()
    regular_number_tests()
  end)
  itp('forbids scope', function()
    flags = tonumber(lib.kELFlagForbidScope)
    stable_tests()

    regular_eoc_tests()
    regular_is_tests()
    regular_number_tests()

    simple_test(
      { 'g:' },
      'PlainIdentifier',
      1,
      { data = { scope = 0, autoload = false }, str = 'g' }
    )
  end)
  itp('allows floats', function()
    flags = tonumber(lib.kELFlagAllowFloat)
    stable_tests()

    regular_eoc_tests()
    regular_scope_tests()
    regular_is_tests()

    simple_test(
      { '2.2' },
      'Number',
      3,
      { data = { is_float = true, base = 10, val = 2.2 }, str = '2.2' }
    )
    simple_test(
      { '2.0e5' },
      'Number',
      5,
      { data = { is_float = true, base = 10, val = 2e5 }, str = '2.0e5' }
    )
    simple_test(
      { '2.0e+5' },
      'Number',
      6,
      { data = { is_float = true, base = 10, val = 2e5 }, str = '2.0e+5' }
    )
    simple_test(
      { '2.0e-5' },
      'Number',
      6,
      { data = { is_float = true, base = 10, val = 2e-5 }, str = '2.0e-5' }
    )
    simple_test(
      { '2.500000e-5' },
      'Number',
      11,
      { data = { is_float = true, base = 10, val = 2.5e-5 }, str = '2.500000e-5' }
    )
    simple_test(
      { '2.5555e2' },
      'Number',
      8,
      { data = { is_float = true, base = 10, val = 2.5555e2 }, str = '2.5555e2' }
    )
    simple_test(
      { '2.5555e+2' },
      'Number',
      9,
      { data = { is_float = true, base = 10, val = 2.5555e2 }, str = '2.5555e+2' }
    )
    simple_test(
      { '2.5555e-2' },
      'Number',
      9,
      { data = { is_float = true, base = 10, val = 2.5555e-2 }, str = '2.5555e-2' }
    )
    simple_test(
      { { data = '2.5e-5', size = 3 } },
      'Number',
      3,
      { data = { is_float = true, base = 10, val = 2.5 }, str = '2.5' }
    )
    simple_test(
      { { data = '2.5e5', size = 4 } },
      'Number',
      1,
      { data = { is_float = false, base = 10, val = 2 }, str = '2' }
    )
    simple_test(
      { { data = '2.5e-50', size = 6 } },
      'Number',
      6,
      { data = { is_float = true, base = 10, val = 2.5e-5 }, str = '2.5e-5' }
    )
  end)
  itp('treats `is` as an identifier', function()
    flags = tonumber(lib.kELFlagIsNotCmp)
    stable_tests()

    regular_eoc_tests()
    regular_scope_tests()
    regular_number_tests()

    simple_test(
      { 'is' },
      'PlainIdentifier',
      2,
      { data = { scope = 0, autoload = false }, str = 'is' }
    )
    simple_test(
      { 'isnot' },
      'PlainIdentifier',
      5,
      { data = { scope = 0, autoload = false }, str = 'isnot' }
    )
    simple_test(
      { 'is?' },
      'PlainIdentifier',
      2,
      { data = { scope = 0, autoload = false }, str = 'is' }
    )
    simple_test(
      { 'isnot?' },
      'PlainIdentifier',
      5,
      { data = { scope = 0, autoload = false }, str = 'isnot' }
    )
    simple_test(
      { 'is#' },
      'PlainIdentifier',
      3,
      { data = { scope = 0, autoload = true }, str = 'is#' }
    )
    simple_test(
      { 'isnot#' },
      'PlainIdentifier',
      6,
      { data = { scope = 0, autoload = true }, str = 'isnot#' }
    )
    simple_test(
      { 'is#foo' },
      'PlainIdentifier',
      6,
      { data = { scope = 0, autoload = true }, str = 'is#foo' }
    )
    simple_test(
      { 'isnot#foo' },
      'PlainIdentifier',
      9,
      { data = { scope = 0, autoload = true }, str = 'isnot#foo' }
    )
  end)
  itp('forbids EOC', function()
    flags = tonumber(lib.kELFlagForbidEOC)
    stable_tests()

    regular_scope_tests()
    regular_is_tests()
    regular_number_tests()

    singl_eltkn_test('Invalid', '|', { error = 'E15: Unexpected EOC character: %.*s' })
    singl_eltkn_test('Invalid', '\0', { error = 'E15: Unexpected EOC character: %.*s' })
    singl_eltkn_test('Invalid', '\n', { error = 'E15: Unexpected EOC character: %.*s' })
  end)
end)