neovim/test/unit/viml/expressions/parser_spec.lua

local helpers = require('test.unit.helpers')(after_each)
local itp = helpers.gen_itp(it)
local viml_helpers = require('test.unit.viml.helpers')

local make_enum_conv_tab = helpers.make_enum_conv_tab
local child_call_once = helpers.child_call_once
local alloc_log_new = helpers.alloc_log_new
local kvi_destroy = helpers.kvi_destroy
local conv_enum = helpers.conv_enum
local debug_log = helpers.debug_log
local ptr2key = helpers.ptr2key
local cimport = helpers.cimport
local ffi = helpers.ffi
local neq = helpers.neq
local eq = helpers.eq
local mergedicts_copy = helpers.mergedicts_copy
local format_string = helpers.format_string
local format_luav = helpers.format_luav
local intchar2lua = helpers.intchar2lua
local dictdiff = helpers.dictdiff

local conv_ccs = viml_helpers.conv_ccs
local new_pstate = viml_helpers.new_pstate
local conv_cmp_type = viml_helpers.conv_cmp_type
local pstate_set_str = viml_helpers.pstate_set_str
local conv_expr_asgn_type = viml_helpers.conv_expr_asgn_type

local lib = cimport('./src/nvim/viml/parser/expressions.h',
                    './src/nvim/syntax.h')

local alloc_log = alloc_log_new()

local predefined_hl_defs = {
  -- From highlight_init_both
  Conceal=true,
  Cursor=true,
  lCursor=true,
  DiffText=true,
  ErrorMsg=true,
  IncSearch=true,
  ModeMsg=true,
  NonText=true,
  PmenuSbar=true,
  StatusLine=true,
  StatusLineNC=true,
  TabLineFill=true,
  TabLineSel=true,
  TermCursor=true,
  VertSplit=true,
  WildMenu=true,
  WinSeparator=true,
  EndOfBuffer=true,
  QuickFixLine=true,
  Substitute=true,
  Whitespace=true,
  Error=true,
  Todo=true,
  String=true,
  Character=true,
  Number=true,
  Boolean=true,
  Float=true,
  Function=true,
  Conditional=true,
  Repeat=true,
  Label=true,
  Operator=true,
  Keyword=true,
  Exception=true,
  Include=true,
  Define=true,
  Macro=true,
  PreCondit=true,
  StorageClass=true,
  Structure=true,
  Typedef=true,
  Tag=true,
  SpecialChar=true,
  Delimiter=true,
  SpecialComment=true,
  Debug=true,

  -- From highlight_init_(dark|light)
  ColorColumn=true,
  CursorColumn=true,
  CursorLine=true,
  CursorLineNr=true,
  DiffAdd=true,
  DiffChange=true,
  DiffDelete=true,
  Directory=true,
  FoldColumn=true,
  Folded=true,
  LineNr=true,
  MatchParen=true,
  MoreMsg=true,
  Pmenu=true,
  PmenuSel=true,
  PmenuThumb=true,
  Question=true,
  Search=true,
  SignColumn=true,
  SpecialKey=true,
  SpellBad=true,
  SpellCap=true,
  SpellLocal=true,
  SpellRare=true,
  TabLine=true,
  Title=true,
  Visual=true,
  WarningMsg=true,
  Normal=true,
  Comment=true,
  Constant=true,
  Special=true,
  Identifier=true,
  Statement=true,
  PreProc=true,
  Type=true,
  Underlined=true,
  Ignore=true,
}

local nvim_hl_defs = {}

child_call_once(function()
  local i = 0
  while lib.highlight_init_cmdline[i] ~= nil do
    local hl_args = lib.highlight_init_cmdline[i]
    local s = ffi.string(hl_args)
    local err, msg = pcall(function()
      if s:sub(1, 13) == 'default link ' then
        local new_grp, grp_link = s:match('^default link (%w+) (%w+)$')
        neq(nil, new_grp)
        -- Note: group to link to must be already defined at the time of
        --       linking, otherwise it will be created as cleared. So existence
        --       of the group is checked here and not in the next pass over
        --       nvim_hl_defs.
        eq(true, not not (nvim_hl_defs[grp_link]
                          or predefined_hl_defs[grp_link]))
        eq(false, not not (nvim_hl_defs[new_grp]
                           or predefined_hl_defs[new_grp]))
        nvim_hl_defs[new_grp] = {'link', grp_link}
      else
        local new_grp, grp_args = s:match('^(%w+) (.*)')
        neq(nil, new_grp)
        eq(false, not not (nvim_hl_defs[new_grp]
                           or predefined_hl_defs[new_grp]))
        nvim_hl_defs[new_grp] = {'definition', grp_args}
      end
    end)
    if not err then
      msg = format_string(
        'Error while processing string %s at position %u:\n%s', s, i, msg)
      error(msg)
    end
    i = i + 1
  end
  for k, _ in ipairs(nvim_hl_defs) do
    eq('Nvim', k:sub(1, 4))
    -- NvimInvalid
    -- 12345678901
    local err, msg = pcall(function()
      if k:sub(5, 11) == 'Invalid' then
        neq(nil, nvim_hl_defs['Nvim' .. k:sub(12)])
      else
        neq(nil, nvim_hl_defs['NvimInvalid' .. k:sub(5)])
      end
    end)
    if not err then
      msg = format_string('Error while processing group %s:\n%s', k, msg)
      error(msg)
    end
  end
end)

local function hls_to_hl_fs(hls)
  local ret = {}
  local next_col = 0
  for i, v in ipairs(hls) do
    local group, line, col, str = v:match('^Nvim([a-zA-Z]+):(%d+):(%d+):(.*)$')
    col = tonumber(col)
    line = tonumber(line)
    assert(line == 0)
    local col_shift = col - next_col
    assert(col_shift >= 0)
    next_col = col + #str
    ret[i] = format_string('hl(%r, %r%s)',
                           group,
                           str,
                           (col_shift == 0
                            and ''
                            or (', %u'):format(col_shift)))
  end
  return ret
end

local function format_check(expr, format_check_data, opts)
  -- That forces specific order.
  local zflags = opts.flags[1]
  local zdata = format_check_data[zflags]
  local dig_len
  if opts.funcname then
    print(format_string('\n%s(%r, {', opts.funcname, expr))
    dig_len = #opts.funcname + 2
  else
    print(format_string('\n_check_parsing(%r, %r, {', opts, expr))
    dig_len = #('_check_parsing(, \'') + #(format_string('%r', opts))
  end
  local digits = '  --' .. (' '):rep(dig_len - #('  --'))
  local digits2 = digits:sub(1, -10)
  for i = 0, #expr - 1 do
    if i % 10 == 0 then
      digits2 = ('%s%10u'):format(digits2, i / 10)
    end
    digits = ('%s%u'):format(digits, i % 10)
  end
  print(digits)
  if #expr > 10 then
    print(digits2)
  end
  if zdata.ast.len then
    print(('  len = %u,'):format(zdata.ast.len))
  end
  print('  ast = ' .. format_luav(zdata.ast.ast, '  ') .. ',')
  if zdata.ast.err then
    print('  err = {')
    print('    arg = ' .. format_luav(zdata.ast.err.arg) .. ',')
    print('    msg = ' .. format_luav(zdata.ast.err.msg) .. ',')
    print('  },')
  end
  print('}, {')
  for _, v in ipairs(zdata.hl_fs) do
    print('  ' .. v .. ',')
  end
  local diffs = {}
  local diffs_num = 0
  for flags, v in pairs(format_check_data) do
    if flags ~= zflags then
      diffs[flags] = dictdiff(zdata, v)
      if diffs[flags] then
        if flags == 3 + zflags then
          if (dictdiff(format_check_data[1 + zflags],
                       format_check_data[3 + zflags]) == nil
              or dictdiff(format_check_data[2 + zflags],
                          format_check_data[3 + zflags]) == nil)
          then
            diffs[flags] = nil
          else
            diffs_num = diffs_num + 1
          end
        else
          diffs_num = diffs_num + 1
        end
      end
    end
  end
  if diffs_num ~= 0 then
    print('}, {')
    local flags = 1
    while diffs_num ~= 0 do
      if diffs[flags] then
        diffs_num = diffs_num - 1
        local diff = diffs[flags]
        print(('  [%u] = {'):format(flags))
        if diff.ast then
          print('    ast = ' .. format_luav(diff.ast, '    ') .. ',')
        end
        if diff.hl_fs then
          print('    hl_fs = ' .. format_luav(diff.hl_fs, '    ', {
            literal_strings=true
          }) .. ',')
        end
        print('  },')
      end
      flags = flags + 1
    end
  end
  print('})')
end

local east_node_type_tab
make_enum_conv_tab(lib, {
  'kExprNodeMissing',
  'kExprNodeOpMissing',
  'kExprNodeTernary',
  'kExprNodeTernaryValue',
  'kExprNodeRegister',
  'kExprNodeSubscript',
  'kExprNodeListLiteral',
  'kExprNodeUnaryPlus',
  'kExprNodeBinaryPlus',
  'kExprNodeNested',
  'kExprNodeCall',
  'kExprNodePlainIdentifier',
  'kExprNodePlainKey',
  'kExprNodeComplexIdentifier',
  'kExprNodeUnknownFigure',
  'kExprNodeLambda',
  'kExprNodeDictLiteral',
  'kExprNodeCurlyBracesIdentifier',
  'kExprNodeComma',
  'kExprNodeColon',
  'kExprNodeArrow',
  'kExprNodeComparison',
  'kExprNodeConcat',
  'kExprNodeConcatOrSubscript',
  'kExprNodeInteger',
  'kExprNodeFloat',
  'kExprNodeSingleQuotedString',
  'kExprNodeDoubleQuotedString',
  'kExprNodeOr',
  'kExprNodeAnd',
  'kExprNodeUnaryMinus',
  'kExprNodeBinaryMinus',
  'kExprNodeNot',
  'kExprNodeMultiplication',
  'kExprNodeDivision',
  'kExprNodeMod',
  'kExprNodeOption',
  'kExprNodeEnvironment',
  'kExprNodeAssignment',
}, 'kExprNode', function(ret) east_node_type_tab = ret end)

local function conv_east_node_type(typ)
  return conv_enum(east_node_type_tab, typ)
end

local eastnodelist2lua

local function eastnode2lua(pstate, eastnode, checked_nodes)
  local key = ptr2key(eastnode)
  if checked_nodes[key] then
    checked_nodes[key].duplicate_key = key
    return { duplicate = key }
  end
  local typ = conv_east_node_type(eastnode.type)
  local ret = {}
  checked_nodes[key] = ret
  ret.children = eastnodelist2lua(pstate, eastnode.children, checked_nodes)
  local str = pstate_set_str(pstate, eastnode.start, eastnode.len)
  local ret_str
  if str.error then
    ret_str = 'error:' .. str.error
  else
    ret_str = ('%u:%u:%s'):format(str.start.line, str.start.col, str.str)
  end
  if typ == 'Register' then
    typ = typ .. ('(name=%s)'):format(
      tostring(intchar2lua(eastnode.data.reg.name)))
  elseif typ == 'PlainIdentifier' then
    typ = typ .. ('(scope=%s,ident=%s)'):format(
      tostring(intchar2lua(eastnode.data.var.scope)),
      ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len))
  elseif typ == 'PlainKey' then
    typ = typ .. ('(key=%s)'):format(
      ffi.string(eastnode.data.var.ident, eastnode.data.var.ident_len))
  elseif (typ == 'UnknownFigure' or typ == 'DictLiteral'
          or typ == 'CurlyBracesIdentifier' or typ == 'Lambda') then
    typ = typ .. ('(%s)'):format(
      (eastnode.data.fig.type_guesses.allow_lambda and '\\' or '-')
      .. (eastnode.data.fig.type_guesses.allow_dict and 'd' or '-')
      .. (eastnode.data.fig.type_guesses.allow_ident and 'i' or '-'))
  elseif typ == 'Comparison' then
    typ = typ .. ('(type=%s,inv=%u,ccs=%s)'):format(
      conv_cmp_type(eastnode.data.cmp.type), eastnode.data.cmp.inv and 1 or 0,
      conv_ccs(eastnode.data.cmp.ccs))
  elseif typ == 'Integer' then
    typ = typ .. ('(val=%u)'):format(tonumber(eastnode.data.num.value))
  elseif typ == 'Float' then
    typ = typ .. format_string('(val=%e)', tonumber(eastnode.data.flt.value))
  elseif typ == 'SingleQuotedString' or typ == 'DoubleQuotedString' then
    if eastnode.data.str.value == nil then
      typ = typ .. '(val=NULL)'
    else
      local s = ffi.string(eastnode.data.str.value, eastnode.data.str.size)
      typ = format_string('%s(val=%q)', typ, s)
    end
  elseif typ == 'Option' then
    typ = ('%s(scope=%s,ident=%s)'):format(
      typ,
      tostring(intchar2lua(eastnode.data.opt.scope)),
      ffi.string(eastnode.data.opt.ident, eastnode.data.opt.ident_len))
  elseif typ == 'Environment' then
    typ = ('%s(ident=%s)'):format(
      typ,
      ffi.string(eastnode.data.env.ident, eastnode.data.env.ident_len))
  elseif typ == 'Assignment' then
    typ = ('%s(%s)'):format(typ, conv_expr_asgn_type(eastnode.data.ass.type))
  end
  ret_str = typ .. ':' .. ret_str
  local can_simplify = not ret.children
  if can_simplify then
    ret = ret_str
  else
    ret[1] = ret_str
  end
  return ret
end

eastnodelist2lua = function(pstate, eastnode, checked_nodes)
  local ret = {}
  while eastnode ~= nil do
    ret[#ret + 1] = eastnode2lua(pstate, eastnode, checked_nodes)
    eastnode = eastnode.next
  end
  if #ret == 0 then
    ret = nil
  end
  return ret
end

local function east2lua(str, pstate, east)
  local checked_nodes = {}
  local len = tonumber(pstate.pos.col)
  if pstate.pos.line == 1 then
    len = tonumber(pstate.reader.lines.items[0].size)
  end
  if type(str) == 'string' and len == #str then
    len = nil
  end
  return {
    err = east.err.msg ~= nil and {
      msg = ffi.string(east.err.msg),
      arg = ffi.string(east.err.arg, east.err.arg_len),
    } or nil,
    len = len,
    ast = eastnodelist2lua(pstate, east.root, checked_nodes),
  }
end

local function phl2lua(pstate)
  local ret = {}
  for i = 0, (tonumber(pstate.colors.size) - 1) do
    local chunk = pstate.colors.items[i]
    local chunk_tbl = pstate_set_str(
      pstate, chunk.start, chunk.end_col - chunk.start.col, {
        group = ffi.string(chunk.group),
      })
    ret[i + 1] = ('%s:%u:%u:%s'):format(
      chunk_tbl.group,
      chunk_tbl.start.line,
      chunk_tbl.start.col,
      chunk_tbl.str)
  end
  return ret
end

describe('Expressions parser', function()
  local function _check_parsing(opts, str, exp_ast, exp_highlighting_fs,
                                nz_flags_exps)
    local zflags = opts.flags[1]
    nz_flags_exps = nz_flags_exps or {}
    local format_check_data = {}
    for _, flags in ipairs(opts.flags) do
      debug_log(('Running test case (%s, %u)'):format(str, flags))
      local err, msg = pcall(function()
        if os.getenv('NVIM_TEST_PARSER_SPEC_PRINT_TEST_CASE') == '1' then
          print(str, flags)
        end
        alloc_log:check({})

        local pstate = new_pstate({str})
        local east = lib.viml_pexpr_parse(pstate, flags)
        local ast = east2lua(str, pstate, east)
        local hls = phl2lua(pstate)
        if exp_ast == nil then
          format_check_data[flags] = {ast=ast, hl_fs=hls_to_hl_fs(hls)}
        else
          local exps = {
            ast = exp_ast,
            hl_fs = exp_highlighting_fs,
          }
          local add_exps = nz_flags_exps[flags]
          if not add_exps and flags == 3 + zflags then
            add_exps = nz_flags_exps[1 + zflags] or nz_flags_exps[2 + zflags]
          end
          if add_exps then
            if add_exps.ast then
              exps.ast = mergedicts_copy(exps.ast, add_exps.ast)
            end
            if add_exps.hl_fs then
              exps.hl_fs = mergedicts_copy(exps.hl_fs, add_exps.hl_fs)
            end
          end
          eq(exps.ast, ast)
          if exp_highlighting_fs then
            local exp_highlighting = {}
            local next_col = 0
            for i, h in ipairs(exps.hl_fs) do
              exp_highlighting[i], next_col = h(next_col)
            end
            eq(exp_highlighting, hls)
          end
        end
        lib.viml_pexpr_free_ast(east)
        kvi_destroy(pstate.colors)
        alloc_log:clear_tmp_allocs(true)
        alloc_log:check({})
      end)
      if not err then
        msg = format_string('Error while processing test (%r, %u):\n%s',
                            str, flags, msg)
        error(msg)
      end
    end
    if exp_ast == nil then
      format_check(str, format_check_data, opts)
    end
  end
  local function hl(group, str, shift)
    return function(next_col)
      if nvim_hl_defs['Nvim' .. group] == nil then
        error(('Unknown group: Nvim%s'):format(group))
      end
      local col = next_col + (shift or 0)
      return (('%s:%u:%u:%s'):format(
        'Nvim' .. group,
        0,
        col,
        str)), (col + #str)
    end
  end
  local function fmtn(typ, args, rest)
    return ('%s(%s)%s'):format(typ, args, rest)
  end
  require('test.unit.viml.expressions.parser_tests')(
      itp, _check_parsing, hl, fmtn)
end)