encoding: update tests

Change shada reencoding tests to check for
correct handling of UTF-8 and binary strings.

Delete enc=latin1 json tests.

test/functional/eval/json_functions_spec.lua

@@ -489,18 +489,6 @@ describe('json_decode() function', function()
                  '{"b": 3, "a": 1, "c": 4, "d": 2, "\\u0000": 4}')
   end)
 
-  it('converts strings to latin1 when &encoding is latin1', function()
-    restart('--cmd', 'set encoding=latin1')
-    eq('\171', funcs.json_decode('"\\u00AB"'))
-    sp_decode_eq({_TYPE='string', _VAL={'\n\171\n'}}, '"\\u0000\\u00AB\\u0000"')
-  end)
-
-  it('fails to convert string to latin1 if it is impossible', function()
-    restart('--cmd', 'set encoding=latin1')
-    eq('Vim(call):E474: Failed to convert string "ꯍ" from UTF-8',
-       exc_exec('call json_decode(\'"\\uABCD"\')'))
-  end)
-
   it('parses U+00C3 correctly', function()
     eq('\195\131', funcs.json_decode('"\195\131"'))
   end)
@@ -528,14 +516,6 @@ describe('json_decode() function', function()
     eq({key={'val', 'val2'}, key2=1}, funcs.json_decode(str))
   end)
 
-  it('always treats input as UTF-8', function()
-    -- When &encoding is latin1 string "«" is U+00C2 U+00AB U+00C2: «Â. So if
-    -- '"«"' was parsed as latin1 json_decode would return three characters, and
-    -- only one U+00AB when this string is parsed as latin1.
-    restart('--cmd', 'set encoding=latin1')
-    eq(('%c'):format(0xAB), funcs.json_decode('"«"'))
-  end)
-
   it('does not overflow when writing error message about decoding ["", ""]',
   function()
     eq('\nE474: Attempt to decode a blank string'
@@ -762,12 +742,6 @@ describe('json_encode() function', function()
        exc_exec('call json_encode(["", ""], 1)'))
   end)
 
-  it('converts strings from latin1 when &encoding is latin1', function()
-    clear('--cmd', 'set encoding=latin1')
-    eq('"\\u00AB"', funcs.json_encode('\171'))
-    eq('"\\u0000\\u00AB\\u0000"', eval('json_encode({"_TYPE": v:msgpack_types.string, "_VAL": ["\\n\171\\n"]})'))
-  end)
-
   it('ignores improper values in &isprint', function()
     meths.set_option('isprint', '1')
     eq(1, eval('"\1" =~# "\\\\p"'))

test/functional/ex_cmds/encoding_spec.lua

@@ -15,27 +15,26 @@ describe('&encoding', function()
     execute('set encoding=latin1')
     -- error message expected
     feed('<cr>')
-    neq(nil, string.find(eval('v:errmsg'), '^E905:'))
+    neq(nil, string.find(eval('v:errmsg'), '^E474:'))
     eq('utf-8', eval('&encoding'))
     -- check nvim is still in utf-8 mode
     eq(3, eval('strwidth("Bär")'))
   end)
 
-  it('can be changed before startup', function()
+  it('cannot be changed before startup', function()
     clear('--cmd', 'set enc=latin1')
-    execute('set encoding=utf-8')
     -- error message expected
     feed('<cr>')
-    eq('latin1', eval('&encoding'))
+    neq(nil, string.find(eval('v:errmsg'), '^E474:'))
-    eq(4, eval('strwidth("Bär")'))
+    eq('utf-8', eval('&encoding'))
+    eq(3, eval('strwidth("Bär")'))
   end)
 
-  it('is not changed by `set all&`', function()
+  it('can be set to utf-8 without error', function()
-    -- we need to set &encoding to something non-default. Use 'latin1'
+    execute('set encoding=utf-8')
-    clear('--cmd', 'set enc=latin1')
+    eq("", eval('v:errmsg'))
-    execute('set all&')
-    eq('latin1', eval('&encoding'))
-    eq(4, eval('strwidth("Bär")'))
-  end)
 
+    clear('--cmd', 'set enc=utf-8')
+    eq("", eval('v:errmsg'))
+  end)
 end)

test/functional/shada/history_spec.lua

@@ -4,9 +4,7 @@ local nvim_command, funcs, meths, nvim_feed, eq =
   helpers.command, helpers.funcs, helpers.meths, helpers.feed, helpers.eq
 
 local shada_helpers = require('test.functional.shada.helpers')
-local reset, set_additional_cmd, clear =
+local reset, clear = shada_helpers.reset, shada_helpers.clear
-  shada_helpers.reset, shada_helpers.set_additional_cmd,
-  shada_helpers.clear
 
 describe('ShaDa support code', function()
   before_each(reset)
@@ -173,120 +171,48 @@ describe('ShaDa support code', function()
     eq('goo', funcs.getline(1))
   end)
 
-  it('dumps and loads history correctly when &encoding is not UTF-8', function()
+  it('dumps and loads history with UTF-8 characters', function()
-    set_additional_cmd('set encoding=latin1')
     reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_feed(':echo "«"\n')
-    nvim_feed(':echo "\171"\n')
-    nvim_command('qall')
-    reset()
-    eq('echo "\171"', funcs.histget(':', -1))
-  end)
-
-  it('dumps and loads history correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_feed(':echo "\171"\n')
-    set_additional_cmd('')
     nvim_command('qall')
     reset()
     eq('echo "«"', funcs.histget(':', -1))
   end)
 
-  it('dumps and loads history correctly when &encoding /= UTF-8 when loading',
+  it('dumps and loads replacement with UTF-8 characters',
   function()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/./«/ge')
-    nvim_feed(':echo "«"\n')
-    set_additional_cmd('set encoding=latin1')
-    nvim_command('qall')
-    reset()
-    eq('echo "\171"', funcs.histget(':', -1))
-  end)
-
-  it('dumps and loads replacement correctly when &encoding is not UTF-8',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('substitute/./\171/ge')
     nvim_command('qall!')
     reset()
     funcs.setline('.', {'.'})
     nvim_command('&')
-    eq('\171', funcs.getline('.'))
-  end)
-
-  it('dumps&loads replacement correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('substitute/./\171/ge')
-    set_additional_cmd('')
-    nvim_command('qall')
-    reset()
-    funcs.setline('.', {'.'})
-    nvim_command('&')
     eq('«', funcs.getline('.'))
   end)
 
-  it('dumps&loads replacement correctly when &encoding /= UTF-8 when loading',
+  it('dumps and loads substitute pattern with UTF-8 characters',
   function()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('substitute/«/./ge')
-    nvim_command('substitute/./«/ge')
-    set_additional_cmd('set encoding=latin1')
-    nvim_command('qall')
-    reset()
-    funcs.setline('.', {'.'})
-    nvim_command('&')
-    eq('\171', funcs.getline('.'))
-  end)
-
-  it('dumps and loads substitute pattern correctly when &encoding is not UTF-8',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('substitute/\171/./ge')
     nvim_command('qall!')
     reset()
-    funcs.setline('.', {'\171«'})
-    nvim_command('&')
-    eq('.«', funcs.getline('.'))
-  end)
-
-  it('dumps&loads s/pattern correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('substitute/\171/./ge')
-    set_additional_cmd('')
-    nvim_command('qall')
-    reset()
     funcs.setline('.', {'«\171'})
     nvim_command('&')
     eq('.\171', funcs.getline('.'))
   end)
 
-  it('dumps&loads s/pattern correctly when &encoding /= UTF-8 when loading',
+  it('dumps and loads search pattern with UTF-8 characters',
   function()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    nvim_command('silent! /«/')
-    nvim_command('substitute/«/./ge')
+    nvim_command('set shada+=/0')
-    set_additional_cmd('set encoding=latin1')
+    nvim_command('qall!')
-    nvim_command('qall')
     reset()
     funcs.setline('.', {'\171«'})
-    nvim_command('&')
+    nvim_command('~&')
-    eq('.«', funcs.getline('.'))
+    eq('\171', funcs.getline('.'))
+    eq('', funcs.histget('/', -1))
   end)
 
-  it('dumps and loads search pattern correctly when &encoding is not UTF-8',
+  it('dumps and loads search pattern with 8-bit single-byte',
   function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
     -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
     nvim_command('silent! /\171/')
     nvim_command('set shada+=/0')
@@ -298,33 +224,4 @@ describe('ShaDa support code', function()
     eq('', funcs.histget('/', -1))
   end)
 
-  it('dumps&loads /pattern correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('silent! /\171/')
-    nvim_command('set shada+=/0')
-    set_additional_cmd('')
-    nvim_command('qall')
-    reset()
-    funcs.setline('.', {'«\171'})
-    nvim_command('~&')
-    eq('\171', funcs.getline('.'))
-    eq('', funcs.histget('/', -1))
-  end)
-
-  it('dumps&loads /pattern correctly when &encoding /= UTF-8 when loading',
-  function()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    nvim_command('silent! /«/')
-    nvim_command('set shada+=/0')
-    set_additional_cmd('set encoding=latin1')
-    nvim_command('qall')
-    reset()
-    funcs.setline('.', {'\171«'})
-    nvim_command('~&')
-    eq('«', funcs.getline('.'))
-    eq('', funcs.histget('/', -1))
-  end)
 end)

test/functional/shada/registers_spec.lua

@@ -128,36 +128,24 @@ describe('ShaDa support code', function()
     eq({{}, ''}, getreg('h'))
   end)
 
-  it('dumps and loads register correctly when &encoding is not UTF-8',
+  it('dumps and loads register correctly with utf-8 contents',
   function()
-    set_additional_cmd('set encoding=latin1')
     reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    setreg('e', {'«'}, 'c')
-    setreg('e', {'\171'}, 'c')
-    nvim_command('qall')
-    reset()
-    eq({{'\171'}, 'v'}, getreg('e'))
-  end)
-
-  it('dumps and loads history correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    setreg('e', {'\171'}, 'c')
-    set_additional_cmd('')
     nvim_command('qall')
     reset()
     eq({{'«'}, 'v'}, getreg('e'))
   end)
 
-  it('dumps and loads history correctly when &encoding /= UTF-8 when loading',
+  it('dumps and loads history correctly with 8-bit single-byte',
   function()
+    reset()
     -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    setreg('e', {'«'}, 'c')
+    setreg('e', {'\171«'}, 'c')
-    set_additional_cmd('set encoding=latin1')
+    set_additional_cmd('')
     nvim_command('qall')
     reset()
-    eq({{'\171'}, 'v'}, getreg('e'))
+    eq({{'\171«'}, 'v'}, getreg('e'))
   end)
+
 end)

test/functional/shada/variables_spec.lua

@@ -91,35 +91,13 @@ describe('ShaDa support code', function()
     eq(0, funcs.exists('g:str_var'))
   end)
 
-  it('dumps and loads variables correctly when &encoding is not UTF-8',
+  it('dumps and loads variables correctly with utf-8 strings',
   function()
-    set_additional_cmd('set encoding=latin1')
     reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
+    meths.set_var('STRVAR', '«')
-    meths.set_var('STRVAR', '\171')
+    meths.set_var('LSTVAR', {'«'})
-    meths.set_var('LSTVAR', {'\171'})
+    meths.set_var('DCTVAR', {['«']='«'})
-    meths.set_var('DCTVAR', {['\171']='\171'})
+    meths.set_var('NESTEDVAR', {['«']={{'«'}, {['«']='«'}, {a='Test'}}})
-    meths.set_var('NESTEDVAR', {['\171']={{'\171'}, {['\171']='\171'},
-                                {a='Test'}}})
-    nvim_command('qall')
-    reset()
-    eq('\171', meths.get_var('STRVAR'))
-    eq({'\171'}, meths.get_var('LSTVAR'))
-    eq({['\171']='\171'}, meths.get_var('DCTVAR'))
-    eq({['\171']={{'\171'}, {['\171']='\171'}, {a='Test'}}},
-       meths.get_var('NESTEDVAR'))
-  end)
-
-  it('dumps and loads variables correctly when &encoding /= UTF-8 when dumping',
-  function()
-    set_additional_cmd('set encoding=latin1')
-    reset()
-    -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    meths.set_var('STRVAR', '\171')
-    meths.set_var('LSTVAR', {'\171'})
-    meths.set_var('DCTVAR', {['\171']='\171'})
-    meths.set_var('NESTEDVAR', {['\171']={{'\171'}, {['\171']='\171'},
-                                {a='Test'}}})
     set_additional_cmd('')
     nvim_command('qall')
     reset()
@@ -129,20 +107,22 @@ describe('ShaDa support code', function()
     eq({['«']={{'«'}, {['«']='«'}, {a='Test'}}}, meths.get_var('NESTEDVAR'))
   end)
 
-  it('dumps and loads variables correctly when &encoding /= UTF-8 when loading',
+  it('dumps and loads variables correctly with 8-bit strings',
   function()
+    reset()
     -- \171 is U+00AB LEFT-POINTING DOUBLE ANGLE QUOTATION MARK in latin1
-    meths.set_var('STRVAR', '«')
+    -- This is invalid unicode, but we should still dump and restore it.
-    meths.set_var('LSTVAR', {'«'})
+    meths.set_var('STRVAR', '\171')
-    meths.set_var('DCTVAR', {['«']='«'})
+    meths.set_var('LSTVAR', {'\171'})
-    meths.set_var('NESTEDVAR', {['«']={{'«'}, {['«']='«'}, {a='Test'}}})
+    meths.set_var('DCTVAR', {['«\171']='«\171'})
-    set_additional_cmd('set encoding=latin1')
+    meths.set_var('NESTEDVAR', {['\171']={{'\171«'}, {['\171']='\171'},
+                                {a='Test'}}})
     nvim_command('qall')
     reset()
     eq('\171', meths.get_var('STRVAR'))
     eq({'\171'}, meths.get_var('LSTVAR'))
-    eq({['\171']='\171'}, meths.get_var('DCTVAR'))
+    eq({['«\171']='«\171'}, meths.get_var('DCTVAR'))
-    eq({['\171']={{'\171'}, {['\171']='\171'}, {a='Test'}}},
+    eq({['\171']={{'\171«'}, {['\171']='\171'}, {a='Test'}}},
        meths.get_var('NESTEDVAR'))
   end)