Files
utf8_api/init.lua
2026-01-17 22:24:37 +03:00

609 lines
16 KiB
Lua

string_utf8 = {}
dofile(core.get_modpath(core.get_current_modname()).."/utf8data.lua")
local byte = string.byte
local char = string.char
local len = string.len
local sub = string.sub
local ensureArgType
-- Ensure API
if Ensure ~= nil then
ensureArgType = Ensure.argType
else
ensureArgType = function(arg, expectedType, n, function_name) end
end
local utf8charpattern = '[%z\1-\127\194-\244][\128-\191]*'
---Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character.
---@param byte number First byte of a character.
---@return integer
local function utf8symbollen(byte)
ensureArgType(byte, "number", 1, "utf8symbollen")
return not byte and 0 or (byte < 0x80 and 1) or (byte >= 0xF0 and 4) or (byte >= 0xE0 and 3) or (byte >= 0xC0 and 2) or 1
end
---@type table<integer, integer>
local head_table = {}
for i = 0, 255 do
head_table[i] = utf8symbollen(i)
end
head_table[256] = 0
---Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`.
---@param str string Input string.
---@param bs integer Position of character in line.
---@return integer
local function utf8charbytes(str, bs)
ensureArgType(str, "string", 1, "utf8charbytes")
ensureArgType(bs, "number", 2, "utf8charbytes")
assert(#str ~= 0, 'bad argument #1 to \'utf8charbytes\' (string cannot be empty)')
return head_table[byte(str, bs) or 256]
end
---Returns the new position in the string `str`, skipping the current character.
---@param str string Input string.
---@param bs integer Position of character in line.
---@return integer
local function utf8next(str, bs)
ensureArgType(str, "string", 1, "utf8next")
ensureArgType(bs, "number", 2, "utf8next")
assert(#str ~= 0, 'bad argument #1 to \'utf8next\' (string cannot be empty)')
return bs + utf8charbytes(str, bs)
end
---Returns the number of characters in a UTF-8 string
---@param str string Input string.
---@return integer
function string_utf8.len(str)
ensureArgType(str, "string", 1, "string_utf8.len")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.len\' (string cannot be empty)')
local bs = 1
local bytes = len(str)
local length = 0
while bs <= bytes do
length = length + 1
bs = utf8next(str, bs)
end
return length
end
---Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes.
---@param str string Input string.
---@param i integer The starting character index (default: 1). Negative values count from the end.
---@param j integer The ending character index (default: i). Negative values count from the end.
---@overload fun(s: string, i: integer)
---@return string
function string_utf8.sub(str, i, j)
j = j or -1
ensureArgType(str, "string", 1, "string_utf8.sub")
ensureArgType(i, "number", 2, "string_utf8.sub")
ensureArgType(j, "number", 3, "string_utf8.sub")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.sub\' (string cannot be empty)')
local bs = 1
local bytes = len(str)
local length = 0
local l = (i >= 0 and j >= 0) or string_utf8.len(str)
i = (i >= 0) and i or l + i + 1
j = (j >= 0) and j or l + j + 1
assert(i < j, 'bad arguments #2 & #3 to \'string_utf8.sub\' (#2 cannot be more #3)')
local start, finish = 1, bytes
while bs <= bytes do
length = length + 1
if length == i then
start = bs
end
bs = utf8next(str, bs)
if length == j then
finish = bs - 1
break
end
end
if i > length then start = bytes + 1 end
if j < 1 then finish = 0 end
return sub(str, start, finish)
end
---Returns a UTF-8 string from the given Unicode codes.
---@param ... integer List of Unicode codes.
---@return string
function string_utf8.char(...)
local codes = {...}
assert(codes ~= {}, 'bad argument #1 to \'string_utf8.char\' (key cannot be nil)')
local result = {}
for _, unicode in ipairs(codes) do
if unicode <= 0x7F then
result[#result + 1] = unicode
elseif unicode <= 0x7FF then
local b0 = 0xC0 + math.floor(unicode / 0x40);
local b1 = 0x80 + (unicode % 0x40);
result[#result + 1] = b0
result[#result + 1] = b1
elseif unicode <= 0xFFFF then
local b0 = 0xE0 + math.floor(unicode / 0x1000);
local b1 = 0x80 + (math.floor(unicode / 0x40) % 0x40);
local b2 = 0x80 + (unicode % 0x40);
result[#result + 1] = b0
result[#result + 1] = b1
result[#result + 1] = b2
elseif unicode <= 0x10FFFF then
local code = unicode
local b3 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b2 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b1 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b0 = 0xF0 + code;
result[#result + 1] = b0
result[#result + 1] = b1
result[#result + 1] = b2
result[#result + 1] = b3
else
error 'Unicode cannot be greater than U+10FFFF!'
end
end
return char(unpack(result, 1, #result))
end
local shift_6 = 16 -- 2^6
local shift_12 = 4096 -- 2^12
local shift_18 = 262144 -- 2^18
---Converts UTF-8 encoded characters in a string to their corresponding Unicode code points.
---@param str string The input string containing UTF-8 encoded characters.
---@param ibs integer The starting byte position in the string (inclusive).
---@param jbs integer The ending byte position in the string (inclusive).
---@return integer...
function string_utf8.unicode(str, ibs, jbs)
ensureArgType(str, "string", 1, "string_utf8.unicode")
ensureArgType(ibs, "number", 2, "string_utf8.unicode")
ensureArgType(jbs, "number", 3, "string_utf8.unicode")
assert(ibs < jbs, 'bad arguments #2 & #3 to \'string_utf8.unicode\' (#2 cannot be more #3)')
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.unicode\' (string cannot be empty)')
local bytes
bytes = utf8charbytes(str, ibs)
if bytes == 0 then return end
---@type integer
local unicode
if bytes == 1 then unicode = byte(str, ibs, ibs) end
if bytes == 2 then
---@type integer, integer
local byte0,byte1 = byte(str, ibs, ibs + 1)
if byte0 and byte1 then
---@type integer, integer
local code0,code1 = byte0-0xC0,byte1-0x80
unicode = code0*shift_6 + code1
else
unicode = byte0
end
end
if bytes == 3 then
---@type integer, integer, integer
local byte0,byte1,byte2 = byte(str, ibs, ibs + 2)
if byte0 and byte1 and byte2 then
---@type integer, integer, integer
local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80
unicode = code0*shift_12 + code1*shift_6 + code2
else
unicode = byte0
end
end
if bytes == 4 then
---@type integer, integer, integer, integer
local byte0,byte1,byte2,byte3 = byte(str, ibs, ibs + 3)
if byte0 and byte1 and byte2 and byte3 then
---@type integer, integer, integer, integer
local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80
unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3
else
unicode = byte0
end
end
if ibs == jbs then
return unicode
else
return unicode, string_utf8.unicode(str, ibs+bytes, jbs)
end
end
---Returns Unicode code points for characters in a UTF-8 string within the specified range.
---@param str string The input UTF-8 encoded string.
---@param i integer The starting character index (default: 1). Negative values count from the end.
---@param j integer The ending character index (default: i). Negative values count from the end.
---@overload fun(str: string)
---@overload fun(str: string, i: number)
---@return integer...
function string_utf8.byte(str, i, j)
ensureArgType(str, "string", 1, "string_utf8.byte")
ensureArgType(i, "number", 2, "string_utf8.byte")
ensureArgType(j, "number", 3, "string_utf8.byte")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.byte\' (string cannot be empty)')
local ibs, jbs
if i or j then
i = i or 1
j = j or i
local str_len = string_utf8.len(str)
i = i < 0 and str_len + i + 1 or i
j = j < 0 and str_len + j + 1 or j
j = j > str_len and str_len or j
if i > j then return end
for p = 1, i - 1 do
ibs = utf8next(str, ibs or 1)
end
if i == j then
jbs = ibs
else
for p = 1, j - 1 do
jbs = utf8next(str, jbs or 1)
end
end
if not ibs or not jbs then
return
end
else
ibs, jbs = 1, 1
end
return string_utf8.unicode(str, ibs, jbs)
end
---Creates an iterator to generate UTF-8 substrings of a specified length.
---@param str string The input UTF-8 encoded string.
---@param sub_len integer? The length of each substring in characters (default: 1).
---@return fun(skip_ptr: table | nil, bs: integer | nil)
function string_utf8.gensub(str, sub_len)
sub_len = sub_len or 1
ensureArgType(str, "string", 1, "string_utf8.gensub")
ensureArgType(sub_len, "number", 2, "string_utf8.gensub")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.gensub\' (string cannot be empty)')
local max_len = #str
---@param skip_ptr table
---@param bs integer
return function(skip_ptr, bs)
bs = (bs and bs or 1) + (skip_ptr and (skip_ptr[1] or 0) or 0)
local nbs = bs
if bs > max_len then return nil end
for i = 1, sub_len do
nbs = utf8next(str, nbs)
end
return nbs, sub(str, bs, nbs - 1), bs
end
end
---Reverses a UTF-8 encoded string while preserving multi-byte characters.
---@param str string The input UTF-8 encoded string.
---@return string
function string_utf8.reverse(str)
ensureArgType(str, "string", 1, "string_utf8.reverse")
local result = ''
for _, w in string_utf8.gensub(str) do
result = w .. result
end
return result
end
---Validates UTF-8 encoded characters in a string starting from a given byte position.
---@param str string The input UTF-8 encoded string to validate.
---@param bs integer The starting byte position for validation (default: 1).
---@overload fun(str: string)
---@return integer?, integer?, integer?, integer?
local function utf8validator(str, bs)
bs = bs or 1
ensureArgType(str, "string", 1, "utf8validator")
ensureArgType(bs, "number", 2, "utf8validator")
assert(#str ~= 0, 'bad argument #1 to \'utf8validator\' (string cannot be empty)')
local c = byte(str, bs)
if not c then return end
-- determine bytes needed for character, based ons RFC 3629
-- UTF8-1
if c >= 0 and c <= 127 then
return bs + 1
elseif c >= 128 and c <= 193 then
return bs + 1, bs, 1, c
-- UTF8-2
elseif c >= 194 and c <= 223 then
local c2 = byte(str, bs + 1)
if not c2 or c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
return bs + 2
-- UTF8-3
elseif c >= 224 and c <= 239 then
---@type integer?
local c2 = byte(str, bs + 1)
if not c2 then
return bs + 2, bs, 2, c2
end
-- validate byte 2
if c == 224 and (c2 < 160 or c2 > 191) then
return bs + 2, bs, 2, c2
elseif c == 237 and (c2 < 128 or c2 > 159) then
return bs + 2, bs, 2, c2
elseif c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
local c3 = byte(str, bs + 2)
if not c3 or c3 < 128 or c3 > 191 then
return bs + 3, bs, 3, c3
end
return bs + 3
-- UTF8-4
elseif c >= 240 and c <= 244 then
---@type integer?
local c2 = byte(str, bs + 1)
if not c2 then
return bs + 2, bs, 2, c2
end
-- validate byte 2
if c == 240 and (c2 < 144 or c2 > 191) then
return bs + 2, bs, 2, c2
elseif c == 244 and (c2 < 128 or c2 > 143) then
return bs + 2, bs, 2, c2
elseif c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
local c3 = byte(str, bs + 2)
if not c3 or c3 < 128 or c3 > 191 then
return bs + 3, bs, 3, c3
end
local c4 = byte(str, bs + 3)
if not c4 or c4 < 128 or c4 > 191 then
return bs + 4, bs, 4, c4
end
return bs + 4
else -- c > 245
return bs + 1, bs, 1, c
end
end
---Validates a UTF-8 encoded string for correctness.
---@param str string The input UTF-8 encoded string to validate.
---@param byte_pos? integer The starting byte position for validation (default: 1).
---@return boolean, table
function string_utf8.validate(str, byte_pos)
byte_pos = byte_pos or 1
ensureArgType(str, "string", 1, "string_utf8.validate")
ensureArgType(byte_pos, "number", 2, "string_utf8.validate")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.validate\' (string cannot be empty)')
local result = {}
for nbs, bs, part, code in utf8validator, str, byte_pos do
if bs then
result[#result + 1] = { pos = bs, part = part, code = code }
end
end
return #result == 0, result
end
---Creates an iterator to traverse UTF-8 encoded characters in a string.
---@param str string The input UTF-8 encoded string.
---@return fun(skip_ptr: table|nil)
function string_utf8.codes(str)
ensureArgType(str, "string", 1, "string_utf8.codes")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.codes\' (string cannot be empty)')
local max_len = #str
local bs = 1
return function(skip_ptr)
if bs > max_len then return nil end
local pbs = bs
bs = utf8next(str, pbs)
return pbs, string_utf8.unicode(str, pbs, pbs), pbs
end
end
--[[
differs from Lua 5.3 utf8.offset in accepting any byte positions (not only head byte) for all n values
h - head, c - continuation, t - tail
hhhccthccthccthcthhh
^ start byte pos
searching current charracter head by moving backwards
hhhccthccthccthcthhh
^ head
n == 0: current position
n > 0: n jumps forward
n < 0: n more scans backwards
--]]
---Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position.
---@param str string The input UTF-8 encoded string.
---@param n integer The character index to find. If `n` is negative, counts from the end.
---@param bs integer The starting byte position (default: 1 for positive `n`, end of string for negative `n`).
---@overload fun(str: string, n: number)
---@return number | nil
function string_utf8.offset(str, n, bs)
local l = #str
if not bs then
if n < 0 then
bs = l + 1
else
bs = 1
end
end
ensureArgType(str, "string", 1, "string_utf8.offset")
ensureArgType(n, "number", 2, "string_utf8.offset")
ensureArgType(bs, "number", 3, "string_utf8.offset")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.offset\' (string cannot be empty)')
if bs <= 0 or bs > l + 1 then
error("bad argument #3 to 'string_utf8.offset' (position out of range)")
end
if n == 0 then
if bs == l + 1 then
return bs
end
while true do
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
return bs
end
bs = bs - 1
if bs < 1 then
return
end
end
elseif n < 0 then
bs = bs - 1
repeat
if bs < 1 then
return
end
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
n = n + 1
end
bs = bs - 1
until n == 0
return bs + 1
else
while true do
if bs > l then
return
end
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
n = n - 1
for i = 1, n do
if bs > l then
return
end
bs = utf8next(str, bs)
end
return bs
end
bs = bs - 1
end
end
end
--- Replaces UTF-8 characters in a string according to a mapping table.
--- @param str string The input UTF-8 encoded string.
--- @param mapping table A table where keys are UTF-8 characters (or patterns) and values are their replacements.
--- @return string
function string_utf8.replace(str, mapping)
ensureArgType(str, "string", 1, "string_utf8.replace")
ensureArgType(mapping, "table", 2, "string_utf8.replace")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.offset\' (string cannot be empty)')
local result = string.gsub( str, utf8charpattern, mapping )
return result
end
--- Converts all UTF-8 characters in a string to uppercase.
--- @param str string The input UTF-8 encoded string.
--- @return string
function string_utf8.upper(str)
ensureArgType(str, "string", 1, "string_utf8.upper")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.upper\' (string cannot be empty)')
return string_utf8.replace(str, string_utf8.lc_uc)
end
--- Converts all UTF-8 characters in a string to lowercase.
--- @param str string The input UTF-8 encoded string.
--- @return string
function string_utf8.lower(str)
ensureArgType(str, "string", 1, "string_utf8.lower")
assert(#str ~= 0, 'bad argument #1 to \'string_utf8.lower\' (string cannot be empty)')
return string_utf8.replace(str, string_utf8.uc_lc)
end