|
|
|
|
@@ -18,26 +18,28 @@ end
|
|
|
|
|
|
|
|
|
|
local utf8charpattern = '[%z\1-\127\194-\244][\128-\191]*'
|
|
|
|
|
|
|
|
|
|
--- Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character.
|
|
|
|
|
--- @param byte number First byte of a character.
|
|
|
|
|
--- @return number
|
|
|
|
|
---Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character.
|
|
|
|
|
---@param byte number First byte of a character.
|
|
|
|
|
---@return integer
|
|
|
|
|
local function utf8symbollen(byte)
|
|
|
|
|
ensureArgType(byte, "number", 1, "utf8symbollen")
|
|
|
|
|
|
|
|
|
|
return not byte and 0 or (byte < 0x80 and 1) or (byte >= 0xF0 and 4) or (byte >= 0xE0 and 3) or (byte >= 0xC0 and 2) or 1
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
---@type table<integer, integer>
|
|
|
|
|
local head_table = {}
|
|
|
|
|
|
|
|
|
|
for i = 0, 255 do
|
|
|
|
|
head_table[i] = utf8symbollen(i)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
head_table[256] = 0
|
|
|
|
|
|
|
|
|
|
--- Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`.
|
|
|
|
|
--- @param str string Input string.
|
|
|
|
|
--- @param bs number Position of character in line.
|
|
|
|
|
--- @return number
|
|
|
|
|
---Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`.
|
|
|
|
|
---@param str string Input string.
|
|
|
|
|
---@param bs integer Position of character in line.
|
|
|
|
|
---@return integer
|
|
|
|
|
local function utf8charbytes(str, bs)
|
|
|
|
|
ensureArgType(str, "string", 1, "utf8charbytes")
|
|
|
|
|
ensureArgType(bs, "number", 2, "utf8charbytes")
|
|
|
|
|
@@ -47,10 +49,10 @@ local function utf8charbytes(str, bs)
|
|
|
|
|
return head_table[byte(str, bs) or 256]
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Returns the new position in the string `str`, skipping the current character.
|
|
|
|
|
--- @param str string Input string.
|
|
|
|
|
--- @param bs number Position of character in line.
|
|
|
|
|
--- @return number
|
|
|
|
|
---Returns the new position in the string `str`, skipping the current character.
|
|
|
|
|
---@param str string Input string.
|
|
|
|
|
---@param bs integer Position of character in line.
|
|
|
|
|
---@return integer
|
|
|
|
|
local function utf8next(str, bs)
|
|
|
|
|
ensureArgType(str, "string", 1, "utf8next")
|
|
|
|
|
ensureArgType(bs, "number", 2, "utf8next")
|
|
|
|
|
@@ -60,9 +62,9 @@ local function utf8next(str, bs)
|
|
|
|
|
return bs + utf8charbytes(str, bs)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Returns the number of characters in a UTF-8 string
|
|
|
|
|
--- @param str string Input string.
|
|
|
|
|
--- @return number
|
|
|
|
|
---Returns the number of characters in a UTF-8 string
|
|
|
|
|
---@param str string Input string.
|
|
|
|
|
---@return integer
|
|
|
|
|
function string_utf8.len(str)
|
|
|
|
|
ensureArgType(str, "string", 1, "string_utf8.len")
|
|
|
|
|
|
|
|
|
|
@@ -81,12 +83,12 @@ function string_utf8.len(str)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
--- Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes.
|
|
|
|
|
--- @param str string Input string.
|
|
|
|
|
--- @param i number The starting character index (default: 1). Negative values count from the end.
|
|
|
|
|
--- @param j number The ending character index (default: i). Negative values count from the end.
|
|
|
|
|
--- @overload fun(s: string, i: number)
|
|
|
|
|
--- @return string
|
|
|
|
|
---Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes.
|
|
|
|
|
---@param str string Input string.
|
|
|
|
|
---@param i integer The starting character index (default: 1). Negative values count from the end.
|
|
|
|
|
---@param j integer The ending character index (default: i). Negative values count from the end.
|
|
|
|
|
---@overload fun(s: string, i: integer)
|
|
|
|
|
---@return string
|
|
|
|
|
function string_utf8.sub(str, i, j)
|
|
|
|
|
j = j or -1
|
|
|
|
|
|
|
|
|
|
@@ -129,15 +131,14 @@ function string_utf8.sub(str, i, j)
|
|
|
|
|
return sub(str, start, finish)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Returns a UTF-8 string from the given Unicode codes.
|
|
|
|
|
--- @param ... number List of Unicode codes.
|
|
|
|
|
--- @return string
|
|
|
|
|
---Returns a UTF-8 string from the given Unicode codes.
|
|
|
|
|
---@param ... integer List of Unicode codes.
|
|
|
|
|
---@return string
|
|
|
|
|
function string_utf8.char(...)
|
|
|
|
|
local codes = {...}
|
|
|
|
|
|
|
|
|
|
assert(codes ~= {}, 'bad argument #1 to \'string_utf8.char\' (key cannot be nil)')
|
|
|
|
|
|
|
|
|
|
--- @type int[]
|
|
|
|
|
local result = {}
|
|
|
|
|
|
|
|
|
|
for _, unicode in ipairs(codes) do
|
|
|
|
|
@@ -183,11 +184,11 @@ local shift_6 = 16 -- 2^6
|
|
|
|
|
local shift_12 = 4096 -- 2^12
|
|
|
|
|
local shift_18 = 262144 -- 2^18
|
|
|
|
|
|
|
|
|
|
--- Converts UTF-8 encoded characters in a string to their corresponding Unicode code points.
|
|
|
|
|
--- @param str string The input string containing UTF-8 encoded characters.
|
|
|
|
|
--- @param ibs number The starting byte position in the string (inclusive).
|
|
|
|
|
--- @param jbs number The ending byte position in the string (inclusive).
|
|
|
|
|
--- @return number | number...
|
|
|
|
|
---Converts UTF-8 encoded characters in a string to their corresponding Unicode code points.
|
|
|
|
|
---@param str string The input string containing UTF-8 encoded characters.
|
|
|
|
|
---@param ibs integer The starting byte position in the string (inclusive).
|
|
|
|
|
---@param jbs integer The ending byte position in the string (inclusive).
|
|
|
|
|
---@return integer?, integer?
|
|
|
|
|
function string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
ensureArgType(str, "string", 1, "string_utf8.unicode")
|
|
|
|
|
ensureArgType(ibs, "number", 2, "string_utf8.unicode")
|
|
|
|
|
@@ -201,12 +202,15 @@ function string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
bytes = utf8charbytes(str, ibs)
|
|
|
|
|
if bytes == 0 then return end
|
|
|
|
|
|
|
|
|
|
---@type integer
|
|
|
|
|
local unicode
|
|
|
|
|
|
|
|
|
|
if bytes == 1 then unicode = byte(str, ibs, ibs) end
|
|
|
|
|
if bytes == 2 then
|
|
|
|
|
---@type integer, integer
|
|
|
|
|
local byte0,byte1 = byte(str, ibs, ibs + 1)
|
|
|
|
|
if byte0 and byte1 then
|
|
|
|
|
---@type integer, integer
|
|
|
|
|
local code0,code1 = byte0-0xC0,byte1-0x80
|
|
|
|
|
unicode = code0*shift_6 + code1
|
|
|
|
|
else
|
|
|
|
|
@@ -214,8 +218,10 @@ function string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
if bytes == 3 then
|
|
|
|
|
---@type integer, integer, integer
|
|
|
|
|
local byte0,byte1,byte2 = byte(str, ibs, ibs + 2)
|
|
|
|
|
if byte0 and byte1 and byte2 then
|
|
|
|
|
---@type integer, integer, integer
|
|
|
|
|
local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80
|
|
|
|
|
unicode = code0*shift_12 + code1*shift_6 + code2
|
|
|
|
|
else
|
|
|
|
|
@@ -223,8 +229,10 @@ function string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
if bytes == 4 then
|
|
|
|
|
---@type integer, integer, integer, integer
|
|
|
|
|
local byte0,byte1,byte2,byte3 = byte(str, ibs, ibs + 3)
|
|
|
|
|
if byte0 and byte1 and byte2 and byte3 then
|
|
|
|
|
---@type integer, integer, integer, integer
|
|
|
|
|
local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80
|
|
|
|
|
unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3
|
|
|
|
|
else
|
|
|
|
|
@@ -239,13 +247,13 @@ function string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Returns Unicode code points for characters in a UTF-8 string within the specified range.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string.
|
|
|
|
|
--- @param i number The starting character index (default: 1). Negative values count from the end.
|
|
|
|
|
--- @param j number The ending character index (default: i). Negative values count from the end.
|
|
|
|
|
--- @overload fun(str: string)
|
|
|
|
|
--- @overload fun(str: string, i: number)
|
|
|
|
|
--- @return number | number...
|
|
|
|
|
---Returns Unicode code points for characters in a UTF-8 string within the specified range.
|
|
|
|
|
---@param str string The input UTF-8 encoded string.
|
|
|
|
|
---@param i integer The starting character index (default: 1). Negative values count from the end.
|
|
|
|
|
---@param j integer The ending character index (default: i). Negative values count from the end.
|
|
|
|
|
---@overload fun(str: string)
|
|
|
|
|
---@overload fun(str: string, i: number)
|
|
|
|
|
---@return integer?, integer?
|
|
|
|
|
function string_utf8.byte(str, i, j)
|
|
|
|
|
ensureArgType(str, "string", 1, "string_utf8.byte")
|
|
|
|
|
ensureArgType(i, "number", 2, "string_utf8.byte")
|
|
|
|
|
@@ -279,7 +287,7 @@ function string_utf8.byte(str, i, j)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
if not ibs or not jbs then
|
|
|
|
|
return nil
|
|
|
|
|
return
|
|
|
|
|
end
|
|
|
|
|
else
|
|
|
|
|
ibs, jbs = 1, 1
|
|
|
|
|
@@ -288,10 +296,10 @@ function string_utf8.byte(str, i, j)
|
|
|
|
|
return string_utf8.unicode(str, ibs, jbs)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Creates an iterator to generate UTF-8 substrings of a specified length.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string.
|
|
|
|
|
--- @param sub_len? number The length of each substring in characters (default: 1).
|
|
|
|
|
--- @return fun(skip_ptr: table | nil, bs: number | nil)
|
|
|
|
|
---Creates an iterator to generate UTF-8 substrings of a specified length.
|
|
|
|
|
---@param str string The input UTF-8 encoded string.
|
|
|
|
|
---@param sub_len integer? The length of each substring in characters (default: 1).
|
|
|
|
|
---@return fun(skip_ptr: table | nil, bs: integer | nil):(integer?, string?, integer?)
|
|
|
|
|
function string_utf8.gensub(str, sub_len)
|
|
|
|
|
sub_len = sub_len or 1
|
|
|
|
|
|
|
|
|
|
@@ -302,6 +310,9 @@ function string_utf8.gensub(str, sub_len)
|
|
|
|
|
|
|
|
|
|
local max_len = #str
|
|
|
|
|
|
|
|
|
|
---@param skip_ptr table
|
|
|
|
|
---@param bs integer
|
|
|
|
|
---@return integer?, string?, integer?
|
|
|
|
|
return function(skip_ptr, bs)
|
|
|
|
|
bs = (bs and bs or 1) + (skip_ptr and (skip_ptr[1] or 0) or 0)
|
|
|
|
|
|
|
|
|
|
@@ -315,9 +326,9 @@ function string_utf8.gensub(str, sub_len)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Reverses a UTF-8 encoded string while preserving multi-byte characters.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string.
|
|
|
|
|
--- @return string
|
|
|
|
|
---Reverses a UTF-8 encoded string while preserving multi-byte characters.
|
|
|
|
|
---@param str string The input UTF-8 encoded string.
|
|
|
|
|
---@return string
|
|
|
|
|
function string_utf8.reverse(str)
|
|
|
|
|
ensureArgType(str, "string", 1, "string_utf8.reverse")
|
|
|
|
|
|
|
|
|
|
@@ -330,11 +341,11 @@ function string_utf8.reverse(str)
|
|
|
|
|
return result
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Validates UTF-8 encoded characters in a string starting from a given byte position.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string to validate.
|
|
|
|
|
--- @param bs number The starting byte position for validation (default: 1).
|
|
|
|
|
--- @overload fun(str: string)
|
|
|
|
|
--- @return number?
|
|
|
|
|
---Validates UTF-8 encoded characters in a string starting from a given byte position.
|
|
|
|
|
---@param str string The input UTF-8 encoded string to validate.
|
|
|
|
|
---@param bs integer The starting byte position for validation (default: 1).
|
|
|
|
|
---@overload fun(str: string)
|
|
|
|
|
---@return integer?, integer?, integer?, integer?
|
|
|
|
|
local function utf8validator(str, bs)
|
|
|
|
|
bs = bs or 1
|
|
|
|
|
|
|
|
|
|
@@ -354,7 +365,8 @@ local function utf8validator(str, bs)
|
|
|
|
|
return bs + 1
|
|
|
|
|
elseif c >= 128 and c <= 193 then
|
|
|
|
|
return bs + 1, bs, 1, c
|
|
|
|
|
-- UTF8-2
|
|
|
|
|
|
|
|
|
|
-- UTF8-2
|
|
|
|
|
elseif c >= 194 and c <= 223 then
|
|
|
|
|
local c2 = byte(str, bs + 1)
|
|
|
|
|
if not c2 or c2 < 128 or c2 > 191 then
|
|
|
|
|
@@ -362,8 +374,10 @@ local function utf8validator(str, bs)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
return bs + 2
|
|
|
|
|
-- UTF8-3
|
|
|
|
|
|
|
|
|
|
-- UTF8-3
|
|
|
|
|
elseif c >= 224 and c <= 239 then
|
|
|
|
|
---@type integer?
|
|
|
|
|
local c2 = byte(str, bs + 1)
|
|
|
|
|
|
|
|
|
|
if not c2 then
|
|
|
|
|
@@ -387,6 +401,7 @@ local function utf8validator(str, bs)
|
|
|
|
|
return bs + 3
|
|
|
|
|
-- UTF8-4
|
|
|
|
|
elseif c >= 240 and c <= 244 then
|
|
|
|
|
---@type integer?
|
|
|
|
|
local c2 = byte(str, bs + 1)
|
|
|
|
|
|
|
|
|
|
if not c2 then
|
|
|
|
|
@@ -418,10 +433,10 @@ local function utf8validator(str, bs)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Validates a UTF-8 encoded string for correctness.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string to validate.
|
|
|
|
|
--- @param byte_pos? number The starting byte position for validation (default: 1).
|
|
|
|
|
--- @return boolean | table
|
|
|
|
|
---Validates a UTF-8 encoded string for correctness.
|
|
|
|
|
---@param str string The input UTF-8 encoded string to validate.
|
|
|
|
|
---@param byte_pos? integer The starting byte position for validation (default: 1).
|
|
|
|
|
---@return boolean, table
|
|
|
|
|
function string_utf8.validate(str, byte_pos)
|
|
|
|
|
byte_pos = byte_pos or 1
|
|
|
|
|
|
|
|
|
|
@@ -437,12 +452,13 @@ function string_utf8.validate(str, byte_pos)
|
|
|
|
|
result[#result + 1] = { pos = bs, part = part, code = code }
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
return #result == 0, result
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
--- Creates an iterator to traverse UTF-8 encoded characters in a string.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string.
|
|
|
|
|
--- @return fun(skip_ptr: table|nil)
|
|
|
|
|
---Creates an iterator to traverse UTF-8 encoded characters in a string.
|
|
|
|
|
---@param str string The input UTF-8 encoded string.
|
|
|
|
|
---@return fun(skip_ptr: table|nil):integer?, integer?, integer?
|
|
|
|
|
function string_utf8.codes(str)
|
|
|
|
|
|
|
|
|
|
ensureArgType(str, "string", 1, "string_utf8.codes")
|
|
|
|
|
@@ -476,12 +492,12 @@ n > 0: n jumps forward
|
|
|
|
|
n < 0: n more scans backwards
|
|
|
|
|
--]]
|
|
|
|
|
|
|
|
|
|
--- Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position.
|
|
|
|
|
--- @param str string The input UTF-8 encoded string.
|
|
|
|
|
--- @param n number The character index to find. If `n` is negative, counts from the end.
|
|
|
|
|
--- @param bs number The starting byte position (default: 1 for positive `n`, end of string for negative `n`).
|
|
|
|
|
--- @overload fun(str: string, n: number)
|
|
|
|
|
--- @return number | nil
|
|
|
|
|
---Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position.
|
|
|
|
|
---@param str string The input UTF-8 encoded string.
|
|
|
|
|
---@param n integer The character index to find. If `n` is negative, counts from the end.
|
|
|
|
|
---@param bs integer The starting byte position (default: 1 for positive `n`, end of string for negative `n`).
|
|
|
|
|
---@overload fun(str: string, n: number)
|
|
|
|
|
---@return number | nil
|
|
|
|
|
function string_utf8.offset(str, n, bs)
|
|
|
|
|
local l = #str
|
|
|
|
|
if not bs then
|
|
|
|
|
|