string_utf8 = {} dofile(core.get_modpath(core.get_current_modname()).."/utf8data.lua") local byte = string.byte local char = string.char local len = string.len local sub = string.sub local ensureArgType -- Ensure API if Ensure ~= nil then ensureArgType = Ensure.argType else ensureArgType = function(arg, expectedType, n, function_name) end end local utf8charpattern = '[%z\1-\127\194-\244][\128-\191]*' ---Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character. ---@param byte number First byte of a character. ---@return integer local function utf8symbollen(byte) ensureArgType(byte, "number", 1, "utf8symbollen") return not byte and 0 or (byte < 0x80 and 1) or (byte >= 0xF0 and 4) or (byte >= 0xE0 and 3) or (byte >= 0xC0 and 2) or 1 end ---@type table local head_table = {} for i = 0, 255 do head_table[i] = utf8symbollen(i) end head_table[256] = 0 ---Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`. ---@param str string Input string. ---@param bs integer Position of character in line. ---@return integer local function utf8charbytes(str, bs) ensureArgType(str, "string", 1, "utf8charbytes") ensureArgType(bs, "number", 2, "utf8charbytes") assert(#str ~= 0, 'bad argument #1 to \'utf8charbytes\' (string cannot be empty)') return head_table[byte(str, bs) or 256] end ---Returns the new position in the string `str`, skipping the current character. ---@param str string Input string. ---@param bs integer Position of character in line. ---@return integer local function utf8next(str, bs) ensureArgType(str, "string", 1, "utf8next") ensureArgType(bs, "number", 2, "utf8next") assert(#str ~= 0, 'bad argument #1 to \'utf8next\' (string cannot be empty)') return bs + utf8charbytes(str, bs) end ---Returns the number of characters in a UTF-8 string ---@param str string Input string. ---@return integer function string_utf8.len(str) ensureArgType(str, "string", 1, "string_utf8.len") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.len\' (string cannot be empty)') local bs = 1 local bytes = len(str) local length = 0 while bs <= bytes do length = length + 1 bs = utf8next(str, bs) end return length end ---Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes. ---@param str string Input string. ---@param i integer The starting character index (default: 1). Negative values count from the end. ---@param j integer The ending character index (default: i). Negative values count from the end. ---@overload fun(s: string, i: integer) ---@return string function string_utf8.sub(str, i, j) j = j or -1 ensureArgType(str, "string", 1, "string_utf8.sub") ensureArgType(i, "number", 2, "string_utf8.sub") ensureArgType(j, "number", 3, "string_utf8.sub") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.sub\' (string cannot be empty)') local bs = 1 local bytes = len(str) local length = 0 local l = (i >= 0 and j >= 0) or string_utf8.len(str) i = (i >= 0) and i or l + i + 1 j = (j >= 0) and j or l + j + 1 assert(i < j, 'bad arguments #2 & #3 to \'string_utf8.sub\' (#2 cannot be more #3)') local start, finish = 1, bytes while bs <= bytes do length = length + 1 if length == i then start = bs end bs = utf8next(str, bs) if length == j then finish = bs - 1 break end end if i > length then start = bytes + 1 end if j < 1 then finish = 0 end return sub(str, start, finish) end ---Returns a UTF-8 string from the given Unicode codes. ---@param ... integer List of Unicode codes. ---@return string function string_utf8.char(...) local codes = {...} assert(codes ~= {}, 'bad argument #1 to \'string_utf8.char\' (key cannot be nil)') local result = {} for _, unicode in ipairs(codes) do if unicode <= 0x7F then result[#result + 1] = unicode elseif unicode <= 0x7FF then local b0 = 0xC0 + math.floor(unicode / 0x40); local b1 = 0x80 + (unicode % 0x40); result[#result + 1] = b0 result[#result + 1] = b1 elseif unicode <= 0xFFFF then local b0 = 0xE0 + math.floor(unicode / 0x1000); local b1 = 0x80 + (math.floor(unicode / 0x40) % 0x40); local b2 = 0x80 + (unicode % 0x40); result[#result + 1] = b0 result[#result + 1] = b1 result[#result + 1] = b2 elseif unicode <= 0x10FFFF then local code = unicode local b3 = 0x80 + (code % 0x40); code = math.floor(code / 0x40) local b2 = 0x80 + (code % 0x40); code = math.floor(code / 0x40) local b1 = 0x80 + (code % 0x40); code = math.floor(code / 0x40) local b0 = 0xF0 + code; result[#result + 1] = b0 result[#result + 1] = b1 result[#result + 1] = b2 result[#result + 1] = b3 else error 'Unicode cannot be greater than U+10FFFF!' end end return char(unpack(result, 1, #result)) end local shift_6 = 16 -- 2^6 local shift_12 = 4096 -- 2^12 local shift_18 = 262144 -- 2^18 ---Converts UTF-8 encoded characters in a string to their corresponding Unicode code points. ---@param str string The input string containing UTF-8 encoded characters. ---@param ibs integer The starting byte position in the string (inclusive). ---@param jbs integer The ending byte position in the string (inclusive). ---@return integer?, integer? function string_utf8.unicode(str, ibs, jbs) ensureArgType(str, "string", 1, "string_utf8.unicode") ensureArgType(ibs, "number", 2, "string_utf8.unicode") ensureArgType(jbs, "number", 3, "string_utf8.unicode") assert(ibs < jbs, 'bad arguments #2 & #3 to \'string_utf8.unicode\' (#2 cannot be more #3)') assert(#str ~= 0, 'bad argument #1 to \'string_utf8.unicode\' (string cannot be empty)') local bytes bytes = utf8charbytes(str, ibs) if bytes == 0 then return end ---@type integer local unicode if bytes == 1 then unicode = byte(str, ibs, ibs) end if bytes == 2 then ---@type integer, integer local byte0,byte1 = byte(str, ibs, ibs + 1) if byte0 and byte1 then ---@type integer, integer local code0,code1 = byte0-0xC0,byte1-0x80 unicode = code0*shift_6 + code1 else unicode = byte0 end end if bytes == 3 then ---@type integer, integer, integer local byte0,byte1,byte2 = byte(str, ibs, ibs + 2) if byte0 and byte1 and byte2 then ---@type integer, integer, integer local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80 unicode = code0*shift_12 + code1*shift_6 + code2 else unicode = byte0 end end if bytes == 4 then ---@type integer, integer, integer, integer local byte0,byte1,byte2,byte3 = byte(str, ibs, ibs + 3) if byte0 and byte1 and byte2 and byte3 then ---@type integer, integer, integer, integer local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80 unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3 else unicode = byte0 end end if ibs == jbs then return unicode else return unicode, string_utf8.unicode(str, ibs+bytes, jbs) end end ---Returns Unicode code points for characters in a UTF-8 string within the specified range. ---@param str string The input UTF-8 encoded string. ---@param i integer The starting character index (default: 1). Negative values count from the end. ---@param j integer The ending character index (default: i). Negative values count from the end. ---@overload fun(str: string) ---@overload fun(str: string, i: number) ---@return integer?, integer? function string_utf8.byte(str, i, j) ensureArgType(str, "string", 1, "string_utf8.byte") ensureArgType(i, "number", 2, "string_utf8.byte") ensureArgType(j, "number", 3, "string_utf8.byte") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.byte\' (string cannot be empty)') local ibs, jbs if i or j then i = i or 1 j = j or i local str_len = string_utf8.len(str) i = i < 0 and str_len + i + 1 or i j = j < 0 and str_len + j + 1 or j j = j > str_len and str_len or j if i > j then return end for p = 1, i - 1 do ibs = utf8next(str, ibs or 1) end if i == j then jbs = ibs else for p = 1, j - 1 do jbs = utf8next(str, jbs or 1) end end if not ibs or not jbs then return end else ibs, jbs = 1, 1 end return string_utf8.unicode(str, ibs, jbs) end ---Creates an iterator to generate UTF-8 substrings of a specified length. ---@param str string The input UTF-8 encoded string. ---@param sub_len integer? The length of each substring in characters (default: 1). ---@return fun(skip_ptr: table | nil, bs: integer | nil):(integer?, string?, integer?) function string_utf8.gensub(str, sub_len) sub_len = sub_len or 1 ensureArgType(str, "string", 1, "string_utf8.gensub") ensureArgType(sub_len, "number", 2, "string_utf8.gensub") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.gensub\' (string cannot be empty)') local max_len = #str ---@param skip_ptr table ---@param bs integer ---@return integer?, string?, integer? return function(skip_ptr, bs) bs = (bs and bs or 1) + (skip_ptr and (skip_ptr[1] or 0) or 0) local nbs = bs if bs > max_len then return nil end for i = 1, sub_len do nbs = utf8next(str, nbs) end return nbs, sub(str, bs, nbs - 1), bs end end ---Reverses a UTF-8 encoded string while preserving multi-byte characters. ---@param str string The input UTF-8 encoded string. ---@return string function string_utf8.reverse(str) ensureArgType(str, "string", 1, "string_utf8.reverse") local result = '' for _, w in string_utf8.gensub(str) do result = w .. result end return result end ---Validates UTF-8 encoded characters in a string starting from a given byte position. ---@param str string The input UTF-8 encoded string to validate. ---@param bs integer The starting byte position for validation (default: 1). ---@overload fun(str: string) ---@return integer?, integer?, integer?, integer? local function utf8validator(str, bs) bs = bs or 1 ensureArgType(str, "string", 1, "utf8validator") ensureArgType(bs, "number", 2, "utf8validator") assert(#str ~= 0, 'bad argument #1 to \'utf8validator\' (string cannot be empty)') local c = byte(str, bs) if not c then return end -- determine bytes needed for character, based ons RFC 3629 -- UTF8-1 if c >= 0 and c <= 127 then return bs + 1 elseif c >= 128 and c <= 193 then return bs + 1, bs, 1, c -- UTF8-2 elseif c >= 194 and c <= 223 then local c2 = byte(str, bs + 1) if not c2 or c2 < 128 or c2 > 191 then return bs + 2, bs, 2, c2 end return bs + 2 -- UTF8-3 elseif c >= 224 and c <= 239 then ---@type integer? local c2 = byte(str, bs + 1) if not c2 then return bs + 2, bs, 2, c2 end -- validate byte 2 if c == 224 and (c2 < 160 or c2 > 191) then return bs + 2, bs, 2, c2 elseif c == 237 and (c2 < 128 or c2 > 159) then return bs + 2, bs, 2, c2 elseif c2 < 128 or c2 > 191 then return bs + 2, bs, 2, c2 end local c3 = byte(str, bs + 2) if not c3 or c3 < 128 or c3 > 191 then return bs + 3, bs, 3, c3 end return bs + 3 -- UTF8-4 elseif c >= 240 and c <= 244 then ---@type integer? local c2 = byte(str, bs + 1) if not c2 then return bs + 2, bs, 2, c2 end -- validate byte 2 if c == 240 and (c2 < 144 or c2 > 191) then return bs + 2, bs, 2, c2 elseif c == 244 and (c2 < 128 or c2 > 143) then return bs + 2, bs, 2, c2 elseif c2 < 128 or c2 > 191 then return bs + 2, bs, 2, c2 end local c3 = byte(str, bs + 2) if not c3 or c3 < 128 or c3 > 191 then return bs + 3, bs, 3, c3 end local c4 = byte(str, bs + 3) if not c4 or c4 < 128 or c4 > 191 then return bs + 4, bs, 4, c4 end return bs + 4 else -- c > 245 return bs + 1, bs, 1, c end end ---Validates a UTF-8 encoded string for correctness. ---@param str string The input UTF-8 encoded string to validate. ---@param byte_pos? integer The starting byte position for validation (default: 1). ---@return boolean, table function string_utf8.validate(str, byte_pos) byte_pos = byte_pos or 1 ensureArgType(str, "string", 1, "string_utf8.validate") ensureArgType(byte_pos, "number", 2, "string_utf8.validate") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.validate\' (string cannot be empty)') local result = {} for nbs, bs, part, code in utf8validator, str, byte_pos do if bs then result[#result + 1] = { pos = bs, part = part, code = code } end end return #result == 0, result end ---Creates an iterator to traverse UTF-8 encoded characters in a string. ---@param str string The input UTF-8 encoded string. ---@return fun(skip_ptr: table|nil):integer?, integer?, integer? function string_utf8.codes(str) ensureArgType(str, "string", 1, "string_utf8.codes") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.codes\' (string cannot be empty)') local max_len = #str local bs = 1 return function(skip_ptr) if bs > max_len then return nil end local pbs = bs bs = utf8next(str, pbs) return pbs, string_utf8.unicode(str, pbs, pbs), pbs end end --[[ differs from Lua 5.3 utf8.offset in accepting any byte positions (not only head byte) for all n values h - head, c - continuation, t - tail hhhccthccthccthcthhh ^ start byte pos searching current charracter head by moving backwards hhhccthccthccthcthhh ^ head n == 0: current position n > 0: n jumps forward n < 0: n more scans backwards --]] ---Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position. ---@param str string The input UTF-8 encoded string. ---@param n integer The character index to find. If `n` is negative, counts from the end. ---@param bs integer The starting byte position (default: 1 for positive `n`, end of string for negative `n`). ---@overload fun(str: string, n: number) ---@return number | nil function string_utf8.offset(str, n, bs) local l = #str if not bs then if n < 0 then bs = l + 1 else bs = 1 end end ensureArgType(str, "string", 1, "string_utf8.offset") ensureArgType(n, "number", 2, "string_utf8.offset") ensureArgType(bs, "number", 3, "string_utf8.offset") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.offset\' (string cannot be empty)') if bs <= 0 or bs > l + 1 then error("bad argument #3 to 'string_utf8.offset' (position out of range)") end if n == 0 then if bs == l + 1 then return bs end while true do local b = byte(str, bs) if (0 < b and b < 127) or (194 < b and b < 244) then return bs end bs = bs - 1 if bs < 1 then return end end elseif n < 0 then bs = bs - 1 repeat if bs < 1 then return end local b = byte(str, bs) if (0 < b and b < 127) or (194 < b and b < 244) then n = n + 1 end bs = bs - 1 until n == 0 return bs + 1 else while true do if bs > l then return end local b = byte(str, bs) if (0 < b and b < 127) or (194 < b and b < 244) then n = n - 1 for i = 1, n do if bs > l then return end bs = utf8next(str, bs) end return bs end bs = bs - 1 end end end --- Replaces UTF-8 characters in a string according to a mapping table. --- @param str string The input UTF-8 encoded string. --- @param mapping table A table where keys are UTF-8 characters (or patterns) and values are their replacements. --- @return string function string_utf8.replace(str, mapping) ensureArgType(str, "string", 1, "string_utf8.replace") ensureArgType(mapping, "table", 2, "string_utf8.replace") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.offset\' (string cannot be empty)') local result = string.gsub( str, utf8charpattern, mapping ) return result end --- Converts all UTF-8 characters in a string to uppercase. --- @param str string The input UTF-8 encoded string. --- @return string function string_utf8.upper(str) ensureArgType(str, "string", 1, "string_utf8.upper") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.upper\' (string cannot be empty)') return string_utf8.replace(str, string_utf8.lc_uc) end --- Converts all UTF-8 characters in a string to lowercase. --- @param str string The input UTF-8 encoded string. --- @return string function string_utf8.lower(str) ensureArgType(str, "string", 1, "string_utf8.lower") assert(#str ~= 0, 'bad argument #1 to \'string_utf8.lower\' (string cannot be empty)') return string_utf8.replace(str, string_utf8.uc_lc) end