Add utf8 API
This commit is contained in:
@@ -1,57 +1,22 @@
|
|||||||
local pairs, string_gsub
|
local pairs, string_gsub
|
||||||
= pairs, string.gsub
|
= pairs, string.gsub
|
||||||
|
|
||||||
-- Cyrillic symbols replacement dictionary.
|
dofile(core.get_modpath(core.get_current_modname()).."/utf8.lua")
|
||||||
local cyrillic_alphabet = {
|
|
||||||
["А"] = "а", ["Б"] = "б", ["В"] = "в", ["Г"] = "г", ["Д"] = "д",
|
|
||||||
["Е"] = "е", ["Ё"] = "ё", ["Ж"] = "ж", ["З"] = "з", ["И"] = "и",
|
|
||||||
["Й"] = "й", ["К"] = "к", ["Л"] = "л", ["М"] = "м", ["Н"] = "н",
|
|
||||||
["О"] = "о", ["П"] = "п", ["Р"] = "р", ["С"] = "с", ["Т"] = "т",
|
|
||||||
["У"] = "у", ["Ф"] = "ф", ["Х"] = "х", ["Ц"] = "ц", ["Ч"] = "ч",
|
|
||||||
["Ш"] = "ш", ["Щ"] = "щ", ["Ъ"] = "ъ", ["Ь"] = "ь", ["Э"] = "э",
|
|
||||||
["Ю"] = "ю", ["Я"] = "я",
|
|
||||||
}
|
|
||||||
|
|
||||||
string.replace = string.gsub
|
string.replace = string.gsub
|
||||||
|
|
||||||
--- Same as `string.lower` but with cyrillic support.
|
--- Writes a string with a capital letter.
|
||||||
--- @param str string Input string.
|
|
||||||
--- @return string
|
|
||||||
function string.lower_cyrillic(str)
|
|
||||||
-- assert
|
|
||||||
|
|
||||||
local new_str = string.lower(str)
|
|
||||||
for S, s in pairs(cyrillic_alphabet) do
|
|
||||||
new_str = string_gsub(new_str, S, s)
|
|
||||||
end
|
|
||||||
|
|
||||||
return new_str
|
|
||||||
end
|
|
||||||
|
|
||||||
--- Same as `string.upper` but with cyrillic support.
|
|
||||||
--- @param str string Input string.
|
|
||||||
--- @return string
|
|
||||||
function string.upper_cyrillic(str)
|
|
||||||
-- assert
|
|
||||||
|
|
||||||
local new_str = string.upper(str)
|
|
||||||
for S, s in pairs(cyrillic_alphabet) do
|
|
||||||
new_str = string_gsub(new_str, s, S)
|
|
||||||
end
|
|
||||||
|
|
||||||
return new_str
|
|
||||||
end
|
|
||||||
|
|
||||||
--- Writes a string with a capital letter. Doesn't work with cyrillic.
|
|
||||||
--- @param str string Input string.
|
--- @param str string Input string.
|
||||||
--- @return string
|
--- @return string
|
||||||
function string.capitalize(str)
|
function string.capitalize(str)
|
||||||
-- assert
|
-- assert
|
||||||
|
local first_char = string_utf8.sub(str,1, 1)
|
||||||
|
first_char = string_utf8.upper(first_char)
|
||||||
|
|
||||||
return str:sub(1, 1):upper() .. str:sub(2, -1)
|
return first_char..string_utf8.sub(str,2, -1)
|
||||||
end
|
end
|
||||||
|
|
||||||
--- Capitalizes each word in a line. Doesn't work with cyrillic.
|
--- Capitalizes each word in a line.
|
||||||
--- @param str string Input string.
|
--- @param str string Input string.
|
||||||
--- @return string
|
--- @return string
|
||||||
function string.to_title(str)
|
function string.to_title(str)
|
||||||
|
|||||||
590
mods/API/String/utf8.lua
Normal file
590
mods/API/String/utf8.lua
Normal file
@@ -0,0 +1,590 @@
|
|||||||
|
-- $Id: utf8.lua 179 2009-04-03 18:10:03Z pasta $
|
||||||
|
--
|
||||||
|
-- Provides UTF-8 aware string functions implemented in pure lua:
|
||||||
|
-- * utf8len(s)
|
||||||
|
-- * utf8sub(s, i, j)
|
||||||
|
-- * utf8reverse(s)
|
||||||
|
-- * utf8char(unicode)
|
||||||
|
-- * utf8unicode(s, i, j)
|
||||||
|
-- * utf8gensub(s, sub_len)
|
||||||
|
-- * utf8find(str, regex, init, plain)
|
||||||
|
-- * utf8match(str, regex, init)
|
||||||
|
-- * utf8gmatch(str, regex, all)
|
||||||
|
-- * utf8gsub(str, regex, repl, limit)
|
||||||
|
--
|
||||||
|
-- All functions behave as their non UTF-8 aware counterparts with the exception
|
||||||
|
-- that UTF-8 characters are used instead of bytes for all units.
|
||||||
|
|
||||||
|
--[[
|
||||||
|
Copyright (c) 2006-2007, Kyle Smith
|
||||||
|
All rights reserved.
|
||||||
|
|
||||||
|
Contributors:
|
||||||
|
Alimov Stepan
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer in the
|
||||||
|
documentation and/or other materials provided with the distribution.
|
||||||
|
* Neither the name of the author nor the names of its contributors may be
|
||||||
|
used to endorse or promote products derived from this software without
|
||||||
|
specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
|
||||||
|
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||||
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||||
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
--]]
|
||||||
|
|
||||||
|
-- ABNF from RFC 3629
|
||||||
|
--
|
||||||
|
-- UTF8-octets = *( UTF8-char )
|
||||||
|
-- UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
|
||||||
|
-- UTF8-1 = %x00-7F
|
||||||
|
-- UTF8-2 = %xC2-DF UTF8-tail
|
||||||
|
-- UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
|
||||||
|
-- %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
|
||||||
|
-- UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
|
||||||
|
-- %xF4 %x80-8F 2( UTF8-tail )
|
||||||
|
-- UTF8-tail = %x80-BF
|
||||||
|
--
|
||||||
|
|
||||||
|
-- Original repo: https://github.com/Stepets/utf8.lua/tree/master
|
||||||
|
-- Commit: 17f4e009a22fb2f2e6ad316a05b2cca8e071fc3b
|
||||||
|
-- Date of copying: 6.03.2025
|
||||||
|
-- Code edited: VanicGame (https://github.com/VanicGame)
|
||||||
|
|
||||||
|
string_utf8 = {}
|
||||||
|
|
||||||
|
dofile(core.get_modpath(core.get_current_modname()).."/utf8data.lua")
|
||||||
|
|
||||||
|
local byte = string.byte
|
||||||
|
local char = string.char
|
||||||
|
local len = string.len
|
||||||
|
local sub = string.sub
|
||||||
|
|
||||||
|
local utf8charpattern = '[%z\1-\127\194-\244][\128-\191]*'
|
||||||
|
|
||||||
|
--- Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character.
|
||||||
|
--- @param byte integer First byte of a character.
|
||||||
|
--- @return number
|
||||||
|
local function utf8symbollen(byte)
|
||||||
|
-- assert
|
||||||
|
|
||||||
|
return not byte and 0 or (byte < 0x80 and 1) or (byte >= 0xF0 and 4) or (byte >= 0xE0 and 3) or (byte >= 0xC0 and 2) or 1
|
||||||
|
end
|
||||||
|
|
||||||
|
local head_table = {}
|
||||||
|
|
||||||
|
for i = 0, 255 do
|
||||||
|
head_table[i] = utf8symbollen(i)
|
||||||
|
end
|
||||||
|
head_table[256] = 0
|
||||||
|
|
||||||
|
--- Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`.
|
||||||
|
--- @param str string Input string.
|
||||||
|
--- @param bs number Position of character in line.
|
||||||
|
--- @return number
|
||||||
|
local function utf8charbytes(str, bs)
|
||||||
|
-- assert
|
||||||
|
|
||||||
|
return head_table[byte(str, bs) or 256]
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Returns the new position in the string `str`, skipping the current character.
|
||||||
|
--- @param str string Input string.
|
||||||
|
--- @param bs number Position of character in line.
|
||||||
|
--- @return number
|
||||||
|
local function utf8next(str, bs)
|
||||||
|
-- assert
|
||||||
|
|
||||||
|
return bs + utf8charbytes(str, bs)
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Returns the number of characters in a UTF-8 string
|
||||||
|
--- @param str string Input string.
|
||||||
|
--- @return number
|
||||||
|
function string_utf8.len(str)
|
||||||
|
-- assert
|
||||||
|
|
||||||
|
local bs = 1
|
||||||
|
local bytes = len(str)
|
||||||
|
local length = 0
|
||||||
|
|
||||||
|
while bs <= bytes do
|
||||||
|
length = length + 1
|
||||||
|
bs = utf8next(str, bs)
|
||||||
|
end
|
||||||
|
|
||||||
|
return length
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
--- Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes.
|
||||||
|
--- @param s string Input string.
|
||||||
|
--- @param i number The starting character index (default: 1). Negative values count from the end.
|
||||||
|
--- @param j? number The ending character index (default: i). Negative values count from the end.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.sub(s, i, j)
|
||||||
|
-- assert
|
||||||
|
|
||||||
|
-- argument defaults
|
||||||
|
j = j or -1
|
||||||
|
|
||||||
|
local bs = 1
|
||||||
|
local bytes = len(s)
|
||||||
|
local length = 0
|
||||||
|
|
||||||
|
local l = (i >= 0 and j >= 0) or string_utf8.len(s)
|
||||||
|
i = (i >= 0) and i or l + i + 1
|
||||||
|
j = (j >= 0) and j or l + j + 1
|
||||||
|
|
||||||
|
if i > j then
|
||||||
|
return ""
|
||||||
|
end
|
||||||
|
|
||||||
|
local start, finish = 1, bytes
|
||||||
|
|
||||||
|
while bs <= bytes do
|
||||||
|
length = length + 1
|
||||||
|
|
||||||
|
if length == i then
|
||||||
|
start = bs
|
||||||
|
end
|
||||||
|
|
||||||
|
bs = utf8next(s, bs)
|
||||||
|
|
||||||
|
if length == j then
|
||||||
|
finish = bs - 1
|
||||||
|
break
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if i > length then start = bytes + 1 end
|
||||||
|
if j < 1 then finish = 0 end
|
||||||
|
|
||||||
|
return sub(s, start, finish)
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Returns a UTF-8 string from the given Unicode codes.
|
||||||
|
--- @param ... integer List of Unicode codes.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.char(...)
|
||||||
|
local codes = {...}
|
||||||
|
local result = {}
|
||||||
|
|
||||||
|
for _, unicode in ipairs(codes) do
|
||||||
|
|
||||||
|
if unicode <= 0x7F then
|
||||||
|
result[#result + 1] = unicode
|
||||||
|
elseif unicode <= 0x7FF then
|
||||||
|
local b0 = 0xC0 + math.floor(unicode / 0x40);
|
||||||
|
local b1 = 0x80 + (unicode % 0x40);
|
||||||
|
result[#result + 1] = b0
|
||||||
|
result[#result + 1] = b1
|
||||||
|
elseif unicode <= 0xFFFF then
|
||||||
|
local b0 = 0xE0 + math.floor(unicode / 0x1000);
|
||||||
|
local b1 = 0x80 + (math.floor(unicode / 0x40) % 0x40);
|
||||||
|
local b2 = 0x80 + (unicode % 0x40);
|
||||||
|
result[#result + 1] = b0
|
||||||
|
result[#result + 1] = b1
|
||||||
|
result[#result + 1] = b2
|
||||||
|
elseif unicode <= 0x10FFFF then
|
||||||
|
local code = unicode
|
||||||
|
local b3 = 0x80 + (code % 0x40);
|
||||||
|
code = math.floor(code / 0x40)
|
||||||
|
local b2 = 0x80 + (code % 0x40);
|
||||||
|
code = math.floor(code / 0x40)
|
||||||
|
local b1 = 0x80 + (code % 0x40);
|
||||||
|
code = math.floor(code / 0x40)
|
||||||
|
local b0 = 0xF0 + code;
|
||||||
|
|
||||||
|
result[#result + 1] = b0
|
||||||
|
result[#result + 1] = b1
|
||||||
|
result[#result + 1] = b2
|
||||||
|
result[#result + 1] = b3
|
||||||
|
else
|
||||||
|
error 'Unicode cannot be greater than U+10FFFF!'
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
return char(unpack(result, 1, #result))
|
||||||
|
end
|
||||||
|
|
||||||
|
local shift_6 = 2^6
|
||||||
|
local shift_12 = 2^12
|
||||||
|
local shift_18 = 2^18
|
||||||
|
|
||||||
|
--- Converts UTF-8 encoded characters in a string to their corresponding Unicode code points.
|
||||||
|
--- @param str string The input string containing UTF-8 encoded characters.
|
||||||
|
--- @param ibs number The starting byte position in the string (inclusive).
|
||||||
|
--- @param jbs number The ending byte position in the string (inclusive).
|
||||||
|
--- @return number | number...
|
||||||
|
function string_utf8.unicode(str, ibs, jbs)
|
||||||
|
if ibs > jbs then return end
|
||||||
|
|
||||||
|
local bytes
|
||||||
|
|
||||||
|
bytes = utf8charbytes(str, ibs)
|
||||||
|
if bytes == 0 then return end
|
||||||
|
|
||||||
|
local unicode
|
||||||
|
|
||||||
|
if bytes == 1 then unicode = byte(str, ibs, ibs) end
|
||||||
|
if bytes == 2 then
|
||||||
|
local byte0,byte1 = byte(str, ibs, ibs + 1)
|
||||||
|
if byte0 and byte1 then
|
||||||
|
local code0,code1 = byte0-0xC0,byte1-0x80
|
||||||
|
unicode = code0*shift_6 + code1
|
||||||
|
else
|
||||||
|
unicode = byte0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if bytes == 3 then
|
||||||
|
local byte0,byte1,byte2 = byte(str, ibs, ibs + 2)
|
||||||
|
if byte0 and byte1 and byte2 then
|
||||||
|
local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80
|
||||||
|
unicode = code0*shift_12 + code1*shift_6 + code2
|
||||||
|
else
|
||||||
|
unicode = byte0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if bytes == 4 then
|
||||||
|
local byte0,byte1,byte2,byte3 = byte(str, ibs, ibs + 3)
|
||||||
|
if byte0 and byte1 and byte2 and byte3 then
|
||||||
|
local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80
|
||||||
|
unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3
|
||||||
|
else
|
||||||
|
unicode = byte0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if ibs == jbs then
|
||||||
|
return unicode
|
||||||
|
else
|
||||||
|
return unicode, string_utf8.unicode(str, ibs+bytes, jbs)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Returns Unicode code points for characters in a UTF-8 string within the specified range.
|
||||||
|
--- @param str string The input UTF-8 encoded string.
|
||||||
|
--- @param i? number The starting character index (default: 1). Negative values count from the end.
|
||||||
|
--- @param j? number The ending character index (default: i). Negative values count from the end.
|
||||||
|
--- @return number | number...
|
||||||
|
function string_utf8.byte(str, i, j)
|
||||||
|
if #str == 0 then return end
|
||||||
|
|
||||||
|
local ibs, jbs
|
||||||
|
|
||||||
|
if i or j then
|
||||||
|
i = i or 1
|
||||||
|
j = j or i
|
||||||
|
|
||||||
|
local str_len = string_utf8.len(str)
|
||||||
|
i = i < 0 and str_len + i + 1 or i
|
||||||
|
j = j < 0 and str_len + j + 1 or j
|
||||||
|
j = j > str_len and str_len or j
|
||||||
|
|
||||||
|
if i > j then return end
|
||||||
|
|
||||||
|
for p = 1, i - 1 do
|
||||||
|
ibs = utf8next(str, ibs or 1)
|
||||||
|
end
|
||||||
|
|
||||||
|
if i == j then
|
||||||
|
jbs = ibs
|
||||||
|
else
|
||||||
|
for p = 1, j - 1 do
|
||||||
|
jbs = utf8next(str, jbs or 1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if not ibs or not jbs then
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
else
|
||||||
|
ibs, jbs = 1, 1
|
||||||
|
end
|
||||||
|
|
||||||
|
return string_utf8.unicode(str, ibs, jbs)
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Creates an iterator to generate UTF-8 substrings of a specified length.
|
||||||
|
--- @param str string The input UTF-8 encoded string.
|
||||||
|
--- @param sub_len? number The length of each substring in characters (default: 1).
|
||||||
|
--- @return fun(skip_ptr: table | nil, bs: number | nil)
|
||||||
|
function string_utf8.gensub(str, sub_len)
|
||||||
|
sub_len = sub_len or 1
|
||||||
|
|
||||||
|
local max_len = #str
|
||||||
|
|
||||||
|
return function(skip_ptr, bs)
|
||||||
|
bs = (bs and bs or 1) + (skip_ptr and (skip_ptr[1] or 0) or 0)
|
||||||
|
|
||||||
|
local nbs = bs
|
||||||
|
if bs > max_len then return nil end
|
||||||
|
for i = 1, sub_len do
|
||||||
|
nbs = utf8next(str, nbs)
|
||||||
|
end
|
||||||
|
|
||||||
|
return nbs, sub(str, bs, nbs - 1), bs
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Reverses a UTF-8 encoded string while preserving multi-byte characters.
|
||||||
|
--- @param s string The input UTF-8 encoded string.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.reverse(s)
|
||||||
|
local result = ''
|
||||||
|
|
||||||
|
for _, w in string_utf8.gensub(s) do
|
||||||
|
result = w .. result
|
||||||
|
end
|
||||||
|
|
||||||
|
return result
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Validates UTF-8 encoded characters in a string starting from a given byte position.
|
||||||
|
--- @param str string The input UTF-8 encoded string to validate.
|
||||||
|
--- @param bs? number The starting byte position for validation (default: 1).
|
||||||
|
--- @return number | nil
|
||||||
|
local function utf8validator(str, bs)
|
||||||
|
bs = bs or 1
|
||||||
|
|
||||||
|
if type(str) ~= "string" then
|
||||||
|
error("bad argument #1 to 'utf8charbytes' (string expected, got ".. type(str).. ")")
|
||||||
|
end
|
||||||
|
if type(bs) ~= "number" then
|
||||||
|
error("bad argument #2 to 'utf8charbytes' (number expected, got ".. type(bs).. ")")
|
||||||
|
end
|
||||||
|
|
||||||
|
local c = byte(str, bs)
|
||||||
|
if not c then return end
|
||||||
|
|
||||||
|
-- determine bytes needed for character, based on RFC 3629
|
||||||
|
|
||||||
|
-- UTF8-1
|
||||||
|
if c >= 0 and c <= 127 then
|
||||||
|
return bs + 1
|
||||||
|
elseif c >= 128 and c <= 193 then
|
||||||
|
return bs + 1, bs, 1, c
|
||||||
|
-- UTF8-2
|
||||||
|
elseif c >= 194 and c <= 223 then
|
||||||
|
local c2 = byte(str, bs + 1)
|
||||||
|
if not c2 or c2 < 128 or c2 > 191 then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
end
|
||||||
|
|
||||||
|
return bs + 2
|
||||||
|
-- UTF8-3
|
||||||
|
elseif c >= 224 and c <= 239 then
|
||||||
|
local c2 = byte(str, bs + 1)
|
||||||
|
|
||||||
|
if not c2 then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
end
|
||||||
|
|
||||||
|
-- validate byte 2
|
||||||
|
if c == 224 and (c2 < 160 or c2 > 191) then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
elseif c == 237 and (c2 < 128 or c2 > 159) then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
elseif c2 < 128 or c2 > 191 then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
end
|
||||||
|
|
||||||
|
local c3 = byte(str, bs + 2)
|
||||||
|
if not c3 or c3 < 128 or c3 > 191 then
|
||||||
|
return bs + 3, bs, 3, c3
|
||||||
|
end
|
||||||
|
|
||||||
|
return bs + 3
|
||||||
|
-- UTF8-4
|
||||||
|
elseif c >= 240 and c <= 244 then
|
||||||
|
local c2 = byte(str, bs + 1)
|
||||||
|
|
||||||
|
if not c2 then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
end
|
||||||
|
|
||||||
|
-- validate byte 2
|
||||||
|
if c == 240 and (c2 < 144 or c2 > 191) then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
elseif c == 244 and (c2 < 128 or c2 > 143) then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
elseif c2 < 128 or c2 > 191 then
|
||||||
|
return bs + 2, bs, 2, c2
|
||||||
|
end
|
||||||
|
|
||||||
|
local c3 = byte(str, bs + 2)
|
||||||
|
if not c3 or c3 < 128 or c3 > 191 then
|
||||||
|
return bs + 3, bs, 3, c3
|
||||||
|
end
|
||||||
|
|
||||||
|
local c4 = byte(str, bs + 3)
|
||||||
|
if not c4 or c4 < 128 or c4 > 191 then
|
||||||
|
return bs + 4, bs, 4, c4
|
||||||
|
end
|
||||||
|
|
||||||
|
return bs + 4
|
||||||
|
else -- c > 245
|
||||||
|
return bs + 1, bs, 1, c
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Validates a UTF-8 encoded string for correctness.
|
||||||
|
--- @param str string The input UTF-8 encoded string to validate.
|
||||||
|
--- @param byte_pos? number The starting byte position for validation (default: 1).
|
||||||
|
--- @return boolean | table
|
||||||
|
function string_utf8.validate(str, byte_pos)
|
||||||
|
local result = {}
|
||||||
|
for nbs, bs, part, code in utf8validator, str, byte_pos do
|
||||||
|
if bs then
|
||||||
|
result[#result + 1] = { pos = bs, part = part, code = code }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return #result == 0, result
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Creates an iterator to traverse UTF-8 encoded characters in a string.
|
||||||
|
--- @param str string The input UTF-8 encoded string.
|
||||||
|
--- @return fun(skip_ptr: table|nil)
|
||||||
|
function string_utf8.codes(str)
|
||||||
|
local max_len = #str
|
||||||
|
local bs = 1
|
||||||
|
return function(skip_ptr)
|
||||||
|
if bs > max_len then return nil end
|
||||||
|
local pbs = bs
|
||||||
|
bs = utf8next(str, pbs)
|
||||||
|
|
||||||
|
return pbs, string_utf8.unicode(str, pbs, pbs), pbs
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
--[[
|
||||||
|
differs from Lua 5.3 utf8.offset in accepting any byte positions (not only head byte) for all n values
|
||||||
|
|
||||||
|
h - head, c - continuation, t - tail
|
||||||
|
hhhccthccthccthcthhh
|
||||||
|
^ start byte pos
|
||||||
|
searching current charracter head by moving backwards
|
||||||
|
hhhccthccthccthcthhh
|
||||||
|
^ head
|
||||||
|
|
||||||
|
n == 0: current position
|
||||||
|
n > 0: n jumps forward
|
||||||
|
n < 0: n more scans backwards
|
||||||
|
--]]
|
||||||
|
|
||||||
|
--- Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position.
|
||||||
|
--- @param str string The input UTF-8 encoded string.
|
||||||
|
--- @param n number The character index to find. If `n` is negative, counts from the end.
|
||||||
|
--- @param bs? number The starting byte position (default: 1 for positive `n`, end of string for negative `n`).
|
||||||
|
--- @return number | nil
|
||||||
|
function string_utf8.offset(str, n, bs)
|
||||||
|
local l = #str
|
||||||
|
if not bs then
|
||||||
|
if n < 0 then
|
||||||
|
bs = l + 1
|
||||||
|
else
|
||||||
|
bs = 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if bs <= 0 or bs > l + 1 then
|
||||||
|
error("bad argument #3 to 'offset' (position out of range)")
|
||||||
|
end
|
||||||
|
|
||||||
|
if n == 0 then
|
||||||
|
if bs == l + 1 then
|
||||||
|
return bs
|
||||||
|
end
|
||||||
|
while true do
|
||||||
|
local b = byte(str, bs)
|
||||||
|
if (0 < b and b < 127)
|
||||||
|
or (194 < b and b < 244) then
|
||||||
|
return bs
|
||||||
|
end
|
||||||
|
bs = bs - 1
|
||||||
|
if bs < 1 then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
end
|
||||||
|
elseif n < 0 then
|
||||||
|
bs = bs - 1
|
||||||
|
repeat
|
||||||
|
if bs < 1 then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
local b = byte(str, bs)
|
||||||
|
if (0 < b and b < 127)
|
||||||
|
or (194 < b and b < 244) then
|
||||||
|
n = n + 1
|
||||||
|
end
|
||||||
|
bs = bs - 1
|
||||||
|
until n == 0
|
||||||
|
return bs + 1
|
||||||
|
else
|
||||||
|
while true do
|
||||||
|
if bs > l then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
|
||||||
|
local b = byte(str, bs)
|
||||||
|
if (0 < b and b < 127)
|
||||||
|
or (194 < b and b < 244) then
|
||||||
|
n = n - 1
|
||||||
|
for i = 1, n do
|
||||||
|
if bs > l then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
bs = utf8next(str, bs)
|
||||||
|
end
|
||||||
|
return bs
|
||||||
|
end
|
||||||
|
bs = bs - 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Replaces UTF-8 characters in a string according to a mapping table.
|
||||||
|
--- @param s string The input UTF-8 encoded string.
|
||||||
|
--- @param mapping table A table where keys are UTF-8 characters (or patterns) and values are their replacements.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.replace(s, mapping)
|
||||||
|
if type(s) ~= "string" then
|
||||||
|
error("bad argument #1 to 'utf8replace' (string expected, got ".. type(s).. ")")
|
||||||
|
end
|
||||||
|
if type(mapping) ~= "table" then
|
||||||
|
error("bad argument #2 to 'utf8replace' (table expected, got ".. type(mapping).. ")")
|
||||||
|
end
|
||||||
|
local result = string.gsub( s, utf8charpattern, mapping )
|
||||||
|
return result
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Converts all UTF-8 characters in a string to uppercase.
|
||||||
|
--- @param s string The input UTF-8 encoded string.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.upper (s)
|
||||||
|
return string_utf8.replace(s, string_utf8.lc_uc)
|
||||||
|
end
|
||||||
|
|
||||||
|
--- Converts all UTF-8 characters in a string to lowercase.
|
||||||
|
--- @param s string The input UTF-8 encoded string.
|
||||||
|
--- @return string
|
||||||
|
function string_utf8.lower (s)
|
||||||
|
return string_utf8.replace(s, string_utf8.uc_lc)
|
||||||
|
end
|
||||||
1859
mods/API/String/utf8data.lua
Normal file
1859
mods/API/String/utf8data.lua
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user