Delete utf8 api

This commit is contained in:
Van
2025-03-07 14:09:41 +00:00
parent a0bc53af4f
commit 2f78206f3f
2 changed files with 0 additions and 2449 deletions

View File

@@ -1,590 +0,0 @@
-- $Id: utf8.lua 179 2009-04-03 18:10:03Z pasta $
--
-- Provides UTF-8 aware string functions implemented in pure lua:
-- * utf8len(s)
-- * utf8sub(s, i, j)
-- * utf8reverse(s)
-- * utf8char(unicode)
-- * utf8unicode(s, i, j)
-- * utf8gensub(s, sub_len)
-- * utf8find(str, regex, init, plain)
-- * utf8match(str, regex, init)
-- * utf8gmatch(str, regex, all)
-- * utf8gsub(str, regex, repl, limit)
--
-- All functions behave as their non UTF-8 aware counterparts with the exception
-- that UTF-8 characters are used instead of bytes for all units.
--[[
Copyright (c) 2006-2007, Kyle Smith
All rights reserved.
Contributors:
Alimov Stepan
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the author nor the names of its contributors may be
used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--]]
-- ABNF from RFC 3629
--
-- UTF8-octets = *( UTF8-char )
-- UTF8-char = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
-- UTF8-1 = %x00-7F
-- UTF8-2 = %xC2-DF UTF8-tail
-- UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
-- %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
-- UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
-- %xF4 %x80-8F 2( UTF8-tail )
-- UTF8-tail = %x80-BF
--
-- Original repo: https://github.com/Stepets/utf8.lua/tree/master
-- Commit: 17f4e009a22fb2f2e6ad316a05b2cca8e071fc3b
-- Date of copying: 6.03.2025
-- Code edited: VanicGame (https://github.com/VanicGame)
string_utf8 = {}
dofile(core.get_modpath(core.get_current_modname()).."/utf8data.lua")
local byte = string.byte
local char = string.char
local len = string.len
local sub = string.sub
local utf8charpattern = '[%z\1-\127\194-\244][\128-\191]*'
--- Determines the length of a character in bytes encoded in UTF-8, based on the first byte of the character.
--- @param byte integer First byte of a character.
--- @return number
local function utf8symbollen(byte)
-- assert
return not byte and 0 or (byte < 0x80 and 1) or (byte >= 0xF0 and 4) or (byte >= 0xE0 and 3) or (byte >= 0xC0 and 2) or 1
end
local head_table = {}
for i = 0, 255 do
head_table[i] = utf8symbollen(i)
end
head_table[256] = 0
--- Returns the length of the character in bytes starting at position `pos` in the string `str` using the table `head_table`.
--- @param str string Input string.
--- @param bs number Position of character in line.
--- @return number
local function utf8charbytes(str, bs)
-- assert
return head_table[byte(str, bs) or 256]
end
--- Returns the new position in the string `str`, skipping the current character.
--- @param str string Input string.
--- @param bs number Position of character in line.
--- @return number
local function utf8next(str, bs)
-- assert
return bs + utf8charbytes(str, bs)
end
--- Returns the number of characters in a UTF-8 string
--- @param str string Input string.
--- @return number
function string_utf8.len(str)
-- assert
local bs = 1
local bytes = len(str)
local length = 0
while bs <= bytes do
length = length + 1
bs = utf8next(str, bs)
end
return length
end
--- Functions identically to `string.sub` except that `i` and `j` are UTF-8 characters instead of bytes.
--- @param s string Input string.
--- @param i number The starting character index (default: 1). Negative values count from the end.
--- @param j? number The ending character index (default: i). Negative values count from the end.
--- @return string
function string_utf8.sub(s, i, j)
-- assert
-- argument defaults
j = j or -1
local bs = 1
local bytes = len(s)
local length = 0
local l = (i >= 0 and j >= 0) or string_utf8.len(s)
i = (i >= 0) and i or l + i + 1
j = (j >= 0) and j or l + j + 1
if i > j then
return ""
end
local start, finish = 1, bytes
while bs <= bytes do
length = length + 1
if length == i then
start = bs
end
bs = utf8next(s, bs)
if length == j then
finish = bs - 1
break
end
end
if i > length then start = bytes + 1 end
if j < 1 then finish = 0 end
return sub(s, start, finish)
end
--- Returns a UTF-8 string from the given Unicode codes.
--- @param ... integer List of Unicode codes.
--- @return string
function string_utf8.char(...)
local codes = {...}
local result = {}
for _, unicode in ipairs(codes) do
if unicode <= 0x7F then
result[#result + 1] = unicode
elseif unicode <= 0x7FF then
local b0 = 0xC0 + math.floor(unicode / 0x40);
local b1 = 0x80 + (unicode % 0x40);
result[#result + 1] = b0
result[#result + 1] = b1
elseif unicode <= 0xFFFF then
local b0 = 0xE0 + math.floor(unicode / 0x1000);
local b1 = 0x80 + (math.floor(unicode / 0x40) % 0x40);
local b2 = 0x80 + (unicode % 0x40);
result[#result + 1] = b0
result[#result + 1] = b1
result[#result + 1] = b2
elseif unicode <= 0x10FFFF then
local code = unicode
local b3 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b2 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b1 = 0x80 + (code % 0x40);
code = math.floor(code / 0x40)
local b0 = 0xF0 + code;
result[#result + 1] = b0
result[#result + 1] = b1
result[#result + 1] = b2
result[#result + 1] = b3
else
error 'Unicode cannot be greater than U+10FFFF!'
end
end
return char(unpack(result, 1, #result))
end
local shift_6 = 2^6
local shift_12 = 2^12
local shift_18 = 2^18
--- Converts UTF-8 encoded characters in a string to their corresponding Unicode code points.
--- @param str string The input string containing UTF-8 encoded characters.
--- @param ibs number The starting byte position in the string (inclusive).
--- @param jbs number The ending byte position in the string (inclusive).
--- @return number | number...
function string_utf8.unicode(str, ibs, jbs)
if ibs > jbs then return end
local bytes
bytes = utf8charbytes(str, ibs)
if bytes == 0 then return end
local unicode
if bytes == 1 then unicode = byte(str, ibs, ibs) end
if bytes == 2 then
local byte0,byte1 = byte(str, ibs, ibs + 1)
if byte0 and byte1 then
local code0,code1 = byte0-0xC0,byte1-0x80
unicode = code0*shift_6 + code1
else
unicode = byte0
end
end
if bytes == 3 then
local byte0,byte1,byte2 = byte(str, ibs, ibs + 2)
if byte0 and byte1 and byte2 then
local code0,code1,code2 = byte0-0xE0,byte1-0x80,byte2-0x80
unicode = code0*shift_12 + code1*shift_6 + code2
else
unicode = byte0
end
end
if bytes == 4 then
local byte0,byte1,byte2,byte3 = byte(str, ibs, ibs + 3)
if byte0 and byte1 and byte2 and byte3 then
local code0,code1,code2,code3 = byte0-0xF0,byte1-0x80,byte2-0x80,byte3-0x80
unicode = code0*shift_18 + code1*shift_12 + code2*shift_6 + code3
else
unicode = byte0
end
end
if ibs == jbs then
return unicode
else
return unicode, string_utf8.unicode(str, ibs+bytes, jbs)
end
end
--- Returns Unicode code points for characters in a UTF-8 string within the specified range.
--- @param str string The input UTF-8 encoded string.
--- @param i? number The starting character index (default: 1). Negative values count from the end.
--- @param j? number The ending character index (default: i). Negative values count from the end.
--- @return number | number...
function string_utf8.byte(str, i, j)
if #str == 0 then return end
local ibs, jbs
if i or j then
i = i or 1
j = j or i
local str_len = string_utf8.len(str)
i = i < 0 and str_len + i + 1 or i
j = j < 0 and str_len + j + 1 or j
j = j > str_len and str_len or j
if i > j then return end
for p = 1, i - 1 do
ibs = utf8next(str, ibs or 1)
end
if i == j then
jbs = ibs
else
for p = 1, j - 1 do
jbs = utf8next(str, jbs or 1)
end
end
if not ibs or not jbs then
return nil
end
else
ibs, jbs = 1, 1
end
return string_utf8.unicode(str, ibs, jbs)
end
--- Creates an iterator to generate UTF-8 substrings of a specified length.
--- @param str string The input UTF-8 encoded string.
--- @param sub_len? number The length of each substring in characters (default: 1).
--- @return fun(skip_ptr: table | nil, bs: number | nil)
function string_utf8.gensub(str, sub_len)
sub_len = sub_len or 1
local max_len = #str
return function(skip_ptr, bs)
bs = (bs and bs or 1) + (skip_ptr and (skip_ptr[1] or 0) or 0)
local nbs = bs
if bs > max_len then return nil end
for i = 1, sub_len do
nbs = utf8next(str, nbs)
end
return nbs, sub(str, bs, nbs - 1), bs
end
end
--- Reverses a UTF-8 encoded string while preserving multi-byte characters.
--- @param s string The input UTF-8 encoded string.
--- @return string
function string_utf8.reverse(s)
local result = ''
for _, w in string_utf8.gensub(s) do
result = w .. result
end
return result
end
--- Validates UTF-8 encoded characters in a string starting from a given byte position.
--- @param str string The input UTF-8 encoded string to validate.
--- @param bs? number The starting byte position for validation (default: 1).
--- @return number | nil
local function utf8validator(str, bs)
bs = bs or 1
if type(str) ~= "string" then
error("bad argument #1 to 'utf8charbytes' (string expected, got ".. type(str).. ")")
end
if type(bs) ~= "number" then
error("bad argument #2 to 'utf8charbytes' (number expected, got ".. type(bs).. ")")
end
local c = byte(str, bs)
if not c then return end
-- determine bytes needed for character, based on RFC 3629
-- UTF8-1
if c >= 0 and c <= 127 then
return bs + 1
elseif c >= 128 and c <= 193 then
return bs + 1, bs, 1, c
-- UTF8-2
elseif c >= 194 and c <= 223 then
local c2 = byte(str, bs + 1)
if not c2 or c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
return bs + 2
-- UTF8-3
elseif c >= 224 and c <= 239 then
local c2 = byte(str, bs + 1)
if not c2 then
return bs + 2, bs, 2, c2
end
-- validate byte 2
if c == 224 and (c2 < 160 or c2 > 191) then
return bs + 2, bs, 2, c2
elseif c == 237 and (c2 < 128 or c2 > 159) then
return bs + 2, bs, 2, c2
elseif c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
local c3 = byte(str, bs + 2)
if not c3 or c3 < 128 or c3 > 191 then
return bs + 3, bs, 3, c3
end
return bs + 3
-- UTF8-4
elseif c >= 240 and c <= 244 then
local c2 = byte(str, bs + 1)
if not c2 then
return bs + 2, bs, 2, c2
end
-- validate byte 2
if c == 240 and (c2 < 144 or c2 > 191) then
return bs + 2, bs, 2, c2
elseif c == 244 and (c2 < 128 or c2 > 143) then
return bs + 2, bs, 2, c2
elseif c2 < 128 or c2 > 191 then
return bs + 2, bs, 2, c2
end
local c3 = byte(str, bs + 2)
if not c3 or c3 < 128 or c3 > 191 then
return bs + 3, bs, 3, c3
end
local c4 = byte(str, bs + 3)
if not c4 or c4 < 128 or c4 > 191 then
return bs + 4, bs, 4, c4
end
return bs + 4
else -- c > 245
return bs + 1, bs, 1, c
end
end
--- Validates a UTF-8 encoded string for correctness.
--- @param str string The input UTF-8 encoded string to validate.
--- @param byte_pos? number The starting byte position for validation (default: 1).
--- @return boolean | table
function string_utf8.validate(str, byte_pos)
local result = {}
for nbs, bs, part, code in utf8validator, str, byte_pos do
if bs then
result[#result + 1] = { pos = bs, part = part, code = code }
end
end
return #result == 0, result
end
--- Creates an iterator to traverse UTF-8 encoded characters in a string.
--- @param str string The input UTF-8 encoded string.
--- @return fun(skip_ptr: table|nil)
function string_utf8.codes(str)
local max_len = #str
local bs = 1
return function(skip_ptr)
if bs > max_len then return nil end
local pbs = bs
bs = utf8next(str, pbs)
return pbs, string_utf8.unicode(str, pbs, pbs), pbs
end
end
--[[
differs from Lua 5.3 utf8.offset in accepting any byte positions (not only head byte) for all n values
h - head, c - continuation, t - tail
hhhccthccthccthcthhh
^ start byte pos
searching current charracter head by moving backwards
hhhccthccthccthcthhh
^ head
n == 0: current position
n > 0: n jumps forward
n < 0: n more scans backwards
--]]
--- Finds the byte position of the `n`-th UTF-8 character in a string, starting from a given byte position.
--- @param str string The input UTF-8 encoded string.
--- @param n number The character index to find. If `n` is negative, counts from the end.
--- @param bs? number The starting byte position (default: 1 for positive `n`, end of string for negative `n`).
--- @return number | nil
function string_utf8.offset(str, n, bs)
local l = #str
if not bs then
if n < 0 then
bs = l + 1
else
bs = 1
end
end
if bs <= 0 or bs > l + 1 then
error("bad argument #3 to 'offset' (position out of range)")
end
if n == 0 then
if bs == l + 1 then
return bs
end
while true do
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
return bs
end
bs = bs - 1
if bs < 1 then
return
end
end
elseif n < 0 then
bs = bs - 1
repeat
if bs < 1 then
return
end
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
n = n + 1
end
bs = bs - 1
until n == 0
return bs + 1
else
while true do
if bs > l then
return
end
local b = byte(str, bs)
if (0 < b and b < 127)
or (194 < b and b < 244) then
n = n - 1
for i = 1, n do
if bs > l then
return
end
bs = utf8next(str, bs)
end
return bs
end
bs = bs - 1
end
end
end
--- Replaces UTF-8 characters in a string according to a mapping table.
--- @param s string The input UTF-8 encoded string.
--- @param mapping table A table where keys are UTF-8 characters (or patterns) and values are their replacements.
--- @return string
function string_utf8.replace(s, mapping)
if type(s) ~= "string" then
error("bad argument #1 to 'utf8replace' (string expected, got ".. type(s).. ")")
end
if type(mapping) ~= "table" then
error("bad argument #2 to 'utf8replace' (table expected, got ".. type(mapping).. ")")
end
local result = string.gsub( s, utf8charpattern, mapping )
return result
end
--- Converts all UTF-8 characters in a string to uppercase.
--- @param s string The input UTF-8 encoded string.
--- @return string
function string_utf8.upper (s)
return string_utf8.replace(s, string_utf8.lc_uc)
end
--- Converts all UTF-8 characters in a string to lowercase.
--- @param s string The input UTF-8 encoded string.
--- @return string
function string_utf8.lower (s)
return string_utf8.replace(s, string_utf8.uc_lc)
end

File diff suppressed because it is too large Load Diff