shawn/op-pedal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Guard against invalid UTF-8 in filenames.

Browse Source
This commit is contained in:
Robin E. R. Davies
2025-06-21 11:27:29 -04:00
parent ca6bf365fb
commit 2dbb9d8c6d
6 changed files with 187 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/launch.json
Vendored
+1 -1
View File
@@ -177,7 +177,7 @@
"args": [
//"[Dev]" -- all dev-machine tests.
//"[promise]" // subtest of your choice, or none to run all of the tests.
"[atom_converter]"
"[json_variants]"
],
"stopAtEntry": false,
PiPedalCommon/src/Utf8Utils.cpp
+87 -45
View File
@@ -260,81 +260,123 @@ namespace pipedal
auto p = v.begin();
while (p != v.end())
{
uint32_t uc;
uint8_t c = (uint8_t)*p++;
if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
{
uc = c;
}
else
{
uint32_t c2 = continuation_byte(p, v.end());
if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
try {
uint32_t uc;
uint8_t c = (uint8_t)*p++;
if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
{
uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK);
if (c1 <= 1 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 6) | c2;
uc = c;
}
else
{
uint32_t c3 = continuation_byte(p, v.end());
uint32_t c2 = continuation_byte(p, v.end());
if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK;
if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding)
uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK);
if (c1 <= 1 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1) << 12 | (c2 << 6) | c3;
uc = (c1 << 6) | c2;
}
else
{
uint32_t c4 = continuation_byte(p, v.end());
if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
uint32_t c3 = continuation_byte(p, v.end());
if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK;
if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding)
uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK;
if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
uc = (c1) << 12 | (c2 << 6) | c3;
}
else
{
// outside legal UCS range.
throw_encoding_error();
uint32_t c4 = continuation_byte(p, v.end());
if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK;
if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
}
else
{
// outside legal UCS range.
throw_encoding_error();
}
}
}
}
}
if (uc < 0x10000ul)
{
os << (char16_t)uc;
}
else
{
// write UTF-16 surrogate pair.
uc -= 0x10000;
if (uc < 0x10000ul)
{
os << (char16_t)uc;
}
else
{
// write UTF-16 surrogate pair.
uc -= 0x10000;
char16_t s1 = (char16_t)(UTF16_SURROGATE_1_BASE + ((uc >> 10) & 0x3FFu));
char16_t s2 = (char16_t)(UTF16_SURROGATE_2_BASE + (uc & 0x03FFu));
// surrogate pair.
os << s1;
os << s2;
char16_t s1 = (char16_t)(UTF16_SURROGATE_1_BASE + ((uc >> 10) & 0x3FFu));
char16_t s2 = (char16_t)(UTF16_SURROGATE_2_BASE + (uc & 0x03FFu));
// surrogate pair.
os << s1;
os << s2;
}
} catch (const std::exception &e)
{
os << u'\uFFFD'; // replacement character for invalid sequences.
}
}
return os.str();
}
bool IsValidUtf8(const std::string &text)
{
auto p = text.begin();
while (p != text.end())
{
uint8_t c = *p++;
if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
{
// 1-byte character (ASCII)
continue;
}
else if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
{
// 2-byte character
if (p == text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
return false;
}
else if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
{
// 3-byte character
if (p + 1 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
return false;
}
else if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
{
// 4-byte character
if (p + 2 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
return false;
}
else
{
// Invalid UTF-8
return false;
}
}
return true;
}
}
PiPedalCommon/src/include/Utf8Utils.hpp
+2
View File
@@ -45,6 +45,8 @@ namespace pipedal {
std::u16string Utf8ToUtf16(const std::string_view&s);
std::string Utf16ToUtf8(const std::u16string_view&s);
bool IsValidUtf8(const std::string &text);
}
PiPedalCommon/src/json.cpp
+36 -30
View File
@@ -70,57 +70,63 @@ void json_writer::write(string_view v,bool enforceValidUtf8Encoding)
{
uint32_t uc;
uint8_t c = (uint8_t)*p++;
if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
{
uc = c;
}
else
{
uint32_t c2 = continuation_byte(p, v.end());
if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
try {
if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
{
uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK);
if (c1 <= 1 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 6) | c2;
uc = c;
}
else
{
uint32_t c3 = continuation_byte(p, v.end());
uint32_t c2 = continuation_byte(p, v.end());
if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK;
if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding)
uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK);
if (c1 <= 1 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1) << 12 | (c2 << 6) | c3;
uc = (c1 << 6) | c2;
}
else
{
uint32_t c4 = continuation_byte(p, v.end());
if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
uint32_t c3 = continuation_byte(p, v.end());
if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK;
if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding)
uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK;
if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
} else {
// outside legal UCS range.
throw_encoding_error();
uc = (c1) << 12 | (c2 << 6) | c3;
}
else
{
uint32_t c4 = continuation_byte(p, v.end());
if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
{
uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK;
if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding)
{
// overlong encoding.
throw_encoding_error();
}
uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4;
} else {
// outside legal UCS range.
throw_encoding_error();
}
}
}
}
} catch (const std::exception &e) {
// invalid UTF-8 sequence.
os << "\\uFFFD"; // replacement character for invalid sequences.
continue;
}
// if ((uc >= UTF16_SURROGATE_1_BASE && uc <= UTF16_SURROGATE_1_BASE + UTF16_SURROGATE_MASK) || (uc >= UTF16_SURROGATE_2_BASE && uc <= UTF16_SURROGATE_2_BASE + UTF16_SURROGATE_MASK))
// {
src/Storage.cpp
+35
View File
@@ -38,6 +38,7 @@
#include <MimeTypes.hpp>
#include "util.hpp"
#include "AudioFiles.hpp"
#include "Utf8Utils.hpp"
using namespace pipedal;
namespace fs = std::filesystem;
@@ -202,6 +203,12 @@ static void CopyDirectory(const std::filesystem::path &source, const std::filesy
{
for (auto &directoryEntry : std::filesystem::directory_iterator(source))
{
if (!IsValidUtf8(directoryEntry.path().string()))
{
// skip invalid UTF-8 paths.
Lv2Log::warning("Skipping invalid UTF-8 path: %s", directoryEntry.path().string().c_str());
continue;
}
if (directoryEntry.is_regular_file())
{
std::filesystem::path sourceFile = directoryEntry.path();
@@ -394,6 +401,12 @@ void Storage::ReIndex()
{
for (const auto &dirEntry : std::filesystem::directory_iterator(GetPresetsDirectory()))
{
if (!IsValidUtf8(dirEntry.path().string()))
{
// skip invalid UTF-8 paths.
Lv2Log::warning("Skipping invalid UTF-8 path: %s", dirEntry.path().string().c_str());
continue;
}
if (!dirEntry.is_directory())
{
auto path = dirEntry.path();
@@ -1646,6 +1659,11 @@ static void AddFilesToResult(
{
for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath))
{
if (!IsValidUtf8(dir_entry.path().string()))
{
Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string());
continue; // skip invalid UTF-8 names.
}
const auto &path = dir_entry.path();
auto name = path.filename().string();
if (dir_entry.is_regular_file())
@@ -1717,6 +1735,11 @@ static void AddTracksToResult(
// Add directories first.
for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath))
{
if (!IsValidUtf8(dir_entry.path().string()))
{
Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string());
continue; // skip invalid UTF-8 names.
}
const auto &path = dir_entry.path();
auto name = path.filename().string();
if (dir_entry.is_directory())
@@ -2295,6 +2318,12 @@ void Storage::FillSampleDirectoryTree(FilePropertyDirectoryTree *node, const std
{
for (auto child : std::filesystem::directory_iterator(directory))
{
if (!IsValidUtf8(child.path().string()))
{
Lv2Log::warning("Invalid UTF-8 name in directory: " + child.path().string());
// skip invalid UTF-8 paths.
continue;
}
if (child.is_directory())
{
const auto &childPath = child.path();
@@ -2317,6 +2346,12 @@ static void GetAllExtensions(std::set<std::string> &result, const std::filesyste
for (const auto &dirEnt : fs::directory_iterator(path))
{
if (!IsValidUtf8(dirEnt.path().string()))
{
// skip invalid UTF-8 paths.
Lv2Log::warning("Invalid UTF-8 name in directory: " + dirEnt.path().string());
continue;
}
if (dirEnt.is_directory())
{
GetAllExtensions(result, dirEnt.path());
src/jsonTest.cpp
+26
View File
@@ -267,11 +267,37 @@ void TestVariantSFINAE()
int i;
REQUIRE(x.write(i) == false);
}
void TestIlleglUtf8Sequences()
{
{
std::string json = "\"\x80\x80\x80\"";
std::stringstream s(json);
json_reader reader(s);
std::string v;
reader.read(&v);
// just sufficient that it doesn't throw.
}
{
std::string illegalJsonString = "123\x80\x80\x80xyz";
std::stringstream ss;
json_writer writer(ss);
writer.write(illegalJsonString);
std::string output = ss.str();
std::string result = ss.str();
REQUIRE(result == "\"123\\uFFFDyz\""); // U+FFFD is the replacement character for illegal UTF-8 sequences.
}
}
TEST_CASE("json variants", "[json_variants][Build][Dev]")
{
{
TestIlleglUtf8Sequences();
TestVariantSFINAE();
TestVariantRoundTrip(0.0);
TestVariantRoundTrip(std::string("abc"));
TestVariantRoundTrip(json_null());