From 2dbb9d8c6d4f5c25df82d67ff26e4b6be2e25d93 Mon Sep 17 00:00:00 2001 From: "Robin E. R. Davies" Date: Sat, 21 Jun 2025 11:27:29 -0400 Subject: [PATCH] Guard against invalid UTF-8 in filenames. --- .vscode/launch.json | 2 +- PiPedalCommon/src/Utf8Utils.cpp | 132 ++++++++++++++++-------- PiPedalCommon/src/include/Utf8Utils.hpp | 2 + PiPedalCommon/src/json.cpp | 66 ++++++------ src/Storage.cpp | 35 +++++++ src/jsonTest.cpp | 26 +++++ 6 files changed, 187 insertions(+), 76 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2ab0750..8f588ff 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -177,7 +177,7 @@ "args": [ //"[Dev]" -- all dev-machine tests. //"[promise]" // subtest of your choice, or none to run all of the tests. - "[atom_converter]" + "[json_variants]" ], "stopAtEntry": false, diff --git a/PiPedalCommon/src/Utf8Utils.cpp b/PiPedalCommon/src/Utf8Utils.cpp index f9f85b9..e9519e8 100644 --- a/PiPedalCommon/src/Utf8Utils.cpp +++ b/PiPedalCommon/src/Utf8Utils.cpp @@ -260,81 +260,123 @@ namespace pipedal auto p = v.begin(); while (p != v.end()) { - uint32_t uc; - uint8_t c = (uint8_t)*p++; - if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS) - { - uc = c; - } - else - { - uint32_t c2 = continuation_byte(p, v.end()); - - if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS) + try { + uint32_t uc; + uint8_t c = (uint8_t)*p++; + if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS) { - uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK); - if (c1 <= 1 && enforceValidUtf8Encoding) - { - // overlong encoding. - throw_encoding_error(); - } - uc = (c1 << 6) | c2; + uc = c; } else { - uint32_t c3 = continuation_byte(p, v.end()); + uint32_t c2 = continuation_byte(p, v.end()); - if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS) + if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS) { - uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK; - if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding) + uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK); + if (c1 <= 1 && enforceValidUtf8Encoding) { // overlong encoding. throw_encoding_error(); } - - uc = (c1) << 12 | (c2 << 6) | c3; + uc = (c1 << 6) | c2; } else { - uint32_t c4 = continuation_byte(p, v.end()); - if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS) + uint32_t c3 = continuation_byte(p, v.end()); + + if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS) { - uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK; - if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding) + uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK; + if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding) { // overlong encoding. throw_encoding_error(); } - uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4; + + uc = (c1) << 12 | (c2 << 6) | c3; } else { - // outside legal UCS range. - throw_encoding_error(); + uint32_t c4 = continuation_byte(p, v.end()); + if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS) + { + uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK; + if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding) + { + // overlong encoding. + throw_encoding_error(); + } + uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4; + } + else + { + // outside legal UCS range. + throw_encoding_error(); + } } } } - } - if (uc < 0x10000ul) - { - os << (char16_t)uc; - } - else - { - // write UTF-16 surrogate pair. - uc -= 0x10000; + if (uc < 0x10000ul) + { + os << (char16_t)uc; + } + else + { + // write UTF-16 surrogate pair. + uc -= 0x10000; - char16_t s1 = (char16_t)(UTF16_SURROGATE_1_BASE + ((uc >> 10) & 0x3FFu)); - char16_t s2 = (char16_t)(UTF16_SURROGATE_2_BASE + (uc & 0x03FFu)); - // surrogate pair. - os << s1; - os << s2; + char16_t s1 = (char16_t)(UTF16_SURROGATE_1_BASE + ((uc >> 10) & 0x3FFu)); + char16_t s2 = (char16_t)(UTF16_SURROGATE_2_BASE + (uc & 0x03FFu)); + // surrogate pair. + os << s1; + os << s2; + } + } catch (const std::exception &e) + { + os << u'\uFFFD'; // replacement character for invalid sequences. } } return os.str(); } + bool IsValidUtf8(const std::string &text) + { + auto p = text.begin(); + while (p != text.end()) + { + uint8_t c = *p++; + if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS) + { + // 1-byte character (ASCII) + continue; + } + else if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS) + { + // 2-byte character + if (p == text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS)) + return false; + } + else if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS) + { + // 3-byte character + if (p + 1 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS)) + return false; + } + else if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS) + { + // 4-byte character + if (p + 2 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS)) + return false; + } + else + { + // Invalid UTF-8 + return false; + } + } + return true; + } } \ No newline at end of file diff --git a/PiPedalCommon/src/include/Utf8Utils.hpp b/PiPedalCommon/src/include/Utf8Utils.hpp index 5b31af4..b0c5d31 100644 --- a/PiPedalCommon/src/include/Utf8Utils.hpp +++ b/PiPedalCommon/src/include/Utf8Utils.hpp @@ -44,6 +44,8 @@ namespace pipedal { std::u16string Utf8ToUtf16(const std::string_view&s); std::string Utf16ToUtf8(const std::u16string_view&s); + + bool IsValidUtf8(const std::string &text); } diff --git a/PiPedalCommon/src/json.cpp b/PiPedalCommon/src/json.cpp index 67460b6..d486bf8 100644 --- a/PiPedalCommon/src/json.cpp +++ b/PiPedalCommon/src/json.cpp @@ -70,57 +70,63 @@ void json_writer::write(string_view v,bool enforceValidUtf8Encoding) { uint32_t uc; uint8_t c = (uint8_t)*p++; - if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS) - { - uc = c; - } - else - { - uint32_t c2 = continuation_byte(p, v.end()); - - if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS) + try { + if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS) { - uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK); - if (c1 <= 1 && enforceValidUtf8Encoding) - { - // overlong encoding. - throw_encoding_error(); - } - uc = (c1 << 6) | c2; + uc = c; } else { - uint32_t c3 = continuation_byte(p, v.end()); + uint32_t c2 = continuation_byte(p, v.end()); - if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS) + if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS) { - uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK; - if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding) + uint32_t c1 = c & (uint32_t)(~UTF8_TWO_BYTES_MASK); + if (c1 <= 1 && enforceValidUtf8Encoding) { // overlong encoding. throw_encoding_error(); } - - uc = (c1) << 12 | (c2 << 6) | c3; + uc = (c1 << 6) | c2; } else { - uint32_t c4 = continuation_byte(p, v.end()); - if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS) + uint32_t c3 = continuation_byte(p, v.end()); + + if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS) { - uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK; - if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding) + uint32_t c1 = c & (uint32_t)~UTF8_THREE_BYTES_MASK; + if (c1 == 0 && c2 < 0x20 && enforceValidUtf8Encoding) { // overlong encoding. throw_encoding_error(); } - uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4; - } else { - // outside legal UCS range. - throw_encoding_error(); + + uc = (c1) << 12 | (c2 << 6) | c3; + } + else + { + uint32_t c4 = continuation_byte(p, v.end()); + if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS) + { + uint32_t c1 = c & (uint32_t)~UTF8_FOUR_BYTES_MASK; + if (c1 == 0 && c2 < 0x10 && enforceValidUtf8Encoding) + { + // overlong encoding. + throw_encoding_error(); + } + uc = (c1 << 18) | (c2 << 12) | (c3 << 6) | c4; + } else { + // outside legal UCS range. + throw_encoding_error(); + } } } } + } catch (const std::exception &e) { + // invalid UTF-8 sequence. + os << "\\uFFFD"; // replacement character for invalid sequences. + continue; } // if ((uc >= UTF16_SURROGATE_1_BASE && uc <= UTF16_SURROGATE_1_BASE + UTF16_SURROGATE_MASK) || (uc >= UTF16_SURROGATE_2_BASE && uc <= UTF16_SURROGATE_2_BASE + UTF16_SURROGATE_MASK)) // { diff --git a/src/Storage.cpp b/src/Storage.cpp index f46b4d8..c3c5dcd 100644 --- a/src/Storage.cpp +++ b/src/Storage.cpp @@ -38,6 +38,7 @@ #include #include "util.hpp" #include "AudioFiles.hpp" +#include "Utf8Utils.hpp" using namespace pipedal; namespace fs = std::filesystem; @@ -202,6 +203,12 @@ static void CopyDirectory(const std::filesystem::path &source, const std::filesy { for (auto &directoryEntry : std::filesystem::directory_iterator(source)) { + if (!IsValidUtf8(directoryEntry.path().string())) + { + // skip invalid UTF-8 paths. + Lv2Log::warning("Skipping invalid UTF-8 path: %s", directoryEntry.path().string().c_str()); + continue; + } if (directoryEntry.is_regular_file()) { std::filesystem::path sourceFile = directoryEntry.path(); @@ -394,6 +401,12 @@ void Storage::ReIndex() { for (const auto &dirEntry : std::filesystem::directory_iterator(GetPresetsDirectory())) { + if (!IsValidUtf8(dirEntry.path().string())) + { + // skip invalid UTF-8 paths. + Lv2Log::warning("Skipping invalid UTF-8 path: %s", dirEntry.path().string().c_str()); + continue; + } if (!dirEntry.is_directory()) { auto path = dirEntry.path(); @@ -1646,6 +1659,11 @@ static void AddFilesToResult( { for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath)) { + if (!IsValidUtf8(dir_entry.path().string())) + { + Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string()); + continue; // skip invalid UTF-8 names. + } const auto &path = dir_entry.path(); auto name = path.filename().string(); if (dir_entry.is_regular_file()) @@ -1717,6 +1735,11 @@ static void AddTracksToResult( // Add directories first. for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath)) { + if (!IsValidUtf8(dir_entry.path().string())) + { + Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string()); + continue; // skip invalid UTF-8 names. + } const auto &path = dir_entry.path(); auto name = path.filename().string(); if (dir_entry.is_directory()) @@ -2295,6 +2318,12 @@ void Storage::FillSampleDirectoryTree(FilePropertyDirectoryTree *node, const std { for (auto child : std::filesystem::directory_iterator(directory)) { + if (!IsValidUtf8(child.path().string())) + { + Lv2Log::warning("Invalid UTF-8 name in directory: " + child.path().string()); + // skip invalid UTF-8 paths. + continue; + } if (child.is_directory()) { const auto &childPath = child.path(); @@ -2317,6 +2346,12 @@ static void GetAllExtensions(std::set &result, const std::filesyste for (const auto &dirEnt : fs::directory_iterator(path)) { + if (!IsValidUtf8(dirEnt.path().string())) + { + // skip invalid UTF-8 paths. + Lv2Log::warning("Invalid UTF-8 name in directory: " + dirEnt.path().string()); + continue; + } if (dirEnt.is_directory()) { GetAllExtensions(result, dirEnt.path()); diff --git a/src/jsonTest.cpp b/src/jsonTest.cpp index cdf88ff..b8f8355 100644 --- a/src/jsonTest.cpp +++ b/src/jsonTest.cpp @@ -267,11 +267,37 @@ void TestVariantSFINAE() int i; REQUIRE(x.write(i) == false); } + +void TestIlleglUtf8Sequences() +{ + { + std::string json = "\"\x80\x80\x80\""; + std::stringstream s(json); + json_reader reader(s); + std::string v; + reader.read(&v); + // just sufficient that it doesn't throw. + } + + + { + std::string illegalJsonString = "123\x80\x80\x80xyz"; + std::stringstream ss; + json_writer writer(ss); + writer.write(illegalJsonString); + std::string output = ss.str(); + std::string result = ss.str(); + REQUIRE(result == "\"123\\uFFFDyz\""); // U+FFFD is the replacement character for illegal UTF-8 sequences. + } +} TEST_CASE("json variants", "[json_variants][Build][Dev]") { { + TestIlleglUtf8Sequences(); + TestVariantSFINAE(); + TestVariantRoundTrip(0.0); TestVariantRoundTrip(std::string("abc")); TestVariantRoundTrip(json_null());