Guard against invalid UTF-8 in filenames.

.vscode/launch.json

@@ -177,7 +177,7 @@
             "args": [
                 //"[Dev]"  -- all dev-machine tests.
                 //"[promise]"  // subtest of your choice, or none to run all of the tests.
-                "[atom_converter]"
+                "[json_variants]"
             ],
 
             "stopAtEntry": false,

PiPedalCommon/src/Utf8Utils.cpp

@@ -260,6 +260,7 @@ namespace pipedal
         auto p = v.begin();
         while (p != v.end())
         {
+            try {
                 uint32_t uc;
                 uint8_t c = (uint8_t)*p++;
                 if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
@@ -331,10 +332,51 @@ namespace pipedal
                     os << s1;
                     os << s2;
                 }
+            } catch (const std::exception &e)
+            {
+                os << u'\uFFFD'; // replacement character for invalid sequences.
+            }
         }
         return os.str();
     }
 
+    bool IsValidUtf8(const std::string &text)
+    {
+        auto p = text.begin();
+        while (p != text.end())
+        {
+            uint8_t c = *p++;
+            if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
+            {
+                // 1-byte character (ASCII)
+                continue;
+            }
+            else if ((c & UTF8_TWO_BYTES_MASK) == UTF8_TWO_BYTES_BITS)
+            {
+                // 2-byte character
+                if (p == text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
+                    return false;
+            }
+            else if ((c & UTF8_THREE_BYTES_MASK) == UTF8_THREE_BYTES_BITS)
+            {
+                // 3-byte character
+                if (p + 1 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
+                    return false;
+            }
+            else if ((c & UTF8_FOUR_BYTES_MASK) == UTF8_FOUR_BYTES_BITS)
+            {
+                // 4-byte character
+                if (p + 2 >= text.end() || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS) || ((*p++ & UTF8_CONTINUATION_MASK) != UTF8_CONTINUATION_BITS))
+                    return false;
+            }
+            else
+            {
+                // Invalid UTF-8
+                return false;
+            }
+        }
+        return true;
+    }
 
  
 }

PiPedalCommon/src/include/Utf8Utils.hpp

@@ -45,6 +45,8 @@ namespace pipedal {
     std::u16string Utf8ToUtf16(const std::string_view&s);
     std::string Utf16ToUtf8(const std::u16string_view&s);
     
+    bool IsValidUtf8(const std::string &text);
+
 
 }
 

PiPedalCommon/src/json.cpp

@@ -70,6 +70,7 @@ void json_writer::write(string_view v,bool enforceValidUtf8Encoding)
     {
         uint32_t uc;
         uint8_t c = (uint8_t)*p++;
+        try {
             if ((c & UTF8_ONE_BYTE_MASK) == UTF8_ONE_BYTE_BITS)
             {
                 uc = c;
@@ -122,6 +123,11 @@ void json_writer::write(string_view v,bool enforceValidUtf8Encoding)
                     }
                 }
             }
+        } catch (const std::exception &e) {
+            // invalid UTF-8 sequence.
+            os << "\\uFFFD"; // replacement character for invalid sequences.
+            continue;
+        }
         // if ((uc >= UTF16_SURROGATE_1_BASE && uc <= UTF16_SURROGATE_1_BASE + UTF16_SURROGATE_MASK) || (uc >= UTF16_SURROGATE_2_BASE && uc <= UTF16_SURROGATE_2_BASE + UTF16_SURROGATE_MASK))
         // {
         //     // MUST not encode UTF16 surrogates in UTF8.

src/Storage.cpp

@@ -38,6 +38,7 @@
 #include <MimeTypes.hpp>
 #include "util.hpp"
 #include "AudioFiles.hpp"
+#include "Utf8Utils.hpp"
 
 using namespace pipedal;
 namespace fs = std::filesystem;
@@ -202,6 +203,12 @@ static void CopyDirectory(const std::filesystem::path &source, const std::filesy
 {
     for (auto &directoryEntry : std::filesystem::directory_iterator(source))
     {
+        if (!IsValidUtf8(directoryEntry.path().string()))
+        {
+            // skip invalid UTF-8 paths.
+            Lv2Log::warning("Skipping invalid UTF-8 path: %s", directoryEntry.path().string().c_str());
+            continue;
+        }
         if (directoryEntry.is_regular_file())
         {
             std::filesystem::path sourceFile = directoryEntry.path();
@@ -394,6 +401,12 @@ void Storage::ReIndex()
 {
     for (const auto &dirEntry : std::filesystem::directory_iterator(GetPresetsDirectory()))
     {
+        if (!IsValidUtf8(dirEntry.path().string()))
+        {
+            // skip invalid UTF-8 paths.
+            Lv2Log::warning("Skipping invalid UTF-8 path: %s", dirEntry.path().string().c_str());
+            continue;
+        }
         if (!dirEntry.is_directory())
         {
             auto path = dirEntry.path();
@@ -1646,6 +1659,11 @@ static void AddFilesToResult(
     {
         for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath))
         {
+            if (!IsValidUtf8(dir_entry.path().string()))
+            {
+                Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string());
+                continue; // skip invalid UTF-8 names.
+            }
             const auto &path = dir_entry.path();
             auto name = path.filename().string();
             if (dir_entry.is_regular_file())
@@ -1717,6 +1735,11 @@ static void AddTracksToResult(
         // Add directories first.
         for (auto const &dir_entry : std::filesystem::directory_iterator(rootPath))
         {
+            if (!IsValidUtf8(dir_entry.path().string()))
+            {
+                Lv2Log::warning("Invalid UTF-8 name in directory: " + dir_entry.path().string());
+                continue; // skip invalid UTF-8 names.
+            }
             const auto &path = dir_entry.path();
             auto name = path.filename().string();
             if (dir_entry.is_directory())
@@ -2295,6 +2318,12 @@ void Storage::FillSampleDirectoryTree(FilePropertyDirectoryTree *node, const std
 {
     for (auto child : std::filesystem::directory_iterator(directory))
     {
+        if (!IsValidUtf8(child.path().string()))
+        {
+            Lv2Log::warning("Invalid UTF-8 name in directory: " + child.path().string());
+            // skip invalid UTF-8 paths.
+            continue;
+        }
         if (child.is_directory())
         {
             const auto &childPath = child.path();
@@ -2317,6 +2346,12 @@ static void GetAllExtensions(std::set<std::string> &result, const std::filesyste
 
     for (const auto &dirEnt : fs::directory_iterator(path))
     {
+        if (!IsValidUtf8(dirEnt.path().string()))
+        {
+            // skip invalid UTF-8 paths.
+            Lv2Log::warning("Invalid UTF-8 name in directory: " + dirEnt.path().string());
+            continue;
+        }
         if (dirEnt.is_directory())
         {
             GetAllExtensions(result, dirEnt.path());

src/jsonTest.cpp

@@ -267,11 +267,37 @@ void TestVariantSFINAE()
     int i;
     REQUIRE(x.write(i) == false);
 }
+
+void TestIlleglUtf8Sequences()
+{
+    {
+        std::string json = "\"\x80\x80\x80\"";
+        std::stringstream s(json);
+        json_reader reader(s);
+        std::string v;
+        reader.read(&v);
+        // just sufficient that it doesn't throw.
+    }
+
+
+    {
+    std::string illegalJsonString = "123\x80\x80\x80xyz";
+        std::stringstream ss;
+        json_writer writer(ss);
+        writer.write(illegalJsonString);
+        std::string output = ss.str();
+        std::string result = ss.str();
+        REQUIRE(result == "\"123\\uFFFDyz\""); // U+FFFD is the replacement character for illegal UTF-8 sequences.
+    }
+}   
 TEST_CASE("json variants", "[json_variants][Build][Dev]")
 {
     {
+        TestIlleglUtf8Sequences();
+
         TestVariantSFINAE();
 
+        
         TestVariantRoundTrip(0.0);
         TestVariantRoundTrip(std::string("abc"));
         TestVariantRoundTrip(json_null());