Toob ML Uploadable models, auto-provision resource directories.

docs/BuildPrerequisites.md

@@ -27,7 +27,7 @@ Run the following commands to install dependent libraries required by the PiPeda
     sudo apt install -y authbind
     sudo apt install -y libavahi-client-dev
     sudo apt install -y libnm-dev
-    sudo apt install -y libsdbus-c++-dev
+    sudo apt install -y libsdbus-c++-dev libzip-dev
     
 
 ### Installing Sources

lv2/aarch64/ToobAmp.lv2/ToobML.ttl

@@ -26,6 +26,12 @@
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 @prefix toobml: <http://two-play.com/plugins/toob-ml#> .
 
+
+toobml:modelFile
+        a lv2:Parameter;
+        rdfs:label "Model";
+        rdfs:range atom:Path.
+
 toobml:filterGroup
     a param:ControlGroup ,
         pg:InputGroup ;
@@ -66,7 +72,25 @@ Generally, TooB ML uses smaller and more efficient neural networks than does Too
 TooB ML models use significantly less CPU, although TooB Neural Amp Modeller is significantly more flexible, and
 has a huge number of downloadable models.
 
-The BluesJR model has an additional Gain control the is used by the BluesJR model to simulate the effect of Blues Junior's gain control.
+TooB ML models are compatible with models from the GuitarML project (). You can download new models from the GuitarML project 
+at 
+
+    https://guitarml.com/tonelibrary/tonelib-pro.html
+
+To use GuitarML models, download them locally using your browser, and then upload them to TooB ML using the following steps:
+
+- Load Toob ML in PiPedal.
+
+- Click on the "Models" control.
+
+- Click on the _Upload_ button in the lower left corner of the selection dialog.
+
+- Select the file that you downloaded localy.
+
+TooB ML can upload indiviual .json files, or collections of models in .zip files downloaded from the GuitarML site.
+
+Some models have an additional Gain control that can be used to simulate the effect of a Gain control on the amplifier 
+being simulated. The gain control will be enabled if the model you have selected has a gain input.
 
 For convenience, tone controls are provided for rough shaping of tone before the input signal is fed to the neural net models. The tone 
 section can be completely bypassed by setting all tone controls to exactly 0.5.
@@ -85,7 +109,6 @@ on the edge of breakup. It does not work particularly well for models of heavily
 compressor. Notionally, Sag Feedback models the effect of transformer sag on intermediate transformer stages (which causes them to clip at a lower level than
 they ordinarily would); and Sag Out models the effect of transformer sag on the final power stage of a guitar amplifier.
 
-
 The TooB ML Amplifier is based heavily on code ported from from the RTNeural project (https://github.com/jatinchowdhury18/RTNeural), by Jatin Chowdhury,
 and uses model files from the GuitarML Neural Pi project (https://github.com/GuitarML/NeuralPi).
 
@@ -97,6 +120,11 @@ and uses model files from the GuitarML Neural Pi project (https://github.com/Gui
         lv2:optionalFeature lv2:hardRTCapable;
         lv2:extensionData state:interface, work:interface;
 
+        patch:readable 
+                toobml:modelFile;
+        patch:writable 
+                toobml:modelFile;
+
         patch:readable 
                 <http://two-play.com/plugins/toob#frequencyResponseVector>
                 ;
@@ -128,12 +156,14 @@ and uses model files from the GuitarML Neural Pi project (https://github.com/Gui
                 units:unit units:db ;
         ],
         [
+                # legagcy for loading of old instances.
                 a lv2:InputPort, lv2:ControlPort ;
+                lv2:portProperty epp:notOnGUI;
                 lv2:index 2 ;
                 lv2:symbol "model" ;
                 lv2:name "Model" ;
-                lv2:default 0.0 ; 
+                lv2:default -1.0 ; 
-                lv2:minimum 0.0 ;
+                lv2:minimum -1.0 ;
                 lv2:maximum 32.0 ;
                 lv2:portProperty lv2:enumeration ;
                 lv2:scalePoint 
@@ -462,6 +492,31 @@ and uses model files from the GuitarML Neural Pi project (https://github.com/Gui
 
 <http://two-play.com/plugins/toob-ml#piPedalUI> 
     a pipedal_ui:ui ;
+    pipedal_ui:fileProperties 
+    [
+        a pipedal_ui:fileProperty;
+        rdfs:label "Model (*.json)" ;
+        pipedal_ui:directory "ToobMlModels";
+        lv2:index 2 ;
+        pipedal_ui:patchProperty toobml:modelFile ;
+        pipedal_ui:resourceDirectory "models/tones";
+
+        pipedal_ui:fileTypes 
+        [
+            a pipedal_ui:fileType;
+            rdfs:label ".json file";
+            pipedal_ui:fileExtension ".json";
+            pipedal_ui:mimeType "application/json";
+        ],[
+            a pipedal_ui:fileType;
+            rdfs:label ".ml file";
+            pipedal_ui:fileExtension ".zip";
+            pipedal_ui:mimeType "application/zip";
+
+        ]
+        ;
+    ];
+
     pipedal_ui:frequencyPlot
     [
         lv2:index 8 ; 

react/src/App.tsx

@@ -79,7 +79,7 @@ const theme = createTheme(
                     main: '#A770E4'// #6750A4"   // #5B5690  #60529A  #5C5694
                 },
                 secondary: {
-                    main: "#0AA102" //'"#FF6060"
+                    main: "#FF6060"
                 },
             },
             mainBackground: "#222",

react/src/Lv2Plugin.tsx

@@ -250,6 +250,7 @@ export class UiFileProperty {
             this.directory = input.directory;
             this.index = input.index;
             this.portGroup = input.portGroup;
+            this.resourceDirectory = input.resourceDirectory??"";
             return this;
         }
         static deserialize_array(input: any): UiFileProperty[]
@@ -303,6 +304,7 @@ export class UiFileProperty {
         directory: string = "";
         index: number = -1;
         portGroup: string = "";
+        resourceDirectory: string = "";
 
 };
 export class  Lv2Plugin implements Deserializable<Lv2Plugin> {

src/CMakeLists.txt

@@ -62,6 +62,7 @@ else()
   message(STATUS "LILV_0_INCLUDE_DIRS: ${LILV_0_INCLUDE_DIRS}")
 endif()
 
+pkg_check_modules(ZIP "lilv-0")
 
 # Use LV2 headers from the /usr/lib directory.
 #set (LV2DEV_INCLUDE_DIRS /usr/lib)
@@ -131,6 +132,8 @@ else()
 endif()
 
 set (PIPEDAL_SOURCES
+    Finally.hpp
+    ZipFile.cpp ZipFile.hpp
     TemporaryFile.cpp TemporaryFile.hpp     
     FilePropertyDirectoryTree.cpp FilePropertyDirectoryTree.hpp 
     FileEntry.cpp FileEntry.hpp
@@ -230,7 +233,7 @@ set (PIPEDAL_INCLUDES
     .
 )
 
-set(PIPEDAL_LIBS libpipedald 
+set(PIPEDAL_LIBS libpipedald zip
     pthread atomic stdc++fs asound avahi-common avahi-client systemd
     ${VST3_LIBRARIES}
     ${LILV_0_LIBRARIES} 

src/FileBrowserFiles.h

@@ -111,7 +111,7 @@ typedef struct {
      get_upload_path returns the supplied path unmodified.
 `
      If the filename is a child of rootResourceDirectory, creates a corresponding link to the file in the file browser 
-     directory and returns the name of that link.
+     directory and returns the path of that link.
      
      The caller must free memory for the returned value with `LV2_FileBrowser_Files.free_path()`.
 

src/Finally.hpp

@@ -0,0 +1,39 @@
+// Copyright (c) 2024 Robin Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#pragma once
+#include <functional>
+
+namespace pipedal {
+
+    class Finally {
+    public:
+        Finally(std::function<void(void)> &&fn) 
+        :fn (std::move(fn))
+        {
+            
+        }
+        ~Finally() {
+            fn();
+        }
+    private:
+        std::function<void(void)> fn;
+    };
+
+}

src/PiPedalModel.cpp

@@ -138,7 +138,7 @@ void PiPedalModel::Init(const PiPedalConfiguration &configuration)
     storage.SetConfigRoot(configuration.GetDocRoot());
     storage.SetDataRoot(configuration.GetLocalStoragePath());
     storage.Initialize();
-    lv2Host.SetPluginStoragePath(storage.GetPluginAudioFileDirectory());
+    lv2Host.SetPluginStoragePath(storage.GetPluginUploadDirectory());
 
     this->systemMidiBindings = storage.GetSystemMidiBindings();
 
@@ -1151,6 +1151,7 @@ void PiPedalModel::RestartAudio()
 
     // do a complete reload.
 
+
     this->audioHost->SetPedalboard(nullptr);
 
     this->jackConfiguration.AlsaInitialize(this->jackServerSettings);
@@ -2093,6 +2094,17 @@ uint64_t PiPedalModel::CreateNewPreset()
     return storage.CreateNewPreset();
 }
 
+void PiPedalModel::CheckForResourceInitialization(Pedalboard &pedalboard)
+{
+    for (auto item: pedalboard.GetAllPlugins())
+    {
+        if (!item->isSplit())
+        {
+            lv2Host.CheckForResourceInitialization(item->uri(),storage.GetPluginUploadDirectory());
+        }
+
+    }
+}
 bool PiPedalModel::LoadCurrentPedalboard()
 {
     Lv2PedalboardErrorList errorMessages;
@@ -2101,6 +2113,7 @@ bool PiPedalModel::LoadCurrentPedalboard()
 
     // apply the error messages to the lv2Pedalboard.
     // return true if the error messages have changed
+    CheckForResourceInitialization(this->pedalboard);
     audioHost->SetPedalboard(lv2Pedalboard);
     return true;
 }

src/PiPedalModel.hpp

@@ -190,7 +190,7 @@ namespace pipedal
 
         PiPedalConfiguration configuration;
 
-
+        void CheckForResourceInitialization(Pedalboard &pedalboard);
     public:
         PiPedalModel();
         virtual ~PiPedalModel();

src/PiPedalUI.cpp

@@ -201,7 +201,15 @@ UiFileProperty::UiFileProperty(PluginHost *pHost, const LilvNode *node, const st
     {
         this->portGroup_ = portGroup.AsUri();
     }
-
+    AutoLilvNode resourceDirectory = lilv_world_get(pWorld,node,pHost->lilvUris->pipedalUI__resourceDirectory,nullptr);
+    if (resourceDirectory)
+    {
+        this->resourceDirectory_ =  resourceDirectory.AsString();
+    }
+    if (this->resourceDirectory_.empty())
+    {
+        this->resourceDirectory_ = "default";
+    }
     this->fileTypes_ = UiFileType::GetArray(pHost, node, pHost->lilvUris->pipedalUI__fileTypes);
 }
 
@@ -460,6 +468,7 @@ JSON_MAP_REFERENCE(UiFileProperty, directory)
 JSON_MAP_REFERENCE(UiFileProperty, patchProperty)
 JSON_MAP_REFERENCE(UiFileProperty, fileTypes)
 JSON_MAP_REFERENCE(UiFileProperty, portGroup)
+JSON_MAP_REFERENCE(UiFileProperty, resourceDirectory)
 JSON_MAP_END()
 
 JSON_MAP_BEGIN(UiFrequencyPlot)

src/PiPedalUI.hpp

@@ -42,6 +42,7 @@
 #define PIPEDAL_UI__patchProperty  PIPEDAL_UI_PREFIX "patchProperty"
 #define PIPEDAL_UI__directory  PIPEDAL_UI_PREFIX "directory"
 #define PIPEDAL_UI__fileTypes  PIPEDAL_UI_PREFIX "fileTypes"
+#define PIPEDAL_UI__resourceDirectory PIPEDAL_UI_PREFIX "resourceDirectory"
 
 #define PIPEDAL_UI__fileType  PIPEDAL_UI_PREFIX "fileType"
 #define PIPEDAL_UI__fileExtension  PIPEDAL_UI_PREFIX "fileExtension"
@@ -112,6 +113,7 @@ namespace pipedal {
         std::vector<UiFileType> fileTypes_;
         std::string patchProperty_;
         std::string portGroup_;
+        std::string resourceDirectory_;
     public:
         using ptr = std::shared_ptr<UiFileProperty>;
         UiFileProperty() { }
@@ -131,6 +133,8 @@ namespace pipedal {
         bool IsValidExtension(const std::string&extension) const;
         static  bool IsDirectoryNameValid(const std::string&value);
 
+        const std::string&resourceDirectory() const { return resourceDirectory_; }
+
     public:
         DECLARE_JSON_MAP(UiFileProperty);
     };

src/PluginHost.cpp

@@ -32,7 +32,7 @@
 #include "JackConfiguration.hpp"
 #include "lv2/urid/urid.h"
 #include "lv2/ui/ui.h"
-//#include "lv2.h"
+// #include "lv2.h"
 #include "lv2/atom/atom.h"
 #include "lv2/time/time.h"
 #include "lv2/state/state.h"
@@ -127,7 +127,7 @@ void PluginHost::LilvUris::Initialize(LilvWorld *pWorld)
     atom__bufferType = lilv_new_uri(pWorld, LV2_ATOM__bufferType);
     atom__Path = lilv_new_uri(pWorld, LV2_ATOM__Path);
     presets__preset = lilv_new_uri(pWorld, LV2_PRESETS__Preset);
-    state__state = lilv_new_uri(pWorld,LV2_STATE__state);
+    state__state = lilv_new_uri(pWorld, LV2_STATE__state);
     rdfs__label = lilv_new_uri(pWorld, LILV_NS_RDFS "label");
 
     lv2core__symbol = lilv_new_uri(pWorld, LV2_CORE__symbol);
@@ -140,6 +140,7 @@ void PluginHost::LilvUris::Initialize(LilvWorld *pWorld)
     pipedalUI__patchProperty = lilv_new_uri(pWorld, PIPEDAL_UI__patchProperty);
     pipedalUI__directory = lilv_new_uri(pWorld, PIPEDAL_UI__directory);
     pipedalUI__fileTypes = lilv_new_uri(pWorld, PIPEDAL_UI__fileTypes);
+    pipedalUI__resourceDirectory = lilv_new_uri(pWorld, PIPEDAL_UI__resourceDirectory);
     pipedalUI__fileProperty = lilv_new_uri(pWorld, PIPEDAL_UI__fileProperty);
     pipedalUI__fileExtension = lilv_new_uri(pWorld, PIPEDAL_UI__fileExtension);
     pipedalUI__mimeType = lilv_new_uri(pWorld, PIPEDAL_UI__mimeType);
@@ -163,9 +164,9 @@ void PluginHost::LilvUris::Initialize(LilvWorld *pWorld)
     lv2__symbol = lilv_new_uri(pWorld, LV2_CORE__symbol);
     lv2__port = lilv_new_uri(pWorld, LV2_CORE__port);
 
-    #define MOD_PREFIX "http://moddevices.com/ns/mod#"
+#define MOD_PREFIX "http://moddevices.com/ns/mod#"
-    mod__label = lilv_new_uri(pWorld,MOD_PREFIX "label");
+    mod__label = lilv_new_uri(pWorld, MOD_PREFIX "label");
-    mod__brand = lilv_new_uri(pWorld,MOD_PREFIX "brand");
+    mod__brand = lilv_new_uri(pWorld, MOD_PREFIX "brand");
     // ui:portNotification
     // [
     //         ui:portIndex 3;
@@ -660,17 +661,16 @@ Lv2PluginInfo::Lv2PluginInfo(PluginHost *lv2Host, LilvWorld *pWorld, const LilvP
     AutoLilvNode name = (lilv_plugin_get_name(pPlugin));
     this->name_ = nodeAsString(name);
 
-    AutoLilvNode brand = lilv_world_get(pWorld,plugUri,lv2Host->lilvUris->mod__brand, nullptr);
+    AutoLilvNode brand = lilv_world_get(pWorld, plugUri, lv2Host->lilvUris->mod__brand, nullptr);
     this->brand_ = nodeAsString(brand);
 
-    AutoLilvNode label = lilv_world_get(pWorld,plugUri,lv2Host->lilvUris->mod__label, nullptr);
+    AutoLilvNode label = lilv_world_get(pWorld, plugUri, lv2Host->lilvUris->mod__label, nullptr);
     this->label_ = nodeAsString(label);
     if (label_.length() == 0)
     {
         this->label_ = this->name_;
     }
 
-
     AutoLilvNode author_name = (lilv_plugin_get_author_name(pPlugin));
     this->author_name_ = nodeAsString(author_name);
 
@@ -1242,14 +1242,16 @@ void PluginHost::PortValueCallback(const char *symbol, void *user_data, const vo
     if (type == pHost->urids->atom__Double)
     {
         (*pState->values)[symbol] = (float)(*static_cast<const double *>(value));
-    } else if (type == pHost->urids->atom__Float)
+    }
+    else if (type == pHost->urids->atom__Float)
     {
         (*pState->values)[symbol] = *static_cast<const float *>(value);
     }
     else if (type == pHost->urids->atom_Int)
     {
         (*pState->values)[symbol] = *static_cast<const int32_t *>(value);
-    } else
+    }
+    else
     {
         pState->failed = true;
     }
@@ -1289,7 +1291,6 @@ PluginPresets PluginHost::GetFactoryPluginPresets(const std::string &pluginUri)
                 PresetCallbackState cbData{this, &controlValues, false};
                 lilv_state_emit_port_values(state, PortValueCallback, (void *)&cbData);
 
-
                 int numProperties = lilv_state_get_num_properties(state);
                 // can't handle std:state part of preset.
                 if (numProperties == 0)
@@ -1298,11 +1299,12 @@ PluginPresets PluginHost::GetFactoryPluginPresets(const std::string &pluginUri)
                     {
                         result.presets_.push_back(PluginPreset(result.nextInstanceId_++, strLabel, controlValues, Lv2PluginState()));
                     }
-                } else {
+                }
+                else
+                {
                     result.presets_.push_back(PluginPreset::MakeLilvPreset(result.nextInstanceId_++, strLabel, controlValues, lilv_node_as_uri(preset)));
                 }
                 lilv_state_free(state);
-
             }
         }
     }
@@ -1362,6 +1364,111 @@ std::shared_ptr<HostWorkerThread> PluginHost::GetHostWorkerThread()
     return pHostWorkerThread;
 }
 
+class ResourceInfo {
+public:
+    ResourceInfo(const std::string&filePropertyDirectory,const std::string&resourceDirectory)
+    :filePropertyDirectory(filePropertyDirectory), resourceDirectory(resourceDirectory) {}
+
+    std::string filePropertyDirectory;
+    std::string resourceDirectory;
+    bool operator==(const ResourceInfo&other) const {
+        return this->filePropertyDirectory == other.filePropertyDirectory && this->resourceDirectory == other.resourceDirectory;
+
+    }
+    bool operator<(const ResourceInfo&other) const {
+        if (this->filePropertyDirectory < other.filePropertyDirectory)
+            return true;
+        if (this->filePropertyDirectory > other.filePropertyDirectory)
+            return false;
+        return  this->resourceDirectory < other.resourceDirectory;
+
+    }
+};
+
+static bool anyTargetFilesExist(const std::filesystem::path &sourceDirectory, const std::filesystem::path&targetDirectory)
+{
+    namespace fs = std::filesystem;
+    try {
+        if (!fs::exists(targetDirectory)) return false;
+        for (auto&directoryEntry : fs::directory_iterator(sourceDirectory))
+        {
+            fs::path thisPath = directoryEntry.path();
+            if (directoryEntry.is_directory())
+            {
+                auto name = thisPath.filename();
+                fs::path childSource = sourceDirectory / name;
+                fs::path childTarget = targetDirectory / name;
+                if (anyTargetFilesExist(childSource,childTarget))
+                {
+                    return true;
+                }
+            } else {
+                fs::path targetPath = targetDirectory / thisPath.filename();
+                if (fs::exists(targetPath))
+                {
+                    return true;
+                }
+            }
+        }
+
+    } catch (const std::exception&) {
+
+    }
+    return false;
+}
+
+static void createTargetLinks(const std::filesystem::path &sourceDirectory, const std::filesystem::path &targetDirectory)
+{
+    namespace fs = std::filesystem;
+
+    fs::create_directories(targetDirectory);
+
+    for (auto &dirEntry : fs::directory_iterator(sourceDirectory))
+    {
+        fs::path childSource = dirEntry.path();
+        fs::path childTarget = targetDirectory / childSource.filename();
+        if (dirEntry.is_directory()) {
+            createTargetLinks(childSource,childTarget);
+        } else {
+            fs::create_symlink(childSource,childTarget);
+        }
+    }
+}
+void PluginHost::CheckForResourceInitialization(const std::string &pluginUri,const std::filesystem::path&pluginUploadDirectory)
+{
+
+    auto plugin = GetPluginInfo(pluginUri);
+    if (plugin)
+    {
+        std::filesystem::path bundlePath = plugin->bundle_path();
+
+        const auto& fileProperties = plugin->piPedalUI()->fileProperties();
+        if (fileProperties.size() != 0 && !pluginsThatHaveBeenCheckedForResources.contains(pluginUri))
+        {
+            pluginsThatHaveBeenCheckedForResources.insert(pluginUri);
+
+            // eliminate duplicates.
+            std::set<ResourceInfo> resourceInfoSet;
+            for (const auto&fileProperty: fileProperties)
+            {
+                if (!fileProperty->resourceDirectory().empty() && !fileProperty->directory().empty())
+                {
+                    resourceInfoSet.insert(ResourceInfo(fileProperty->directory(),fileProperty->resourceDirectory()));
+                }
+            }
+            for (const ResourceInfo&resourceInfo: resourceInfoSet)
+            {
+                std::filesystem::path sourcePath = bundlePath / resourceInfo.resourceDirectory;
+                std::filesystem::path targetPath = pluginUploadDirectory / resourceInfo.filePropertyDirectory;
+                if (!anyTargetFilesExist(sourcePath,targetPath))
+                {
+                    createTargetLinks(sourcePath,targetPath);
+                }
+
+            }
+        }
+    }
+}
 // void PiPedalHostLogError(const std::string &error)
 // {
 //     Lv2Log::error("%s",error.c_str());

src/PluginHost.hpp

@@ -30,6 +30,7 @@
 #include <cmath>
 #include <string>
 #include "IHost.hpp"
+#include <set>
 
 //#include "lv2.h"
 #include "Units.hpp"
@@ -685,7 +686,7 @@ namespace pipedal
             AutoLilvNode pipedalUI__patchProperty;
 
             AutoLilvNode pipedalUI__fileProperty;
-
+            AutoLilvNode pipedalUI__resourceDirectory;
             AutoLilvNode pipedalUI__fileTypes;
             AutoLilvNode pipedalUI__fileExtension;
             AutoLilvNode pipedalUI__mimeType;
@@ -811,8 +812,14 @@ namespace pipedal
 
     public:
         virtual MapFeature &GetMapFeature() { return this->mapFeature; }
+        void CheckForResourceInitialization(const std::string& pluginUri,const std::filesystem::path& pluginUploadDirectory);
 
     private:
+
+        std::set<std::string> pluginsThatHaveBeenCheckedForResources;
+
+        void CheckForResourceInitization(const std::string pluginUri);
+
         virtual LV2_URID_Map *GetLv2UridMap()
         {
             return this->mapFeature.GetMap();

src/Storage.cpp

@@ -266,7 +266,7 @@ std::filesystem::path Storage::GetPluginPresetsDirectory() const
 {
     return this->dataRoot / "plugin_presets";
 }
-std::filesystem::path Storage::GetPluginAudioFileDirectory() const
+std::filesystem::path Storage::GetPluginUploadDirectory() const
 {
     return this->dataRoot / "audio_uploads";
 }
@@ -1498,7 +1498,7 @@ std::vector<std::string> Storage::GetFileList(const UiFileProperty &fileProperty
     // if fileProperty has a user-accessible directory, push the entire file path.
     if (fileProperty.directory().size() != 0)
     {
-        std::filesystem::path audioFileDirectory = this->GetPluginAudioFileDirectory() / fileProperty.directory();
+        std::filesystem::path audioFileDirectory = this->GetPluginUploadDirectory() / fileProperty.directory();
         try
         {
             for (auto const &dir_entry : std::filesystem::directory_iterator(audioFileDirectory))
@@ -1576,7 +1576,7 @@ std::vector<FileEntry> Storage::GetFileList2(const std::string &relativePath,con
     // if fileProperty has a user-accessible directory, push the entire file path.
     if (fileProperty.directory().size() != 0)
     {
-        std::filesystem::path audioFileDirectory = this->GetPluginAudioFileDirectory() / fileProperty.directory() / relativePath;
+        std::filesystem::path audioFileDirectory = this->GetPluginUploadDirectory() / fileProperty.directory() / relativePath;
         try
         {
             for (auto const &dir_entry : std::filesystem::directory_iterator(audioFileDirectory))
@@ -1629,7 +1629,7 @@ bool Storage::IsValidSampleFileName(const std::filesystem::path &fileName)
         return false;
     }
     
-    std::filesystem::path audioFilePath = this->GetPluginAudioFileDirectory();
+    std::filesystem::path audioFilePath = this->GetPluginUploadDirectory();
 
     std::filesystem::path parentDirectory = fileName.parent_path();
     while (true)
@@ -1688,7 +1688,7 @@ std::filesystem::path Storage::MakeUserFilePath(const std::string &directory, co
     {
         throw std::logic_error("Permission denied.");
     }
-    std::filesystem::path result = this->GetPluginAudioFileDirectory() / directory / filename;
+    std::filesystem::path result = this->GetPluginUploadDirectory() / directory / filename;
     if (!this->IsValidSampleFileName(result))
     {
         throw std::logic_error("Permission denied.");
@@ -1753,7 +1753,7 @@ std::string Storage::CreateNewSampleDirectory(const std::string&relativePath, co
     {
         throw std::runtime_error("Invalid UI File Property.");
     }
-    std::filesystem::path path = this->GetPluginAudioFileDirectory() / uiFileProperty.directory() / relativePath;
+    std::filesystem::path path = this->GetPluginUploadDirectory() / uiFileProperty.directory() / relativePath;
     if (!this->IsValidSampleFileName(path))
     {
         throw std::runtime_error("Invalid file name.");
@@ -1775,7 +1775,7 @@ std::string Storage::RenameFilePropertyFile(
     {
         throw std::runtime_error("Invalid UI File Property.");
     }
-    std::filesystem::path oldPath = this->GetPluginAudioFileDirectory() / uiFileProperty.directory() / oldRelativePath;
+    std::filesystem::path oldPath = this->GetPluginUploadDirectory() / uiFileProperty.directory() / oldRelativePath;
     if (!this->IsValidSampleFileName(oldPath))
     {
         throw std::runtime_error("Invalid file name.");
@@ -1785,7 +1785,7 @@ std::string Storage::RenameFilePropertyFile(
         throw std::runtime_error("Original path does not exist.");
     }
 
-    std::filesystem::path newPath = this->GetPluginAudioFileDirectory() / uiFileProperty.directory() / newRelativePath;
+    std::filesystem::path newPath = this->GetPluginUploadDirectory() / uiFileProperty.directory() / newRelativePath;
     if (!this->IsValidSampleFileName(newPath))
     {
         throw std::runtime_error("Invalid file name.");
@@ -1834,7 +1834,7 @@ FilePropertyDirectoryTree::ptr Storage::GetFilePropertydirectoryTree(const UiFil
     {
         throw std::runtime_error("Invalid uiFileProperty");
     }
-    std::filesystem::path rootDirectory = this->GetPluginAudioFileDirectory() / uiFileProperty.directory();
+    std::filesystem::path rootDirectory = this->GetPluginUploadDirectory() / uiFileProperty.directory();
 
     FillSampleDirectoryTree(result.get(),rootDirectory);
 

src/Storage.hpp

@@ -112,7 +112,7 @@ public:
     void SetDataRoot(const std::filesystem::path& path);
     void SetConfigRoot(const std::filesystem::path& path);
 
-    std::filesystem::path GetPluginAudioFileDirectory() const;
+    std::filesystem::path GetPluginUploadDirectory() const;
 
     std::vector<std::string> GetPedalboards();
 
@@ -225,7 +225,6 @@ public:
         const std::string&newRelativePath,
         const UiFileProperty&uiFileProperty);
     FilePropertyDirectoryTree::ptr GetFilePropertydirectoryTree(const UiFileProperty&uiFileProperty) const;
-
 };
 
 

src/WebServer.cpp

@@ -78,6 +78,8 @@ public:
         return m_ready;
     }
     std::istream&get_body_input_stream();
+
+    const std::filesystem::path& get_body_input_file();
     size_t content_length() const  { return m_content_length;}
 
     /// Returns the full raw request (including the body)
@@ -139,13 +141,19 @@ public:
     size_t m_content_length;
     bool m_ready;
     bool m_uploading_to_file = false;
-    size_t m_max_in_memory_upload = 1 ; // 1 MiB
+    size_t m_max_in_memory_upload = 0; // saver to have one code path. And any input body is going to be an upload anyway.
     std::shared_ptr<TemporaryFile> m_temporaryFile;
     std::ofstream m_outputStream;
     std::ifstream m_inputStream;
     std::stringstream m_stringInputStream;
     bool m_outputOpen = false;
 };
+const std::filesystem::path& request_with_file_upload::get_body_input_file() 
+{
+    if (!this->m_temporaryFile)
+        throw std::runtime_error("Request does not have a body.");
+    return this->m_temporaryFile->Path();
+}
 std::istream&request_with_file_upload::get_body_input_stream()
 {
     if (!m_outputOpen)
@@ -649,8 +657,11 @@ namespace pipedal
             {
             }
 
-            virtual const std::string &body() const { return m_request.get_body(); }
+            virtual std::istream &get_body_input_stream() override { return m_request.get_body_input_stream(); }
-            virtual std::istream &get_body_input_stream() { return m_request.get_body_input_stream(); }
+            virtual const std::filesystem::path& get_body_temporary_file()override {
+                return m_request.get_body_input_file();
+            } 
+
             virtual size_t content_length() const { return m_request.content_length(); }
 
 

src/WebServer.hpp

@@ -28,6 +28,7 @@ class  HttpRequest {
 public:
     //virtual const std::string&body() const = 0;
     virtual std::istream &get_body_input_stream() = 0;
+    virtual const std::filesystem::path& get_body_temporary_file() = 0;
     virtual size_t content_length() const = 0;
     virtual const std::string &method() const = 0;
     virtual const std::string&get(const std::string&key) const = 0;

src/ZipFile.cpp

@@ -0,0 +1,133 @@
+// Copyright (c) 2024 Robin Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#include "ZipFile.hpp"
+#include "zip.h"
+#include <stdexcept>
+#include <map>
+#include "ss.hpp"
+#include "Finally.hpp"
+
+using namespace pipedal;
+
+ZipFile::ZipFile()
+{
+
+}
+ZipFile::~ZipFile()
+{
+
+}
+
+class ZipFileImpl: public ZipFile {
+public:
+    ZipFileImpl(const std::filesystem::path&path) 
+    :path(path)
+    {
+        int errorOp = 0;
+        zipFile = zip_open(path.c_str(),ZIP_RDONLY,&errorOp);
+        if (zipFile == nullptr)
+        {
+            throw std::runtime_error("Can't open zip file.");
+        }
+    }
+    virtual ~ZipFileImpl();
+    virtual std::vector<std::string> GetFiles() override;
+    virtual void ExtractTo(const std::string &zipName, const std::filesystem::path& path) override;
+
+private:
+    std::map<std::string,zip_int64_t> nameMap; // avoid o(2) extraction operations.
+    const std::filesystem::path path;
+    zip_t*zipFile = nullptr;
+};
+
+ZipFile::ptr ZipFile::Create(const std::filesystem::path &path)
+{
+    return std::shared_ptr<ZipFile>(new ZipFileImpl(path));
+}
+ZipFileImpl::~ZipFileImpl()
+{
+    if (zipFile)
+    {
+        zip_close(zipFile);
+        zipFile = nullptr;
+    }
+}
+
+std::vector<std::string> ZipFileImpl::GetFiles() 
+{
+    std::vector<std::string> result;
+    zip_int64_t nEntries = zip_get_num_entries(zipFile,0);
+    for (zip_int64_t i = 0; i < nEntries; ++i)
+    {
+        const char*name = zip_get_name(zipFile,i,ZIP_FL_ENC_STRICT);
+        if (name)
+        {
+            result.push_back(name);
+            nameMap[name] = i;
+        }
+    }
+    return result;
+}
+
+void ZipFileImpl::ExtractTo(const std::string &zipName, const std::filesystem::path& path) {
+    auto fi = nameMap.find(zipName);
+    
+    if (fi == nameMap.end())
+    {
+        // must call GetFiles() firest.
+        throw std::runtime_error("Zip content file not found.");
+    }
+    zip_int64_t fileIndex = fi->second;
+
+    zip_file_t*fIn = zip_fopen_index(this->zipFile,fileIndex,0);
+    if (fIn == nullptr)
+    {
+        zip_error_t *error  = zip_get_error(this->zipFile);
+        const char *strError = zip_error_strerror(error);
+
+        throw std::runtime_error(SS("Failed to read zip content file. " << strError));
+    }
+    Finally t{[fIn] () mutable{
+        zip_fclose(fIn);
+    }};
+
+    std::ofstream fo {path,std::ios_base::out |std::ios_base::trunc | std::ios_base::binary};
+
+    constexpr int BUFFER_SIZE = 64*1024;
+    std::vector<char> vBuff(BUFFER_SIZE);
+    char*pBuff = (char*)&(vBuff[0]);
+    while (true)
+    {
+        zip_int64_t nRead = zip_fread(fIn,pBuff, BUFFER_SIZE);
+        if (nRead = 0) break;
+        if (nRead == -1) {
+            zip_error_t*error = zip_file_get_error(fIn);
+            const char *strError = zip_error_strerror(error);
+            throw std::runtime_error(SS("Error reading zip content file." << strError));
+        }
+        fo.write(pBuff,(std::streamsize)nRead);
+        if (!fo) {
+            throw std::runtime_error(SS("Unable to write to " << path));
+        }
+    }
+
+
+
+}

src/ZipFile.hpp

@@ -0,0 +1,41 @@
+// Copyright (c) 2024 Robin Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#pragma once
+#include <memory>
+#include <string>
+#include <vector>
+#include <filesystem>
+namespace pipedal {
+    class ZipFile {
+    protected:
+        ZipFile();
+
+    public:
+        using ptr = std::shared_ptr<ZipFile>;
+        static ptr Create(const std::filesystem::path &path);
+        ZipFile(const ZipFile&) = delete;
+        ZipFile&operator=(const ZipFile&) = delete;
+        virtual ~ZipFile();
+
+        virtual std::vector<std::string> GetFiles() = 0;
+        virtual void ExtractTo(const std::string &zipName, const std::filesystem::path& path) = 0;
+
+    };
+}