Thred priorities on no-PREEMPT_RT (Ubunutu)

.vscode/c_cpp_properties.json

@@ -7,6 +7,7 @@
                 "${workspaceFolder}",
                 "${workspaceFolder}/src",
                 "/usr/include/lilv-0",
+                "/usr/include/x86_64-linux-gnu",
                 "/usr/lib",
                 "~/src/vst3sdk"
             ],
@@ -16,6 +17,24 @@
             "intelliSenseMode": "linux-gcc-arm64",
             "compileCommands": "${workspaceFolder}/build/compile_commands.json",
             "configurationProvider": "ms-vscode.cmake-tools"
+        },
+        {
+            "name": "Linux x86_64",
+            "includePath": [
+                "${default}",
+                "${workspaceFolder}",
+                "${workspaceFolder}/src",
+                "/usr/include/x86_64-linux-gnu",
+                "/usr/include/lilv-0",
+                "/usr/lib"
+            ],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c17",
+            "cppStandard": "c++20",
+            "intelliSenseMode": "linux-gcc-x64",
+            "compileCommands": "${workspaceFolder}/build/compile_commands.json",
+            "configurationProvider": "ms-vscode.cmake-tools",
+            "mergeConfigurations": true
         }
     ],
     "version": 4

PiPedalCommon/src/CMakeLists.txt

@@ -27,12 +27,12 @@ endif()
 # endif()
 
 # nlgenl-3 library.
-execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET)
+# execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET)
-if(LNL3_MISSING)
+# if(LNL3_MISSING)
-    message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ")
+#     message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ")
-endif()
+# endif()
-set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3)
+# set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3)
-set(LIBNL3_LIBRARIES nl-3 nl-genl-3)
+# set(LIBNL3_LIBRARIES nl-3 nl-genl-3)
 
 
 

src/AlsaDriver.cpp

@@ -34,6 +34,7 @@
 #include "RtInversionGuard.hpp"
 #include "PiPedalException.hpp"
 #include "DummyAudioDriver.hpp"
+#include "SchedulerPriority.hpp"
 
 #include "CpuUse.hpp"
 
@@ -1565,32 +1566,7 @@ namespace pipedal
             SetThreadName("alsaDriver");
             try
             {
-#if defined(__WIN32)
+                SetThreadPriority(SchedulerPriority::RealtimeAudio);
-                // bump thread prioriy two levels to
-                // ensure that the service thread doesn't
-                // get bogged down by UIwork. Doesn't have to be realtime, but it
-                // MUST run at higher priority than UI threads.
-                xxx; // TO DO.
-#elif defined(__linux__)
-                int min = sched_get_priority_min(SCHED_RR);
-                int max = sched_get_priority_max(SCHED_RR);
-
-                struct sched_param param;
-                memset(&param, 0, sizeof(param));
-                param.sched_priority = RT_THREAD_PRIORITY;
-
-                int result = sched_setscheduler(0, SCHED_RR, &param);
-                if (result == 0)
-                {
-                    Lv2Log::debug("Service thread priority successfully boosted.");
-                }
-                else
-                {
-                    Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")"));
-                }
-#else
-                xxx; // TODO!
-#endif
 
                 bool ok = true;
 

src/AudioHost.cpp

@@ -20,9 +20,11 @@
 #include "AudioHost.hpp"
 #include "util.hpp"
 #include <lv2/atom/atom.h>
+#include "SchedulerPriority.hpp"
 
 #include "Lv2Log.hpp"
 
+#include "SchedulerPriority.hpp"
 #include "JackDriver.hpp"
 #include "AlsaDriver.hpp"
 #include "DummyAudioDriver.hpp"
@@ -1263,30 +1265,9 @@ public:
     bool terminateThread;
     void ThreadProc()
     {
-
-#if defined(__WIN32)
-        // bump thread prioriy two levels to
-        // ensure that the service thread doesn't
-        // get bogged down by UIwork. Doesn't have to be realtime, but it
-        // MUST run at higher _ than UI threads.
-        xxx; // TO DO.
-#elif defined(__linux__)
-        int min = sched_get_priority_min(SCHED_RR);
-        int max = sched_get_priority_max(SCHED_RR);
-
-        struct sched_param param;
-        memset(&param, 0, sizeof(param));
-        param.sched_priority = min;
-
-        int result = sched_setscheduler(0, SCHED_RR, &param);
-        if (result == 0)
-        {
-            Lv2Log::debug("Service thread priority successfully boosted.");
-        }
         SetThreadName("rtsvc");
-#else
+        SetThreadPriority(SchedulerPriority::AudioService);
-        xxx; // TODO!
+        
-#endif
         int underrunMessagesGiven = 0;
         try
         {

src/CMakeLists.txt

@@ -87,13 +87,14 @@ set (LV2DEV_INCLUDE_DIRS ) # use lvt headers fro /usr/include
 
 set (WEBSOCKETPP_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/modules/websocketpp) 
 message(STATUS "WEBSOCKETPP_INCLUDE_DIRS ${WEBSOCKETPP_INCLUDE_DIRS}" )
-pkg_check_modules(JACK "jack")
+
-if(!JACK_FOUND)
+# pkg_check_modules(JACK "jack")
-  message(ERROR "jack package not installed.")
+# if(!JACK_FOUND)
-else()
+#   message(ERROR "jack package not installed.")
-  message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}")
+# else()
-  message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}")
+#   message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}")
-endif()
+#   message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}")
+# endif()
 
 
 # specify the C++ standard
@@ -157,6 +158,7 @@ else()
 endif()
 
 set (PIPEDAL_SOURCES
+    SchedulerPriority.hpp SchedulerPriority.cpp
     ModFileTypes.cpp ModFileTypes.hpp
     PatchPropertyWriter.hpp
     PresetBundle.cpp PresetBundle.hpp
@@ -659,6 +661,7 @@ add_executable(pipedal_latency_test
     PiPedalAlsa.hpp PiPedalAlsa.cpp
     asan_options.cpp
     AlsaDriver.cpp AlsaDriver.hpp
+    SchedulerPriority.cpp SchedulerPriority.hpp
     DummyAudioDriver.cpp DummyAudioDriver.hpp
     JackConfiguration.hpp JackConfiguration.cpp
     JackServerSettings.hpp JackServerSettings.cpp

src/ConfigMain.cpp

@@ -483,17 +483,25 @@ void InstallLimits()
         throw std::runtime_error("Failed to create audio service group.");
     }
 
-    fs::path limitsConfig = "/etc/security/limits.d/audio.conf";
+    // old limits config file bumped limits for members of the audio group, and didn't have a proper priority.
+    // remove it
+    fs::path oldLimitsConfig = "/etc/security/limits.d/audio.conf";
+    if (fs::exists(oldLimitsConfig))
+    {
+        fs::remove(oldLimitsConfig);
+    }
+
+    // bump limits for members of the pipedal_d group.
+    fs::path limitsConfig = "/etc/security/limits.d/90-pipedal.conf";
 
     if (!fs::exists(limitsConfig))
     {
         ofstream output(limitsConfig);
-        output << "# Realtime priority group used by pipedal audio (and also by jack)"
+        output << "# Realtime priority limits for the pipedal services"
-               << "\n";
-        output << "@audio   -  rtprio     95"
-               << "\n";
-        output << "@audio   -  memlock    unlimited"
                << "\n";
+        output << "@pipedal_d   -  rtprio     95" << "\n";
+        output << "@pipedal_d   -  nice     -20" << "\n";
+        output << "@pipedal_d   -  memlock    unlimited" << "\n";
     }
 }
 

src/DummyAudioDriver.cpp

@@ -37,6 +37,7 @@
 #include <thread>
 #include <stdexcept>
 #include "ss.hpp"
+#include "SchedulerPriority.hpp"
 
 #include "CpuUse.hpp"
 
@@ -223,32 +224,8 @@ namespace pipedal
             SetThreadName("dummyAudioDriver");
             try
             {
-#if defined(__WIN32)
-                // bump thread prioriy two levels to
-                // ensure that the service thread doesn't
-                // get bogged down by UIwork. Doesn't have to be realtime, but it
-                // MUST run at higher priority than UI threads.
-                xxx; // TO DO.
-#elif defined(__linux__)
-                int min = sched_get_priority_min(SCHED_RR);
-                int max = sched_get_priority_max(SCHED_RR);
 
-                struct sched_param param;
+                SetThreadPriority(SchedulerPriority::RealtimeAudio);
-                memset(&param, 0, sizeof(param));
-                param.sched_priority = RT_THREAD_PRIORITY;
-
-                int result = sched_setscheduler(0, SCHED_RR, &param);
-                if (result == 0)
-                {
-                    Lv2Log::debug("Service thread priority successfully boosted.");
-                }
-                else
-                {
-                    Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")"));
-                }
-#else
-                xxx; // TODO for your platform.
-#endif
 
                 bool ok = true;
 

src/JackDriver.cpp

@@ -25,13 +25,23 @@
 /* Status of Jack support.
 
 PiPedal was originally written for Jack, but subsequently ported to Alsa because running
-Jack as a systemd daemon proved to be unsupportable,. 
+Jack as a systemd daemon proved to be unsupportable. (Completely broken when pipewire is 
+installed).
 
+This was all functional code at one point, but is no longer in sync with the current 
+codebase. It won't work as is. It probably implemenets the driver interface properly. 
+You would have to go through AudioHost.cpp and substitute JackDriver for AlsaDriver 
+as appropriate.
 
 
 */
 
+
+
+#if JACK_HOST
+
 #include "pch.h"
+
 #include "JackDriver.hpp"
 
 #include <jack/jack.h>
@@ -40,8 +50,6 @@ Jack as a systemd daemon proved to be unsupportable,.
 #include <jack/midiport.h>
 #include "Lv2Log.hpp"
 
-#if JACK_HOST
-
 namespace pipedal {
 
 

src/PiPedalModel.cpp

@@ -299,11 +299,6 @@ void PiPedalModel::Load()
 #endif
     }
 
-    struct sched_param scheduler_params;
-    scheduler_params.sched_priority = 10;
-    memset(&scheduler_params, 0, sizeof(sched_param));
-    sched_setscheduler(0, SCHED_RR, &scheduler_params);
-
     RestartAudio();
 }
 

src/RtInversionGuard.hpp

@@ -24,6 +24,8 @@
 
 #pragma once
 
+#ifdef JUNK //yyx delete me.
+
 #include "ss.hpp"
 #include "Lv2Log.hpp"
 
@@ -66,4 +68,6 @@ namespace pipedal
             sched_setscheduler(0, currentScheduler, &currentPriority);
         }
     };
-}
+}
+
+#endif

src/SchedulerPriority.cpp

@@ -0,0 +1,147 @@
+// Copyright (c) 2024 Robin E. R. Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#include "SchedulerPriority.hpp"
+#include "Lv2Log.hpp"
+#include "memory.h"
+#include "sched.h"
+#include "ss.hpp"
+#include <stdexcept>
+
+#ifdef __linux__
+#include "unistd.h" // for nice()`
+#endif
+
+using namespace pipedal;
+
+static constexpr int EXPECTED_RT_THREAD_PRIORITY_MAX = 81;
+
+static constexpr int RT_AUDIO_THREAD_PRIORITY = 80;
+static constexpr int NICE_AUDIO_THREAD_PRIORITY = -19;
+
+static constexpr int RT_AUDIOSERVICE_THREAD_PRIORITY = 10;
+static constexpr int NICE_AUDIOSERVICE_THREAD_PRIORITY = -17;
+
+static constexpr int RT_LV2SCHEDULER_THREAD_PRIORITY = -1;
+static constexpr int NICE_LV2SCHEDULER_THREAD_PRIORITY = 2; // nice(2): don't let plugins screw things up by pinning the CPU 
+
+static constexpr int RT_WEBSERVER_THREAD_PRIORITY = -1;
+static constexpr int NICE_WEBSERVER_THREAD_PRIORITY = -2; // don't get bogged down by UI threads.
+
+bool pipedal::IsRtPreemptKernel(SchedulerPriority priority)
+{
+    #ifdef __linux__
+        struct oldPolicy;
+        struct sched_param oldParam = {0};
+
+        struct sched_param param = {0};
+
+        auto oldPolicy = sched_getscheduler(0);
+        if (sched_getparam(0,&oldParam) == -1)
+        {
+            return false;
+        }
+        auto priorityMin = sched_get_priority_min(SCHED_RR);
+        param.sched_priority = priorityMin;
+        if (sched_setscheduler(0,SCHED_RR,&param) == -1)
+        {
+            return false;
+        }
+        sched_setscheduler(0,oldPolicy,&oldParam);
+        return true;
+
+    #else
+        return true; // let SetPrority sort out whether it can or can't.
+    #endif
+}
+
+static void SetPriority(int realtimePriority, int nicePriority, const char *priorityName)
+{
+
+    if (realtimePriority != -1)
+    {
+        int initialPriority = realtimePriority;
+        int min = sched_get_priority_min(SCHED_RR);
+        int max = sched_get_priority_max(SCHED_RR);
+        if (realtimePriority > max) {
+            realtimePriority = max + EXPECTED_RT_THREAD_PRIORITY_MAX-realtimePriority;
+        }
+        if (realtimePriority < min) {
+            realtimePriority = min;
+        }
+        if (initialPriority != realtimePriority)
+        {
+            Lv2Log::warning(
+                SS("Realtime priority adjusted from " << initialPriority << " to " << realtimePriority << ". (" << priorityName << ")")
+                );
+        }
+
+        struct sched_param param;
+        memset(&param, 0, sizeof(param));
+        param.sched_priority = realtimePriority;
+
+        int result = sched_setscheduler(0, SCHED_RR, &param);
+        if (result == 0)
+        {
+            Lv2Log::debug("Service thread priority successfully boosted.");
+            return;
+        }
+        else
+        {
+            Lv2Log::debug(SS("Failed to set RT thread priority for " << priorityName << " (" << strerror(result) << ")"));
+        }
+    }
+    // If the RT scheduler isn't working, fall back to using NICE priority.
+    int result = nice(nicePriority);
+    if (result == -1)
+    {
+        Lv2Log::error(SS("Failed Failed to set thread priority. (" << priorityName << ")"));
+    }
+}
+
+void pipedal::SetThreadPriority(SchedulerPriority priority)
+{
+
+#if defined(__linux__)
+    switch (priority)
+    {
+    case SchedulerPriority::RealtimeAudio:
+        SetPriority(RT_AUDIO_THREAD_PRIORITY, NICE_AUDIO_THREAD_PRIORITY, "RealtimeAudio");
+        break;
+    case SchedulerPriority::AudioService:
+        SetPriority(RT_AUDIOSERVICE_THREAD_PRIORITY, NICE_AUDIOSERVICE_THREAD_PRIORITY, "AudioService");
+        break;
+    case SchedulerPriority::Lv2Scheduler:
+        SetPriority(RT_LV2SCHEDULER_THREAD_PRIORITY, NICE_LV2SCHEDULER_THREAD_PRIORITY, "Lv2Scheduler");
+        break;
+    case SchedulerPriority::WebServerThread:
+        SetPriority(RT_WEBSERVER_THREAD_PRIORITY, NICE_WEBSERVER_THREAD_PRIORITY, "Lv2Scheduler");
+        break;
+    default:
+        Lv2Log::error("Invalid scheduler priority.");
+        throw std::runtime_error("Invalid value.");
+    }
+#elif defined(__WIN32)
+// Realtime thread priority must be set using MM scheduler.
+// Others must run with elevated priority, but not realtime priority (MUST run with higher priority than UI threads).
+#error "Realtime scheduling not implmented for WIN32"
+#else
+#error "Realtime scheduling not implmented for your platform"
+#endif
+}

src/SchedulerPriority.hpp

@@ -0,0 +1,33 @@
+// Copyright (c) 2024 Robin E. R. Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#pragma once
+
+namespace pipedal {
+    enum class SchedulerPriority {
+        RealtimeAudio, // the audio service thread.
+        AudioService, // non-realtime servicing of AudioThread responses.
+        Lv2Scheduler, // LV2 Scheduler service thread.
+        WebServerThread, // Web server threads.
+    };
+
+    bool IsRtPreemptKernel(SchedulerPriority priority);
+
+    void SetThreadPriority(SchedulerPriority priority);
+}

src/Worker.cpp

@@ -42,6 +42,7 @@
 #include <unistd.h> // for nice(
 #include <utility>
 #include "util.hpp"
+#include "SchedulerPriority.hpp"
 
 using namespace pipedal;
 
@@ -190,12 +191,8 @@ void HostWorkerThread::ThreadProc() noexcept
 {
     // run nice +2 (priority -2 on Windows)
     SetThreadName("lv2_worker");
-    errno = 0;
+    SetThreadPriority(SchedulerPriority::Lv2Scheduler);
-    std::ignore = nice(2);
+
-    if (errno != 0)
-    {
-        std::cout << "Warning: Unable to run Lv2 schedule thread at nice +1" << std::endl;
-    }
 
     try
     {

src/main.cpp

@@ -44,10 +44,11 @@
 
 #include <signal.h>
 #include <semaphore.h>
+#include "SchedulerPriority.hpp"
 
 #include <systemd/sd-daemon.h>
 
-using namespace pipedal;
+using namespace  pipedal;
 
 #ifdef __ARM_ARCH_ISA_A64
 #define AARCH64
@@ -244,6 +245,9 @@ int main(int argc, char *argv[])
     std::shared_ptr<WebServer> server;
     try
     {
+        // (web server threads inherit the main   thread priority)
+        SetThreadPriority(SchedulerPriority::WebServerThread);
+
         auto const address = boost::asio::ip::make_address(configuration.GetSocketServerAddress());
         port = static_cast<uint16_t>(configuration.GetSocketServerPort());