Thred priorities on no-PREEMPT_RT (Ubunutu)

2024-11-14 16:15:32 -05:00
parent a52a7a2a9a
commit c01f0351f0
14 changed files with 261 additions and 109 deletions
@@ -7,6 +7,7 @@
                "${workspaceFolder}",
                "${workspaceFolder}/src",
                "/usr/include/lilv-0",
+                "/usr/include/x86_64-linux-gnu",
                "/usr/lib",
                "~/src/vst3sdk"
            ],
@@ -16,6 +17,24 @@
            "intelliSenseMode": "linux-gcc-arm64",
            "compileCommands": "${workspaceFolder}/build/compile_commands.json",
            "configurationProvider": "ms-vscode.cmake-tools"
+        },
+        {
+            "name": "Linux x86_64",
+            "includePath": [
+                "${default}",
+                "${workspaceFolder}",
+                "${workspaceFolder}/src",
+                "/usr/include/x86_64-linux-gnu",
+                "/usr/include/lilv-0",
+                "/usr/lib"
+            ],
+            "compilerPath": "/usr/bin/gcc",
+            "cStandard": "c17",
+            "cppStandard": "c++20",
+            "intelliSenseMode": "linux-gcc-x64",
+            "compileCommands": "${workspaceFolder}/build/compile_commands.json",
+            "configurationProvider": "ms-vscode.cmake-tools",
+            "mergeConfigurations": true
        }
    ],
    "version": 4
@@ -27,12 +27,12 @@ endif()
 # endif()

 # nlgenl-3 library.
-execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET)
-if(LNL3_MISSING)
-    message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ")
-endif()
-set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3)
-set(LIBNL3_LIBRARIES nl-3 nl-genl-3)
+# execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET)
+# if(LNL3_MISSING)
+#     message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ")
+# endif()
+# set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3)
+# set(LIBNL3_LIBRARIES nl-3 nl-genl-3)



@@ -34,6 +34,7 @@
 #include "RtInversionGuard.hpp"
 #include "PiPedalException.hpp"
 #include "DummyAudioDriver.hpp"
+#include "SchedulerPriority.hpp"

 #include "CpuUse.hpp"

@@ -1565,32 +1566,7 @@ namespace pipedal
            SetThreadName("alsaDriver");
            try
            {
-#if defined(__WIN32)
-                // bump thread prioriy two levels to
-                // ensure that the service thread doesn't
-                // get bogged down by UIwork. Doesn't have to be realtime, but it
-                // MUST run at higher priority than UI threads.
-                xxx; // TO DO.
-#elif defined(__linux__)
-                int min = sched_get_priority_min(SCHED_RR);
-                int max = sched_get_priority_max(SCHED_RR);
-
-                struct sched_param param;
-                memset(&param, 0, sizeof(param));
-                param.sched_priority = RT_THREAD_PRIORITY;
-
-                int result = sched_setscheduler(0, SCHED_RR, &param);
-                if (result == 0)
-                {
-                    Lv2Log::debug("Service thread priority successfully boosted.");
-                }
-                else
-                {
-                    Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")"));
-                }
-#else
-                xxx; // TODO!
-#endif
+                SetThreadPriority(SchedulerPriority::RealtimeAudio);

                bool ok = true;

@@ -20,9 +20,11 @@
 #include "AudioHost.hpp"
 #include "util.hpp"
 #include <lv2/atom/atom.h>
+#include "SchedulerPriority.hpp"

 #include "Lv2Log.hpp"

+#include "SchedulerPriority.hpp"
 #include "JackDriver.hpp"
 #include "AlsaDriver.hpp"
 #include "DummyAudioDriver.hpp"
@@ -1263,30 +1265,9 @@ public:
    bool terminateThread;
    void ThreadProc()
    {
-
-#if defined(__WIN32)
-        // bump thread prioriy two levels to
-        // ensure that the service thread doesn't
-        // get bogged down by UIwork. Doesn't have to be realtime, but it
-        // MUST run at higher _ than UI threads.
-        xxx; // TO DO.
-#elif defined(__linux__)
-        int min = sched_get_priority_min(SCHED_RR);
-        int max = sched_get_priority_max(SCHED_RR);
-
-        struct sched_param param;
-        memset(&param, 0, sizeof(param));
-        param.sched_priority = min;
-
-        int result = sched_setscheduler(0, SCHED_RR, &param);
-        if (result == 0)
-        {
-            Lv2Log::debug("Service thread priority successfully boosted.");
-        }
        SetThreadName("rtsvc");
-#else
-        xxx; // TODO!
-#endif
+        SetThreadPriority(SchedulerPriority::AudioService);
+        
        int underrunMessagesGiven = 0;
        try
        {
@@ -87,13 +87,14 @@ set (LV2DEV_INCLUDE_DIRS ) # use lvt headers fro /usr/include

 set (WEBSOCKETPP_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/modules/websocketpp) 
 message(STATUS "WEBSOCKETPP_INCLUDE_DIRS ${WEBSOCKETPP_INCLUDE_DIRS}" )
-pkg_check_modules(JACK "jack")
-if(!JACK_FOUND)
-  message(ERROR "jack package not installed.")
-else()
-  message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}")
-  message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}")
-endif()
+
+# pkg_check_modules(JACK "jack")
+# if(!JACK_FOUND)
+#   message(ERROR "jack package not installed.")
+# else()
+#   message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}")
+#   message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}")
+# endif()


 # specify the C++ standard
@@ -157,6 +158,7 @@ else()
 endif()

 set (PIPEDAL_SOURCES
+    SchedulerPriority.hpp SchedulerPriority.cpp
    ModFileTypes.cpp ModFileTypes.hpp
    PatchPropertyWriter.hpp
    PresetBundle.cpp PresetBundle.hpp
@@ -659,6 +661,7 @@ add_executable(pipedal_latency_test
    PiPedalAlsa.hpp PiPedalAlsa.cpp
    asan_options.cpp
    AlsaDriver.cpp AlsaDriver.hpp
+    SchedulerPriority.cpp SchedulerPriority.hpp
    DummyAudioDriver.cpp DummyAudioDriver.hpp
    JackConfiguration.hpp JackConfiguration.cpp
    JackServerSettings.hpp JackServerSettings.cpp
@@ -483,17 +483,25 @@ void InstallLimits()
        throw std::runtime_error("Failed to create audio service group.");
    }

-    fs::path limitsConfig = "/etc/security/limits.d/audio.conf";
+    // old limits config file bumped limits for members of the audio group, and didn't have a proper priority.
+    // remove it
+    fs::path oldLimitsConfig = "/etc/security/limits.d/audio.conf";
+    if (fs::exists(oldLimitsConfig))
+    {
+        fs::remove(oldLimitsConfig);
+    }
+
+    // bump limits for members of the pipedal_d group.
+    fs::path limitsConfig = "/etc/security/limits.d/90-pipedal.conf";

    if (!fs::exists(limitsConfig))
    {
        ofstream output(limitsConfig);
-        output << "# Realtime priority group used by pipedal audio (and also by jack)"
-               << "\n";
-        output << "@audio   -  rtprio     95"
-               << "\n";
-        output << "@audio   -  memlock    unlimited"
+        output << "# Realtime priority limits for the pipedal services"
               << "\n";
+        output << "@pipedal_d   -  rtprio     95" << "\n";
+        output << "@pipedal_d   -  nice     -20" << "\n";
+        output << "@pipedal_d   -  memlock    unlimited" << "\n";
    }
 }

@@ -37,6 +37,7 @@
 #include <thread>
 #include <stdexcept>
 #include "ss.hpp"
+#include "SchedulerPriority.hpp"

 #include "CpuUse.hpp"

@@ -223,32 +224,8 @@ namespace pipedal
            SetThreadName("dummyAudioDriver");
            try
            {
-#if defined(__WIN32)
-                // bump thread prioriy two levels to
-                // ensure that the service thread doesn't
-                // get bogged down by UIwork. Doesn't have to be realtime, but it
-                // MUST run at higher priority than UI threads.
-                xxx; // TO DO.
-#elif defined(__linux__)
-                int min = sched_get_priority_min(SCHED_RR);
-                int max = sched_get_priority_max(SCHED_RR);

-                struct sched_param param;
-                memset(&param, 0, sizeof(param));
-                param.sched_priority = RT_THREAD_PRIORITY;
-
-                int result = sched_setscheduler(0, SCHED_RR, &param);
-                if (result == 0)
-                {
-                    Lv2Log::debug("Service thread priority successfully boosted.");
-                }
-                else
-                {
-                    Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")"));
-                }
-#else
-                xxx; // TODO for your platform.
-#endif
+                SetThreadPriority(SchedulerPriority::RealtimeAudio);

                bool ok = true;

@@ -25,13 +25,23 @@
 /* Status of Jack support.

 PiPedal was originally written for Jack, but subsequently ported to Alsa because running
-Jack as a systemd daemon proved to be unsupportable,. 
+Jack as a systemd daemon proved to be unsupportable. (Completely broken when pipewire is 
+installed).

+This was all functional code at one point, but is no longer in sync with the current 
+codebase. It won't work as is. It probably implemenets the driver interface properly. 
+You would have to go through AudioHost.cpp and substitute JackDriver for AlsaDriver 
+as appropriate.


 */

+
+
+#if JACK_HOST
+
 #include "pch.h"
+
 #include "JackDriver.hpp"

 #include <jack/jack.h>
@@ -40,8 +50,6 @@ Jack as a systemd daemon proved to be unsupportable,.
 #include <jack/midiport.h>
 #include "Lv2Log.hpp"

-#if JACK_HOST
-
 namespace pipedal {


@@ -299,11 +299,6 @@ void PiPedalModel::Load()
 #endif
    }

-    struct sched_param scheduler_params;
-    scheduler_params.sched_priority = 10;
-    memset(&scheduler_params, 0, sizeof(sched_param));
-    sched_setscheduler(0, SCHED_RR, &scheduler_params);
-
    RestartAudio();
 }

@@ -24,6 +24,8 @@

 #pragma once

+#ifdef JUNK //yyx delete me.
+
 #include "ss.hpp"
 #include "Lv2Log.hpp"

@@ -66,4 +68,6 @@ namespace pipedal
            sched_setscheduler(0, currentScheduler, &currentPriority);
        }
    };
-}
+}
+
+#endif
@@ -0,0 +1,147 @@
+// Copyright (c) 2024 Robin E. R. Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#include "SchedulerPriority.hpp"
+#include "Lv2Log.hpp"
+#include "memory.h"
+#include "sched.h"
+#include "ss.hpp"
+#include <stdexcept>
+
+#ifdef __linux__
+#include "unistd.h" // for nice()`
+#endif
+
+using namespace pipedal;
+
+static constexpr int EXPECTED_RT_THREAD_PRIORITY_MAX = 81;
+
+static constexpr int RT_AUDIO_THREAD_PRIORITY = 80;
+static constexpr int NICE_AUDIO_THREAD_PRIORITY = -19;
+
+static constexpr int RT_AUDIOSERVICE_THREAD_PRIORITY = 10;
+static constexpr int NICE_AUDIOSERVICE_THREAD_PRIORITY = -17;
+
+static constexpr int RT_LV2SCHEDULER_THREAD_PRIORITY = -1;
+static constexpr int NICE_LV2SCHEDULER_THREAD_PRIORITY = 2; // nice(2): don't let plugins screw things up by pinning the CPU 
+
+static constexpr int RT_WEBSERVER_THREAD_PRIORITY = -1;
+static constexpr int NICE_WEBSERVER_THREAD_PRIORITY = -2; // don't get bogged down by UI threads.
+
+bool pipedal::IsRtPreemptKernel(SchedulerPriority priority)
+{
+    #ifdef __linux__
+        struct oldPolicy;
+        struct sched_param oldParam = {0};
+
+        struct sched_param param = {0};
+
+        auto oldPolicy = sched_getscheduler(0);
+        if (sched_getparam(0,&oldParam) == -1)
+        {
+            return false;
+        }
+        auto priorityMin = sched_get_priority_min(SCHED_RR);
+        param.sched_priority = priorityMin;
+        if (sched_setscheduler(0,SCHED_RR,&param) == -1)
+        {
+            return false;
+        }
+        sched_setscheduler(0,oldPolicy,&oldParam);
+        return true;
+
+    #else
+        return true; // let SetPrority sort out whether it can or can't.
+    #endif
+}
+
+static void SetPriority(int realtimePriority, int nicePriority, const char *priorityName)
+{
+
+    if (realtimePriority != -1)
+    {
+        int initialPriority = realtimePriority;
+        int min = sched_get_priority_min(SCHED_RR);
+        int max = sched_get_priority_max(SCHED_RR);
+        if (realtimePriority > max) {
+            realtimePriority = max + EXPECTED_RT_THREAD_PRIORITY_MAX-realtimePriority;
+        }
+        if (realtimePriority < min) {
+            realtimePriority = min;
+        }
+        if (initialPriority != realtimePriority)
+        {
+            Lv2Log::warning(
+                SS("Realtime priority adjusted from " << initialPriority << " to " << realtimePriority << ". (" << priorityName << ")")
+                );
+        }
+
+        struct sched_param param;
+        memset(&param, 0, sizeof(param));
+        param.sched_priority = realtimePriority;
+
+        int result = sched_setscheduler(0, SCHED_RR, &param);
+        if (result == 0)
+        {
+            Lv2Log::debug("Service thread priority successfully boosted.");
+            return;
+        }
+        else
+        {
+            Lv2Log::debug(SS("Failed to set RT thread priority for " << priorityName << " (" << strerror(result) << ")"));
+        }
+    }
+    // If the RT scheduler isn't working, fall back to using NICE priority.
+    int result = nice(nicePriority);
+    if (result == -1)
+    {
+        Lv2Log::error(SS("Failed Failed to set thread priority. (" << priorityName << ")"));
+    }
+}
+
+void pipedal::SetThreadPriority(SchedulerPriority priority)
+{
+
+#if defined(__linux__)
+    switch (priority)
+    {
+    case SchedulerPriority::RealtimeAudio:
+        SetPriority(RT_AUDIO_THREAD_PRIORITY, NICE_AUDIO_THREAD_PRIORITY, "RealtimeAudio");
+        break;
+    case SchedulerPriority::AudioService:
+        SetPriority(RT_AUDIOSERVICE_THREAD_PRIORITY, NICE_AUDIOSERVICE_THREAD_PRIORITY, "AudioService");
+        break;
+    case SchedulerPriority::Lv2Scheduler:
+        SetPriority(RT_LV2SCHEDULER_THREAD_PRIORITY, NICE_LV2SCHEDULER_THREAD_PRIORITY, "Lv2Scheduler");
+        break;
+    case SchedulerPriority::WebServerThread:
+        SetPriority(RT_WEBSERVER_THREAD_PRIORITY, NICE_WEBSERVER_THREAD_PRIORITY, "Lv2Scheduler");
+        break;
+    default:
+        Lv2Log::error("Invalid scheduler priority.");
+        throw std::runtime_error("Invalid value.");
+    }
+#elif defined(__WIN32)
+// Realtime thread priority must be set using MM scheduler.
+// Others must run with elevated priority, but not realtime priority (MUST run with higher priority than UI threads).
+#error "Realtime scheduling not implmented for WIN32"
+#else
+#error "Realtime scheduling not implmented for your platform"
+#endif
+}
@@ -0,0 +1,33 @@
+// Copyright (c) 2024 Robin E. R. Davies
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of
+// this software and associated documentation files (the "Software"), to deal in
+// the Software without restriction, including without limitation the rights to
+// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+// the Software, and to permit persons to whom the Software is furnished to do so,
+// subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+#pragma once
+
+namespace pipedal {
+    enum class SchedulerPriority {
+        RealtimeAudio, // the audio service thread.
+        AudioService, // non-realtime servicing of AudioThread responses.
+        Lv2Scheduler, // LV2 Scheduler service thread.
+        WebServerThread, // Web server threads.
+    };
+
+    bool IsRtPreemptKernel(SchedulerPriority priority);
+
+    void SetThreadPriority(SchedulerPriority priority);
+}
@@ -42,6 +42,7 @@
 #include <unistd.h> // for nice(
 #include <utility>
 #include "util.hpp"
+#include "SchedulerPriority.hpp"

 using namespace pipedal;

@@ -190,12 +191,8 @@ void HostWorkerThread::ThreadProc() noexcept
 {
    // run nice +2 (priority -2 on Windows)
    SetThreadName("lv2_worker");
-    errno = 0;
-    std::ignore = nice(2);
-    if (errno != 0)
-    {
-        std::cout << "Warning: Unable to run Lv2 schedule thread at nice +1" << std::endl;
-    }
+    SetThreadPriority(SchedulerPriority::Lv2Scheduler);
+

    try
    {
@@ -44,10 +44,11 @@

 #include <signal.h>
 #include <semaphore.h>
+#include "SchedulerPriority.hpp"

 #include <systemd/sd-daemon.h>

-using namespace pipedal;
+using namespace  pipedal;

 #ifdef __ARM_ARCH_ISA_A64
 #define AARCH64
@@ -244,6 +245,9 @@ int main(int argc, char *argv[])
    std::shared_ptr<WebServer> server;
    try
    {
+        // (web server threads inherit the main   thread priority)
+        SetThreadPriority(SchedulerPriority::WebServerThread);
+
        auto const address = boost::asio::ip::make_address(configuration.GetSocketServerAddress());
        port = static_cast<uint16_t>(configuration.GetSocketServerPort());