From c01f0351f01d3029ff0e55605238aaac57685e26 Mon Sep 17 00:00:00 2001 From: "Robin E. R. Davies" Date: Thu, 14 Nov 2024 16:15:32 -0500 Subject: [PATCH] Thred priorities on no-PREEMPT_RT (Ubunutu) --- .vscode/c_cpp_properties.json | 19 ++++ PiPedalCommon/src/CMakeLists.txt | 12 +-- src/AlsaDriver.cpp | 28 +----- src/AudioHost.cpp | 27 +----- src/CMakeLists.txt | 17 ++-- src/ConfigMain.cpp | 20 +++-- src/DummyAudioDriver.cpp | 27 +----- src/JackDriver.cpp | 14 ++- src/PiPedalModel.cpp | 5 -- src/RtInversionGuard.hpp | 6 +- src/SchedulerPriority.cpp | 147 +++++++++++++++++++++++++++++++ src/SchedulerPriority.hpp | 33 +++++++ src/Worker.cpp | 9 +- src/main.cpp | 6 +- 14 files changed, 261 insertions(+), 109 deletions(-) create mode 100644 src/SchedulerPriority.cpp create mode 100644 src/SchedulerPriority.hpp diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 033e617..d7afa68 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -7,6 +7,7 @@ "${workspaceFolder}", "${workspaceFolder}/src", "/usr/include/lilv-0", + "/usr/include/x86_64-linux-gnu", "/usr/lib", "~/src/vst3sdk" ], @@ -16,6 +17,24 @@ "intelliSenseMode": "linux-gcc-arm64", "compileCommands": "${workspaceFolder}/build/compile_commands.json", "configurationProvider": "ms-vscode.cmake-tools" + }, + { + "name": "Linux x86_64", + "includePath": [ + "${default}", + "${workspaceFolder}", + "${workspaceFolder}/src", + "/usr/include/x86_64-linux-gnu", + "/usr/include/lilv-0", + "/usr/lib" + ], + "compilerPath": "/usr/bin/gcc", + "cStandard": "c17", + "cppStandard": "c++20", + "intelliSenseMode": "linux-gcc-x64", + "compileCommands": "${workspaceFolder}/build/compile_commands.json", + "configurationProvider": "ms-vscode.cmake-tools", + "mergeConfigurations": true } ], "version": 4 diff --git a/PiPedalCommon/src/CMakeLists.txt b/PiPedalCommon/src/CMakeLists.txt index 80e2108..3926a61 100644 --- a/PiPedalCommon/src/CMakeLists.txt +++ b/PiPedalCommon/src/CMakeLists.txt @@ -27,12 +27,12 @@ endif() # endif() # nlgenl-3 library. -execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET) -if(LNL3_MISSING) - message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ") -endif() -set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3) -set(LIBNL3_LIBRARIES nl-3 nl-genl-3) +# execute_process(COMMAND ls /usr/include/libnl3/netlink/netlink.h RESULT_VARIABLE LNL3_MISSING OUTPUT_QUIET ERROR_QUIET) +# if(LNL3_MISSING) +# message(ERROR " Need to: sudo apt install libnl-3-dev libnl-genl-3-dev ") +# endif() +# set(LIBNL3_INCLUDE_DIRS /usr/include/libnl3) +# set(LIBNL3_LIBRARIES nl-3 nl-genl-3) diff --git a/src/AlsaDriver.cpp b/src/AlsaDriver.cpp index e92382a..726c9b9 100644 --- a/src/AlsaDriver.cpp +++ b/src/AlsaDriver.cpp @@ -34,6 +34,7 @@ #include "RtInversionGuard.hpp" #include "PiPedalException.hpp" #include "DummyAudioDriver.hpp" +#include "SchedulerPriority.hpp" #include "CpuUse.hpp" @@ -1565,32 +1566,7 @@ namespace pipedal SetThreadName("alsaDriver"); try { -#if defined(__WIN32) - // bump thread prioriy two levels to - // ensure that the service thread doesn't - // get bogged down by UIwork. Doesn't have to be realtime, but it - // MUST run at higher priority than UI threads. - xxx; // TO DO. -#elif defined(__linux__) - int min = sched_get_priority_min(SCHED_RR); - int max = sched_get_priority_max(SCHED_RR); - - struct sched_param param; - memset(¶m, 0, sizeof(param)); - param.sched_priority = RT_THREAD_PRIORITY; - - int result = sched_setscheduler(0, SCHED_RR, ¶m); - if (result == 0) - { - Lv2Log::debug("Service thread priority successfully boosted."); - } - else - { - Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")")); - } -#else - xxx; // TODO! -#endif + SetThreadPriority(SchedulerPriority::RealtimeAudio); bool ok = true; diff --git a/src/AudioHost.cpp b/src/AudioHost.cpp index 3f5cce8..5c238aa 100644 --- a/src/AudioHost.cpp +++ b/src/AudioHost.cpp @@ -20,9 +20,11 @@ #include "AudioHost.hpp" #include "util.hpp" #include +#include "SchedulerPriority.hpp" #include "Lv2Log.hpp" +#include "SchedulerPriority.hpp" #include "JackDriver.hpp" #include "AlsaDriver.hpp" #include "DummyAudioDriver.hpp" @@ -1263,30 +1265,9 @@ public: bool terminateThread; void ThreadProc() { - -#if defined(__WIN32) - // bump thread prioriy two levels to - // ensure that the service thread doesn't - // get bogged down by UIwork. Doesn't have to be realtime, but it - // MUST run at higher _ than UI threads. - xxx; // TO DO. -#elif defined(__linux__) - int min = sched_get_priority_min(SCHED_RR); - int max = sched_get_priority_max(SCHED_RR); - - struct sched_param param; - memset(¶m, 0, sizeof(param)); - param.sched_priority = min; - - int result = sched_setscheduler(0, SCHED_RR, ¶m); - if (result == 0) - { - Lv2Log::debug("Service thread priority successfully boosted."); - } SetThreadName("rtsvc"); -#else - xxx; // TODO! -#endif + SetThreadPriority(SchedulerPriority::AudioService); + int underrunMessagesGiven = 0; try { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2ccb744..ff6b328 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,13 +87,14 @@ set (LV2DEV_INCLUDE_DIRS ) # use lvt headers fro /usr/include set (WEBSOCKETPP_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/modules/websocketpp) message(STATUS "WEBSOCKETPP_INCLUDE_DIRS ${WEBSOCKETPP_INCLUDE_DIRS}" ) -pkg_check_modules(JACK "jack") -if(!JACK_FOUND) - message(ERROR "jack package not installed.") -else() - message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}") - message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}") -endif() + +# pkg_check_modules(JACK "jack") +# if(!JACK_FOUND) +# message(ERROR "jack package not installed.") +# else() +# message(STATUS "JACK_LIBRARIES: ${JACK_LIBRARIES}") +# message(STATUS "JACK_INCLUDE_DIRS: ${JACK_INCLUDE_DIRS}") +# endif() # specify the C++ standard @@ -157,6 +158,7 @@ else() endif() set (PIPEDAL_SOURCES + SchedulerPriority.hpp SchedulerPriority.cpp ModFileTypes.cpp ModFileTypes.hpp PatchPropertyWriter.hpp PresetBundle.cpp PresetBundle.hpp @@ -659,6 +661,7 @@ add_executable(pipedal_latency_test PiPedalAlsa.hpp PiPedalAlsa.cpp asan_options.cpp AlsaDriver.cpp AlsaDriver.hpp + SchedulerPriority.cpp SchedulerPriority.hpp DummyAudioDriver.cpp DummyAudioDriver.hpp JackConfiguration.hpp JackConfiguration.cpp JackServerSettings.hpp JackServerSettings.cpp diff --git a/src/ConfigMain.cpp b/src/ConfigMain.cpp index ffb809e..12d6bf9 100644 --- a/src/ConfigMain.cpp +++ b/src/ConfigMain.cpp @@ -483,17 +483,25 @@ void InstallLimits() throw std::runtime_error("Failed to create audio service group."); } - fs::path limitsConfig = "/etc/security/limits.d/audio.conf"; + // old limits config file bumped limits for members of the audio group, and didn't have a proper priority. + // remove it + fs::path oldLimitsConfig = "/etc/security/limits.d/audio.conf"; + if (fs::exists(oldLimitsConfig)) + { + fs::remove(oldLimitsConfig); + } + + // bump limits for members of the pipedal_d group. + fs::path limitsConfig = "/etc/security/limits.d/90-pipedal.conf"; if (!fs::exists(limitsConfig)) { ofstream output(limitsConfig); - output << "# Realtime priority group used by pipedal audio (and also by jack)" - << "\n"; - output << "@audio - rtprio 95" - << "\n"; - output << "@audio - memlock unlimited" + output << "# Realtime priority limits for the pipedal services" << "\n"; + output << "@pipedal_d - rtprio 95" << "\n"; + output << "@pipedal_d - nice -20" << "\n"; + output << "@pipedal_d - memlock unlimited" << "\n"; } } diff --git a/src/DummyAudioDriver.cpp b/src/DummyAudioDriver.cpp index 8353c8b..a4b58c5 100644 --- a/src/DummyAudioDriver.cpp +++ b/src/DummyAudioDriver.cpp @@ -37,6 +37,7 @@ #include #include #include "ss.hpp" +#include "SchedulerPriority.hpp" #include "CpuUse.hpp" @@ -223,32 +224,8 @@ namespace pipedal SetThreadName("dummyAudioDriver"); try { -#if defined(__WIN32) - // bump thread prioriy two levels to - // ensure that the service thread doesn't - // get bogged down by UIwork. Doesn't have to be realtime, but it - // MUST run at higher priority than UI threads. - xxx; // TO DO. -#elif defined(__linux__) - int min = sched_get_priority_min(SCHED_RR); - int max = sched_get_priority_max(SCHED_RR); - struct sched_param param; - memset(¶m, 0, sizeof(param)); - param.sched_priority = RT_THREAD_PRIORITY; - - int result = sched_setscheduler(0, SCHED_RR, ¶m); - if (result == 0) - { - Lv2Log::debug("Service thread priority successfully boosted."); - } - else - { - Lv2Log::error(SS("Failed to set ALSA AudioThread priority. (" << strerror(result) << ")")); - } -#else - xxx; // TODO for your platform. -#endif + SetThreadPriority(SchedulerPriority::RealtimeAudio); bool ok = true; diff --git a/src/JackDriver.cpp b/src/JackDriver.cpp index d0b4764..e26880f 100644 --- a/src/JackDriver.cpp +++ b/src/JackDriver.cpp @@ -25,13 +25,23 @@ /* Status of Jack support. PiPedal was originally written for Jack, but subsequently ported to Alsa because running -Jack as a systemd daemon proved to be unsupportable,. +Jack as a systemd daemon proved to be unsupportable. (Completely broken when pipewire is +installed). +This was all functional code at one point, but is no longer in sync with the current +codebase. It won't work as is. It probably implemenets the driver interface properly. +You would have to go through AudioHost.cpp and substitute JackDriver for AlsaDriver +as appropriate. */ + + +#if JACK_HOST + #include "pch.h" + #include "JackDriver.hpp" #include @@ -40,8 +50,6 @@ Jack as a systemd daemon proved to be unsupportable,. #include #include "Lv2Log.hpp" -#if JACK_HOST - namespace pipedal { diff --git a/src/PiPedalModel.cpp b/src/PiPedalModel.cpp index ace8873..5c95a4e 100644 --- a/src/PiPedalModel.cpp +++ b/src/PiPedalModel.cpp @@ -299,11 +299,6 @@ void PiPedalModel::Load() #endif } - struct sched_param scheduler_params; - scheduler_params.sched_priority = 10; - memset(&scheduler_params, 0, sizeof(sched_param)); - sched_setscheduler(0, SCHED_RR, &scheduler_params); - RestartAudio(); } diff --git a/src/RtInversionGuard.hpp b/src/RtInversionGuard.hpp index 295b9eb..74e0a29 100644 --- a/src/RtInversionGuard.hpp +++ b/src/RtInversionGuard.hpp @@ -24,6 +24,8 @@ #pragma once +#ifdef JUNK //yyx delete me. + #include "ss.hpp" #include "Lv2Log.hpp" @@ -66,4 +68,6 @@ namespace pipedal sched_setscheduler(0, currentScheduler, ¤tPriority); } }; -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/SchedulerPriority.cpp b/src/SchedulerPriority.cpp new file mode 100644 index 0000000..5b64d44 --- /dev/null +++ b/src/SchedulerPriority.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2024 Robin E. R. Davies +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#include "SchedulerPriority.hpp" +#include "Lv2Log.hpp" +#include "memory.h" +#include "sched.h" +#include "ss.hpp" +#include + +#ifdef __linux__ +#include "unistd.h" // for nice()` +#endif + +using namespace pipedal; + +static constexpr int EXPECTED_RT_THREAD_PRIORITY_MAX = 81; + +static constexpr int RT_AUDIO_THREAD_PRIORITY = 80; +static constexpr int NICE_AUDIO_THREAD_PRIORITY = -19; + +static constexpr int RT_AUDIOSERVICE_THREAD_PRIORITY = 10; +static constexpr int NICE_AUDIOSERVICE_THREAD_PRIORITY = -17; + +static constexpr int RT_LV2SCHEDULER_THREAD_PRIORITY = -1; +static constexpr int NICE_LV2SCHEDULER_THREAD_PRIORITY = 2; // nice(2): don't let plugins screw things up by pinning the CPU + +static constexpr int RT_WEBSERVER_THREAD_PRIORITY = -1; +static constexpr int NICE_WEBSERVER_THREAD_PRIORITY = -2; // don't get bogged down by UI threads. + +bool pipedal::IsRtPreemptKernel(SchedulerPriority priority) +{ + #ifdef __linux__ + struct oldPolicy; + struct sched_param oldParam = {0}; + + struct sched_param param = {0}; + + auto oldPolicy = sched_getscheduler(0); + if (sched_getparam(0,&oldParam) == -1) + { + return false; + } + auto priorityMin = sched_get_priority_min(SCHED_RR); + param.sched_priority = priorityMin; + if (sched_setscheduler(0,SCHED_RR,¶m) == -1) + { + return false; + } + sched_setscheduler(0,oldPolicy,&oldParam); + return true; + + #else + return true; // let SetPrority sort out whether it can or can't. + #endif +} + +static void SetPriority(int realtimePriority, int nicePriority, const char *priorityName) +{ + + if (realtimePriority != -1) + { + int initialPriority = realtimePriority; + int min = sched_get_priority_min(SCHED_RR); + int max = sched_get_priority_max(SCHED_RR); + if (realtimePriority > max) { + realtimePriority = max + EXPECTED_RT_THREAD_PRIORITY_MAX-realtimePriority; + } + if (realtimePriority < min) { + realtimePriority = min; + } + if (initialPriority != realtimePriority) + { + Lv2Log::warning( + SS("Realtime priority adjusted from " << initialPriority << " to " << realtimePriority << ". (" << priorityName << ")") + ); + } + + struct sched_param param; + memset(¶m, 0, sizeof(param)); + param.sched_priority = realtimePriority; + + int result = sched_setscheduler(0, SCHED_RR, ¶m); + if (result == 0) + { + Lv2Log::debug("Service thread priority successfully boosted."); + return; + } + else + { + Lv2Log::debug(SS("Failed to set RT thread priority for " << priorityName << " (" << strerror(result) << ")")); + } + } + // If the RT scheduler isn't working, fall back to using NICE priority. + int result = nice(nicePriority); + if (result == -1) + { + Lv2Log::error(SS("Failed Failed to set thread priority. (" << priorityName << ")")); + } +} + +void pipedal::SetThreadPriority(SchedulerPriority priority) +{ + +#if defined(__linux__) + switch (priority) + { + case SchedulerPriority::RealtimeAudio: + SetPriority(RT_AUDIO_THREAD_PRIORITY, NICE_AUDIO_THREAD_PRIORITY, "RealtimeAudio"); + break; + case SchedulerPriority::AudioService: + SetPriority(RT_AUDIOSERVICE_THREAD_PRIORITY, NICE_AUDIOSERVICE_THREAD_PRIORITY, "AudioService"); + break; + case SchedulerPriority::Lv2Scheduler: + SetPriority(RT_LV2SCHEDULER_THREAD_PRIORITY, NICE_LV2SCHEDULER_THREAD_PRIORITY, "Lv2Scheduler"); + break; + case SchedulerPriority::WebServerThread: + SetPriority(RT_WEBSERVER_THREAD_PRIORITY, NICE_WEBSERVER_THREAD_PRIORITY, "Lv2Scheduler"); + break; + default: + Lv2Log::error("Invalid scheduler priority."); + throw std::runtime_error("Invalid value."); + } +#elif defined(__WIN32) +// Realtime thread priority must be set using MM scheduler. +// Others must run with elevated priority, but not realtime priority (MUST run with higher priority than UI threads). +#error "Realtime scheduling not implmented for WIN32" +#else +#error "Realtime scheduling not implmented for your platform" +#endif +} diff --git a/src/SchedulerPriority.hpp b/src/SchedulerPriority.hpp new file mode 100644 index 0000000..2309eeb --- /dev/null +++ b/src/SchedulerPriority.hpp @@ -0,0 +1,33 @@ +// Copyright (c) 2024 Robin E. R. Davies +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +#pragma once + +namespace pipedal { + enum class SchedulerPriority { + RealtimeAudio, // the audio service thread. + AudioService, // non-realtime servicing of AudioThread responses. + Lv2Scheduler, // LV2 Scheduler service thread. + WebServerThread, // Web server threads. + }; + + bool IsRtPreemptKernel(SchedulerPriority priority); + + void SetThreadPriority(SchedulerPriority priority); +} \ No newline at end of file diff --git a/src/Worker.cpp b/src/Worker.cpp index 6f14779..1f0361d 100644 --- a/src/Worker.cpp +++ b/src/Worker.cpp @@ -42,6 +42,7 @@ #include // for nice( #include #include "util.hpp" +#include "SchedulerPriority.hpp" using namespace pipedal; @@ -190,12 +191,8 @@ void HostWorkerThread::ThreadProc() noexcept { // run nice +2 (priority -2 on Windows) SetThreadName("lv2_worker"); - errno = 0; - std::ignore = nice(2); - if (errno != 0) - { - std::cout << "Warning: Unable to run Lv2 schedule thread at nice +1" << std::endl; - } + SetThreadPriority(SchedulerPriority::Lv2Scheduler); + try { diff --git a/src/main.cpp b/src/main.cpp index e211bde..2f22fb9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -44,10 +44,11 @@ #include #include +#include "SchedulerPriority.hpp" #include -using namespace pipedal; +using namespace pipedal; #ifdef __ARM_ARCH_ISA_A64 #define AARCH64 @@ -244,6 +245,9 @@ int main(int argc, char *argv[]) std::shared_ptr server; try { + // (web server threads inherit the main thread priority) + SetThreadPriority(SchedulerPriority::WebServerThread); + auto const address = boost::asio::ip::make_address(configuration.GetSocketServerAddress()); port = static_cast(configuration.GetSocketServerPort());