diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0041d5e..f3e918a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -210,6 +210,7 @@ set (PIPEDAL_SOURCES Tone3000DownloadProgress.cpp Tone3000DownloadProgress.hpp Tone3000Download.cpp Tone3000Download.hpp Tone3000Downloader.cpp Tone3000Downloader.hpp + Tone3000Throttler.cpp Tone3000Throttler.hpp CrashGuard.cpp CrashGuard.hpp ModTemplateGenerator.cpp ModTemplateGenerator.hpp diff --git a/src/Tone3000Download.cpp b/src/Tone3000Download.cpp index 2ecc86a..ee0c695 100644 --- a/src/Tone3000Download.cpp +++ b/src/Tone3000Download.cpp @@ -23,6 +23,7 @@ #include "ss.hpp" #include "Tone3000Download.hpp" +#include "Tone3000Throttler.hpp" #include "TemporaryFile.hpp" #include #include @@ -142,80 +143,6 @@ static std::string mdSanitize(const std::string &str, const std::string &illegal return os.str(); } -#ifdef JUNK -static std::string mdDataUrl(const std::string &mimeType, const std::filesystem::path &filePath) -{ - // maximum 512k! - size_t fileSize = fs::file_size(filePath); - if (fileSize > 512UL * 1024UL * 1024UL) - { - return ""; - } - std::ifstream file(filePath, std::ios::binary); - if (!file.is_open()) - { - return ""; - } - - static const char kBase64Alphabet[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - std::string result; - - std::ostringstream os; - os << "data:" << mimeType << ";base64,"; - - result = os.str(); - size_t encodedDataSize = ((fileSize + 2) / 3) * 4; - result.reserve(result.length() + encodedDataSize); - - constexpr size_t kBufferSize = 32 * 1024; - std::vector buffer; - buffer.resize(kBufferSize); - - size_t offset = 0; - while (offset < fileSize) - { - size_t thisTime = std::min(kBufferSize, fileSize - offset); - file.read(buffer.data(), thisTime); - - size_t i = 0; - while (i + 2 < thisTime) - { - unsigned int triple = (buffer[i] << 16) | (buffer[i + 1] << 8) | buffer[i + 2]; - result.push_back(kBase64Alphabet[(triple >> 18) & 0x3F]); - result.push_back(kBase64Alphabet[(triple >> 12) & 0x3F]); - result.push_back(kBase64Alphabet[(triple >> 6) & 0x3F]); - result.push_back(kBase64Alphabet[triple & 0x3F]); - i += 3; - } - - if (i < thisTime) - { - unsigned int triple = buffer[i] << 16; - result.push_back(kBase64Alphabet[(triple >> 18) & 0x3F]); - if (i + 1 < thisTime) - { - triple |= buffer[i + 1] << 8; - result.push_back(kBase64Alphabet[(triple >> 12) & 0x3F]); - result.push_back(kBase64Alphabet[(triple >> 6) & 0x3F]); - result.push_back('='); - } - else - { - result.push_back(kBase64Alphabet[(triple >> 12) & 0x3F]); - result.push_back('='); - result.push_back('='); - } - } - - offset += thisTime; - } - - return result; -} -#endif - static std::string mdDate(const tone3000_time_point &date_) { std::ostringstream os; @@ -668,6 +595,27 @@ static std::string GetHeader(const std::string &headerName, const std::vector &isCancelled) +{ + using clock_t = std::chrono::steady_clock; + + auto start = clock_t::now(); + while (true) + { + auto now = clock_t::now(); + auto elapsed = now - start; + if (elapsed > duration) + { + return true; + } + if (isCancelled()) + { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} std::string pipedal::tone3000::DownloadTone3000Bundle( const std::filesystem::path &destinationFolder, const std::string &downloadUrl, @@ -740,44 +688,82 @@ std::string pipedal::tone3000::DownloadTone3000Bundle( } try { - int result = CurlGet( - requests, - [&](size_t completed, size_t total) + size_t downloadIndex = 0; + while (downloadIndex < requests.size()) + { + if (isCancelled()) { + throw std::runtime_error("Cancelled"); + } + size_t thisTime = requests.size() - downloadIndex; + thisTime = Tone3000Throttler::instance().ReserveDownloadSlots(thisTime); + if (thisTime != 0) + { + std::vector requestsThisTime; + for (size_t i = 0; i < thisTime; ++i) + { + requestsThisTime.push_back(requests[downloadIndex + i]); + } + + int result = CurlGet( + requestsThisTime, + [&](size_t completed, size_t total) + { + if (isCancelled()) + { + return false; + } + progress.progress(completed + downloadIndex); + progress.total(requests.size()); + progressCallback(progress); + return true; + }, + &headers); + if (isCancelled()) { - return false; + throw std::runtime_error("Cancelled"); } - progress.progress(completed); - progress.total(total); - progressCallback(progress); - return true; - }, - &headers); - if (isCancelled()) - { - CleanUpFailedDownload(bundlePath, requests); - return ""; - } - if (result != 200) - { - CleanUpFailedDownload(bundlePath, requests); - throw std::runtime_error(SS("Server download failed. HTTP Error " << HtmlHelper::httpErrorString(result))); + + if (result != 200) + { + throw std::runtime_error(SS("Server download failed. HTTP Error " << HtmlHelper::httpErrorString(result))); + } + + downloadIndex += thisTime; + } + if (thisTime <= 1) + { + CancellableSleep(std::chrono::milliseconds(1000), isCancelled); + } } + fs::path readmePath = bundlePath / "README.md"; std::string thumbnailUrl; - if (tone3000Download.images().has_value() && tone3000Download.images().value().size() != 0) + if (tone3000Download.images().has_value() && tone3000Download.images().value().size() != 0) { TemporaryFile imageFile{TONE3000_THUMBNAIL_PATH}; - - std::string imageUrl = tone3000Download.images().value()[0]; std::vector imageHeaders; try { + size_t reserveCount; + while (true) + { + reserveCount = Tone3000Throttler::instance().ReserveDownloadSlots(1); + if (reserveCount != 0) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + if (isCancelled()) + { + throw std::runtime_error("Cancelled"); + } + } if (CurlGet(imageUrl, imageFile.Path(), &imageHeaders) == 200) { std::string mediaType = GetHeader("Content-Type", imageHeaders); @@ -785,13 +771,17 @@ std::string pipedal::tone3000::DownloadTone3000Bundle( if (mediaType == "image/jpeg") { extension = ".jpeg"; - } else if (mediaType == "image/webp") + } + else if (mediaType == "image/webp") { extension = ".webp"; - } else if (mediaType == "image/png") + } + else if (mediaType == "image/png") { extension = ".png"; - } else { + } + else + { Lv2Log::warning("Tone3000 thumbnail has an unexpected media type of '%s'", mediaType.c_str()); throw std::runtime_error("Invalid thumbnail media type."); } @@ -806,7 +796,7 @@ std::string pipedal::tone3000::DownloadTone3000Bundle( fs::create_directories(finalPath.parent_path()); fs::rename(thumnbailTempPath, finalPath); thumbnailUrl = SS("var/tone3000_thumbnail?id=" << tone3000Download.id()); -// thumbnailUrl = mdDataUrl("image/jpeg", finalPath); + // thumbnailUrl = mdDataUrl("image/jpeg", finalPath); } } } diff --git a/src/Tone3000Throttler.cpp b/src/Tone3000Throttler.cpp new file mode 100644 index 0000000..de27b65 --- /dev/null +++ b/src/Tone3000Throttler.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 Robin E. R. Davies + * All rights reserved. + + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "Tone3000Throttler.hpp" + +using namespace pipedal; + +static constexpr size_t THROTTLING_LIMIT = 80; +static constexpr size_t BULK_DOWNLOAD_LIMIT = 20; + + +Tone3000Throttler Tone3000Throttler::instance_; + +void Tone3000Throttler::ClearExpiredDownloads_() +{ + auto expiryTime = clock_t::now(); + expiryTime -= std::chrono::seconds(60); + + while (downloadTimes.size() > 0 && downloadTimes[0] < expiryTime) { + downloadTimes.erase(downloadTimes.begin()); + } + +} +void Tone3000Throttler::AddDownload() +{ + std::lock_guard lock{m_mutex}; + this->downloadTimes.push_back(clock_t::now()); +} + +bool Tone3000Throttler::IsThrottled() +{ + std::lock_guard lock{m_mutex}; + + ClearExpiredDownloads_(); + return this->downloadTimes.size() >= THROTTLING_LIMIT; +} + +size_t Tone3000Throttler::ReserveDownloadSlots(size_t requested) +{ + ClearExpiredDownloads_(); + size_t result; + + size_t reserved = this->downloadTimes.size(); + if (reserved == THROTTLING_LIMIT) { + return 0; + } + if (reserved < BULK_DOWNLOAD_LIMIT) + { + if (requested+reserved >= BULK_DOWNLOAD_LIMIT) + { + result = BULK_DOWNLOAD_LIMIT-reserved; + if (result == 0) + { + result = 1; + } + } else { + result = requested; + } + } else { + result = 1; // download one at a time! + } + auto now = clock_t::now(); + for (size_t i = 0; i < result; ++i) + { + this->downloadTimes.push_back(now); + } + return result; +} + +size_t Tone3000Throttler::AvailableDownloadSlots() { + ClearExpiredDownloads_(); + return THROTTLING_LIMIT-this->downloadTimes.size(); +} \ No newline at end of file diff --git a/src/Tone3000Throttler.hpp b/src/Tone3000Throttler.hpp new file mode 100644 index 0000000..3accf74 --- /dev/null +++ b/src/Tone3000Throttler.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026 Robin E. R. Davies + * All rights reserved. + + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +#include +#include +#include + +namespace pipedal +{ + class Tone3000Throttler + { + private: + Tone3000Throttler() = default; + Tone3000Throttler(const Tone3000Throttler&) = delete; + Tone3000Throttler(Tone3000Throttler&&) = delete; + Tone3000Throttler&operator=(const Tone3000Throttler&) = delete; + + public: + static Tone3000Throttler&instance() { return instance_; } + + using clock_t = std::chrono::steady_clock; + + void AddDownload(); + size_t ReserveDownloadSlots(size_t requested); + size_t AvailableDownloadSlots(); + bool IsThrottled(); + + private: + std::mutex m_mutex; + static Tone3000Throttler instance_; + + void ClearExpiredDownloads_(); + std::vector downloadTimes; + }; + +} \ No newline at end of file diff --git a/vite/src/pipedal/Tone3000Dialog.tsx b/vite/src/pipedal/Tone3000Dialog.tsx index adb09d0..c2bdb71 100644 --- a/vite/src/pipedal/Tone3000Dialog.tsx +++ b/vite/src/pipedal/Tone3000Dialog.tsx @@ -75,7 +75,9 @@ export class Tone3000DownloadHandler { let redirectUrl_ = this.redirectUrl(); let result = `https://www.tone3000.com/api/v1/select?app_id=${this.appId}&redirect_url=${redirectUrl_}`; if (downloadType === Tone3000DownloadType.CabIr) { - result += "&gear=ir"; + result += "&platform=ir"; + } else if (downloadType === Tone3000DownloadType.Nam) { + result += "&platform=nam"; } return result; }