/* * Copyright (c) Robin E.R. Davies * All rights reserved. * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "ss.hpp" #include "Tone3000Download.hpp" #include "TemporaryFile.hpp" #include #include #include "util.hpp" #include "Tone3000DownloadProgress.hpp" #include #include "HtmlHelper.hpp" using namespace pipedal; using namespace pipedal::tone3000; namespace fs = std::filesystem; JSON_MAP_BEGIN(Tone3000Model) JSON_MAP_REFERENCE(Tone3000Model, id) JSON_MAP_REFERENCE(Tone3000Model, name) JSON_MAP_REFERENCE(Tone3000Model, model_url) JSON_MAP_REFERENCE(Tone3000Model, created_at) JSON_MAP_REFERENCE(Tone3000Model, size) JSON_MAP_REFERENCE(Tone3000Model, user_id) JSON_MAP_END() JSON_MAP_BEGIN(Tone3000User) JSON_MAP_REFERENCE(Tone3000User, id) JSON_MAP_REFERENCE(Tone3000User, username) JSON_MAP_REFERENCE(Tone3000User, avatar_url) JSON_MAP_REFERENCE(Tone3000User, url) JSON_MAP_END() JSON_MAP_BEGIN(Tone3000Download) JSON_MAP_REFERENCE(Tone3000Download, id) JSON_MAP_REFERENCE(Tone3000Download, user_id) JSON_MAP_REFERENCE(Tone3000Download, title) JSON_MAP_REFERENCE(Tone3000Download, description) JSON_MAP_REFERENCE(Tone3000Download, created_at) JSON_MAP_REFERENCE(Tone3000Download, updated_at) JSON_MAP_REFERENCE(Tone3000Download, platform) JSON_MAP_REFERENCE(Tone3000Download, gear) JSON_MAP_REFERENCE(Tone3000Download, images) JSON_MAP_REFERENCE(Tone3000Download, is_public) JSON_MAP_REFERENCE(Tone3000Download, links) JSON_MAP_REFERENCE(Tone3000Download, model_count) JSON_MAP_REFERENCE(Tone3000Download, favorites_count) JSON_MAP_REFERENCE(Tone3000Download, license) JSON_MAP_REFERENCE(Tone3000Download, sizes) JSON_MAP_REFERENCE(Tone3000Download, user) JSON_MAP_REFERENCE(Tone3000Download, models) JSON_MAP_END() JSON_MAP_BEGIN(Tone3000DownloadResult) JSON_MAP_REFERENCE(Tone3000DownloadResult, success) JSON_MAP_REFERENCE(Tone3000DownloadResult, errorMessage) JSON_MAP_END() static const std::filesystem::path WEB_TEMP_DIR{"/var/pipedal/web_temp"}; static bool enableLogging = true; static std::string getCurlErrorFromExitCode(int exitCode) { switch (exitCode) { case 3 * 256: // malformed URL return "Invalid URL."; case 6 * 256: // could not resolve hostname return "Could not resolve hostname. Check that the PiPedal server is connected to the Internet."; case 7 * 256: // failed to connect to host return "Failed to connect to host."; case 22 * 256: // HTTP error return "HTTP error downloading file."; case 28 * 256: // timeout return "Download timeout."; case 63 * 256: // file too large return "File too large."; default: return SS("Failed to download file (exit code: " << (exitCode / 256) << ")"); } } static void cancellableSleep(std::chrono::steady_clock::duration duration, std::function &isCancelled) { std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); while (true) { if (std::chrono::steady_clock::now() - start >= duration) { break; } if (isCancelled()) { break; } std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } static void downloadFile( const std::string &url, const fs::path &path, std::function &isCancelled) { // Create temporary file for HTTP status code TemporaryFile httpCodeFile{WEB_TEMP_DIR}; fs::path httpCodePath = httpCodeFile.Path(); TemporaryFile headersFile{WEB_TEMP_DIR}; fs::path headersFilePath = headersFile.Path(); for (size_t retry = 0; retry < 10; ++retry) { // Build curl command with error detection and proper escaping std::ostringstream os; os << "/usr/bin/curl -f -s -S -L --max-time 60 --retry 5 --retry-delay 1 --retry-max-time 10 --max-filesize 104857600"; os << " -w '%{http_code}' -o "; os << ShellEscape(path.string()); os << " -D " << ShellEscape(headersFilePath); os << " "; os << ShellEscape(url); os << " > "; os << ShellEscape(httpCodePath.string()); std::string command = os.str(); // Execute curl command int exitCode = std::system(command.c_str()); // Read HTTP status code int httpCode = 0; if (fs::exists(httpCodePath)) { std::ifstream httpCodeStream(httpCodePath); std::string httpCodeStr; std::getline(httpCodeStream, httpCodeStr); try { httpCode = std::stoi(httpCodeStr); } catch (...) { // If parsing fails, keep httpCode as 0 } } if (exitCode != 0) { if (exitCode == 22 * 256) // HTTP error { std::ifstream headersStream(headersFilePath); std::string headerStr; while (std::getline(headersStream, headerStr)) { std::cout << headerStr << std::endl; } if (httpCode == 429 || httpCode == 403 || httpCode == 503) { cancellableSleep(std::chrono::milliseconds(2000), isCancelled); if (isCancelled()) { return; } continue; } throw std::runtime_error( SS("Server was unabled to download the file. Http error: " << HtmlHelper::httpErrorString(httpCode))); } else { throw std::runtime_error( SS("Server was unable to download the file. " << getCurlErrorFromExitCode(exitCode) << " " << "url: " << url)); } } else { break; } } // Verify file was downloaded if (!fs::exists(path) || fs::file_size(path) == 0) { throw std::runtime_error("Downloaded file is empty or missing"); } } static std::string mdSanitize(const std::string &str, const std::string &illegalCharacters = "") { std::ostringstream os; for (char c : str) { if ((unsigned char)c < 0x20) continue; if (c == '<') { os << "<"; continue; } auto npos = illegalCharacters.find_first_of(c); if (npos != std::string::npos) { continue; } os << c; } return os.str(); } static std::string mdDate(const tone3000_time_point &date_) { std::ostringstream os; // C++ doesn't handle timezones. Just zap the timezone, and replace it with "Z" std::string date = date_; // Remove timezone offset if present and replace with Z size_t plusPos = date.find('+'); if (plusPos != std::string::npos) { date = date.substr(0, plusPos) + "Z"; } // Parse ISO 8601 date string into tm std::tm tm = {}; std::istringstream ss(date); ss >> std::get_time(&tm, "%Y-%m-%dT%H:%M:%S"); if (ss.fail()) { os << date; return os.str(); } os << std::put_time(&tm, "%x"); return os.str(); } static std::string mdEnum(const std::string &value, const std::map &enumValues) { auto ff = enumValues.find(value); if (ff == enumValues.end()) { return value; } return ff->second; } static std::string mdEnumList( const std::vector &values, const std::map &enumValues) { if (values.size() == 1) { return mdEnum(values[0], enumValues); } std::ostringstream os; os << '['; bool first = true; for (const auto &value : values) { if (first) { first = false; } else { os << ", "; } os << mdEnum(value, enumValues); } os << ']'; return os.str(); } static std::map sizeEnumValues = { {"standard", "Standard"}, {"lite", "Lite"}, {"feather", "Feather"}, {"nano", "Nano"}, {"custom", "Custom"}}; static std::string mdSizes(const std::vector &sizes) { return mdEnumList(sizes, sizeEnumValues); } static std::map gearEnumValues = { {"amp", "Amp only"}, {"full-rig", "Full rig"}, {"pedal", "Pedal"}, {"outboard", "Outboard"}, {"ir", "IR"}}; static std::string mdGear(const std::string &gear) { return mdEnum(gear, gearEnumValues); } namespace { enum class LicenseFlags { None = 0, Cc = 1, By = 2, CcBy = 3, // = Cc | By Sa = 4, Nc = 8, Nd = 16, Cc0 = 32, }; static bool operator&(LicenseFlags v1, LicenseFlags v2) { return (((int)v1) & ((int)v2)) != 0; } static LicenseFlags operator|(LicenseFlags v1, LicenseFlags v2) { return (LicenseFlags)(((int)(v1)) | ((int)(v2))); } struct LicenseInfo { std::string key; std::string displayName; std::string url; LicenseFlags licenseFlags; }; } static std::vector licenses{ {.key = "t3k", .displayName = "T3K", .url = "licenses/t3k_license.html", .licenseFlags = LicenseFlags::None}, {.key = "cc-by", .displayName = "CC BY 4.0", .url = "https://creativecommons.org/licenses/by/4.0/", .licenseFlags = LicenseFlags::CcBy}, {.key = "cc-by-sa", .displayName = "CC BY-SA 4.0", .url = "https://creativecommons.org/licenses/by-sa/4.0/", .licenseFlags = LicenseFlags::CcBy | LicenseFlags::Sa}, {.key = "cc-by-nc", .displayName = "CC BY-NC 4.0", .url = "https://creativecommons.org/licenses/by-nc/4.0/", .licenseFlags = LicenseFlags::CcBy | LicenseFlags::Nc}, {.key = "cc-by-nc-sa", .displayName = "CC BY-NC-SA 4.0", .url = "https://creativecommons.org/licenses/by-nc-sa/4.0/", .licenseFlags = LicenseFlags::CcBy | LicenseFlags::Nc | LicenseFlags::Sa}, {.key = "cc-by-nd", .displayName = "CC BY-ND 4.0", .url = "https://creativecommons.org/licenses/by-nd/4.0/", .licenseFlags = LicenseFlags::CcBy | LicenseFlags::Nd}, {.key = "cc-by-nc-nd", .displayName = "CC BY-NC-ND 4.0", .url = "https://creativecommons.org/licenses/by-nc-nd/4.0/", .licenseFlags = LicenseFlags::CcBy | LicenseFlags::Nc | LicenseFlags::Nd}, { .key = "cco", .displayName = "CC0", .url = "https://creativecommons.org/publicdomain/zero/1.0/", .licenseFlags = LicenseFlags::Cc | LicenseFlags::Cc0, }, }; static std::map makeLicenseEnum() { std::map result; for (const auto &license : licenses) { result[license.key] = license; } return result; } static std::map licenseEnum = makeLicenseEnum(); static std::string mdLicenseIcon(LicenseFlags licenseFlag) { std::ostringstream os; std::string url; switch (licenseFlag) { case LicenseFlags::Cc: url = "img/cc.svg"; break; case LicenseFlags::By: url = "img/by.svg"; break; case LicenseFlags::Sa: url = "img/sa.svg"; break; case LicenseFlags::Nc: url = "img/nc.svg"; break; case LicenseFlags::Nd: url = "img/nd.svg"; break; case LicenseFlags::Cc0: url = "img/cc0.svg"; break; case LicenseFlags::None: throw std::runtime_error("Invalid argument."); } os << ""; return os.str(); } static void mdEscapeString(std::ostream &f, const std::string &str) { size_t ix = 0; while (ix < str.size()) { char c = str[ix++]; if (c == '\n') { if (ix < str.size() && str[ix] == '\n') { f << "\n\n"; } else { f << " \n"; // a
in md. } } else { f << c; } } } static std::string mdHref(const std::string &url) { std::ostringstream os; os << '\''; for (char c : url) { if (c == '\'') { continue; } os << c; } os << '\''; return os.str(); } static std::string mdLicense(const std::string &license) { auto ff = licenseEnum.find(license); LicenseInfo licenseInfo; if (ff != licenseEnum.end()) { licenseInfo = ff->second; } else { licenseInfo.displayName = license; } std::ostringstream os; if (licenseInfo.licenseFlags != LicenseFlags::None) { for (LicenseFlags licenseFlag : std::vector{LicenseFlags::Cc, LicenseFlags::By, LicenseFlags::Cc0, LicenseFlags::Nc, LicenseFlags::Sa, LicenseFlags::Nd}) { if (licenseInfo.licenseFlags & licenseFlag) { os << mdLicenseIcon(licenseFlag); } } os << " "; } if (licenseInfo.url != "") { os << "" << mdSanitize(licenseInfo.displayName) << ""; } else { os << mdSanitize(licenseInfo.displayName); } return os.str(); } // static std::string mdTestLicenses() // { // std::ostringstream os; // for (auto license: licenseEnum) // { // os << "
" << mdLicense(license.first) << "\n"; // } // return os.str(); // } static std::map platformEnumValues = { {"nam", "NAM"}, {"ir", "I/R"}, {"aida-x", "Aida X"}, {"aa-snapshot", "aa-snapshot"}, {"proteus", "Proteus"}}; static std::string mdPlatform(const std::string &platform) { return mdEnum(platform, platformEnumValues); } static std::string mdUser(const tone3000::Tone3000User &user) { // clang-format off std::ostringstream os; os << "" << mdSanitize(user.username()) << ""; return os.str(); // clang-format on } static void mdLink(std::ostream &f, const std::string &label, const std::string &url) { f << "[" << mdSanitize(label, "]") << "](" << mdSanitize(url, ")") << ")"; } static void mdImage(std::ostream &f, const std::string url) { f << "![" << mdSanitize("Plugin Thumbname", "]") << "](" << mdSanitize(url, "]") << "#tone3000_thumbnail" << ")"; f << "\n\n"; } static void writeReadme(const fs::path &path, const Tone3000Download &download) { std::ofstream of{path}; if (!of.is_open()) { throw std::runtime_error(SS("Unable to create file " << path)); } std::string imageLink; if (download.images().size() > 0) { imageLink = download.images()[0]; } // clang-format off of << "
\n" << " \n" << "
\n" << "

" << mdSanitize(download.title()) << "

\n" << "
\n" << "

\n" << mdDate(download.updated_at()) << " " << mdUser(download.user()); if (download.sizes().has_value()) { of << "
\n" << " " << mdSizes(download.sizes().value()) << ", " ; } of << mdGear(download.gear()) << ", " << mdPlatform(download.platform()) << "
\n"; if (download.license() != "") { of << " License: " << mdLicense(download.license()) << "\n"; } of // << " " << mdTestLicenses() << "

\n" << "
\n" << "
\n" << "
\n"; // clang-format on of << "\n\n"; mdEscapeString(of, download.description()); if (download.links().size() > 0) { of << std::endl; of << "## Links" << std::endl << std::endl; for (auto &link : download.links()) { of << "- " << "[" << link << "](" << link << ")" << std::endl; } } } std::string pipedal::tone3000::DownloadTone3000File( const std::filesystem::path &destinationFolder, const std::string &downloadUrl, int64_t handle, std::function progressCallback, std::function isCancelled) { TemporaryFile downloadTemporaryFile{WEB_TEMP_DIR}; fs::path downloadPath = downloadTemporaryFile.Path(); std::string resultDirectory; Tone3000DownloadProgress progress; progress.handle(handle); progressCallback(progress); downloadFile(downloadUrl, downloadPath, isCancelled); if (isCancelled()) { return resultDirectory; } std::ifstream f{downloadPath}; if (!f.is_open()) { throw std::runtime_error(SS("Can't open file " << downloadPath)); } json_reader reader(f); Tone3000Download tone3000Download; reader.read(&tone3000Download); if (isCancelled()) { return resultDirectory; } progress.title(tone3000Download.title()); progress.total(tone3000Download.models().size()); progressCallback(progress); if (tone3000Download.platform() != "nam" && tone3000Download.platform() != "ir") { throw std::runtime_error(SS("Unexpected file format. (platform=\"" << tone3000Download.platform() << "\")")); } fs::path bundlePath = destinationFolder / StringToSafeFilename(tone3000Download.title()); resultDirectory = bundlePath; fs::create_directories(bundlePath); uint64_t nModel = 0; for (auto &model : tone3000Download.models()) { if (isCancelled()) { return resultDirectory; } fs::path modelPath; if (tone3000Download.platform() == "ir") { modelPath = bundlePath / SS(StringToSafeFilename(model.name()) << ".wav"); } else { modelPath = bundlePath / SS(StringToSafeFilename(model.name()) << ".nam"); } downloadFile(model.model_url(), modelPath, isCancelled); if (isCancelled()) { return resultDirectory; } progress.progress(++nModel); progressCallback(progress); } if (isCancelled()) { return resultDirectory; } fs::path readmePath = bundlePath / "README.md"; writeReadme(readmePath, tone3000Download); return resultDirectory; } Tone3000DownloadResult::Tone3000DownloadResult() : success_(true) { } Tone3000DownloadResult::Tone3000DownloadResult(const std::exception &e) : success_(false), errorMessage_(e.what()) { }