Files
op-pedal/vite/src/pipedal/Tone3000Downloader.tsx
T

652 lines
24 KiB
TypeScript

import { PiPedalModel, getErrorMessage, Tone3000PkceParams } from "./PiPedalModel";
import Tone3000DownloadType from "./Tone3000DownloadType";
import { T3K_DEBUG, PkceParams, setServerPkceParams, handleOAuthCallback } from "./t3k/tone3000-client.ts";
import { Model, PaginatedResponse, Platform, Tone } from "./t3k/types.ts";
import { PUBLISHABLE_KEY } from './t3k/config.ts';
import { startSelectFlowPopup, buildPkceParams,T3KClient } from "./t3k/tone3000-client.ts";
import Tone3000DownloadProgress from './Tone3000DownloadProgress';
import { safeFilenameEncode } from "./SafeFilename";
// const T3K_API = "https://www.tone3000.com";
// used to check for online status.
let TONE3000_PING_URL = "https://www.tone3000.com/robots.txt";
async function asyncSleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
const USE_SERVER_PCKE=true;
const RUN_THROTTLER_TEST = true;
const ALLOWED_DOWNLOADS_PER_MINUTE = 25;
const THROTTLING_TRIGGGER_LEVEL = Math.floor(ALLOWED_DOWNLOADS_PER_MINUTE/2);
class DownloadThrottler {
private fifo: number[] = [];
private downloadCount: number = 0;
resetThrottling(now: number = Date.now()): void {
// clean out entries that are no longer counted by the Ton3000 throttler.
while (this.fifo.length > 0 && this.fifo[0] + 60000 <= now) {
this.fifo.shift();
}
this.downloadCount = this.fifo.length;
}
getThrottleDelay(now : number = Date.now()): number {
while (this.fifo.length > 0 && this.fifo[0] + 60000 <= now) {
this.fifo.shift();
}
let downloadCount = this.downloadCount;
let delay: number;
if (downloadCount < THROTTLING_TRIGGGER_LEVEL) {
delay = 0;
} else if (downloadCount < ALLOWED_DOWNLOADS_PER_MINUTE) {
delay = Math.ceil((60000 / (ALLOWED_DOWNLOADS_PER_MINUTE-THROTTLING_TRIGGGER_LEVEL)));
} else {
delay = Math.ceil(60000 / (ALLOWED_DOWNLOADS_PER_MINUTE-1));
}
this.fifo.push(now + delay);
console.debug(` ${now}: Download: ${downloadCount} Delay: ${delay} ms Throttle count: ${this.fifo.length}`);
++this.downloadCount;
return delay;
}
}
function runThrottlerTest() {
// just dump the delay times for a few downloads to the debugger console, to verify that the throttler is working as expected;
let throttler = new DownloadThrottler();
let download = 0;
let now = 0;
console.debug("Tone3000 Throttler Test");
while (download < ALLOWED_DOWNLOADS_PER_MINUTE *4) {
let delay = throttler.getThrottleDelay(now);
now += delay;
++download;
}
}
export class Tone3000DownloadHandler {
// private async handleTone3000Download(
// code: string, toneId: String, downloadType: Tone3000DownloadType): Promise<void>
// {
// const {access_token} = await exchangeCode(code);
// try {
// const xxx;
// await this.model.downloadModelsFromTone3000(tone3000DownloadUrl, this.downloadPath, downloadType);
// return;
// } catch (e) {
// this.handleTone3000DownloadError(getErrorMessage(e));
// return;
// }
// }
private messageEventListener: ((event: MessageEvent) => void) | null = null;
private t3kClient: T3KClient;
private throttler: DownloadThrottler = new DownloadThrottler();
private model: PiPedalModel;
pkceParams: PkceParams | null = null;
public constructor(model: PiPedalModel) {
if (RUN_THROTTLER_TEST) {
runThrottlerTest();
}
this.model = model;
this.t3kClient = new T3KClient(
PUBLISHABLE_KEY,
() => {
model.showAlert("TONE3000 authentication failed. Please try again.");
}
);
let this_ = this;
this.messageEventListener = (event: MessageEvent) => {
if (event.data?.type === "t3k_response") {
let uri = event.data.uri;
this.handleTone3000DownloadComplete();
if (uri) {
this_.handleT3kSelectResponse(uri, event.data.storedState, event.data.codeVerifier)
.then(() => { })
.catch((error) => {
model.showAlert(getErrorMessage(error));
});
}
}
};
window.addEventListener("message", this.messageEventListener);
}
public dispose(): void {
if (this.messageEventListener) {
window.removeEventListener("message", this.messageEventListener);
this.messageEventListener = null;
}
}
private async handleT3kSelectResponse(
uri: string,
popupStoredState: string | null,
popupCodeVerifier: string | null
): Promise<void> {
if (this.pkceParams === null) {
this.model.showAlert("Invalid PKCE parameters. Please try again.");
return;
}
let tone3000PckceParams: Tone3000PkceParams = {
publishableKey: PUBLISHABLE_KEY,
redirectUrl: this.redirectUrl(),
codeChallenge: this.pkceParams.codeChallenge,
codeVerifier: this.pkceParams.codeVerifier,
state: this.pkceParams.state
};
this.DownloadModelsFromTone3000(uri, tone3000PckceParams, this.downloadPath, this.downloadType);
}
private handle = 0;
private nextHandle(): number {
return ++this.handle;
}
private progress: Tone3000DownloadProgress = new Tone3000DownloadProgress();
private async throttleRequests(): Promise<boolean> {
let now = Date.now();
let delay = this.throttler.getThrottleDelay(now);
while (delay > 0) {
await asyncSleep(250);
if (this.checkForCancel())
{
return false;
}
delay -= 250;
}
return true;
}
private onTone3000DownloadStarted() {
this.throttler.resetThrottling(); // allow full-rate burst of downloads at the start of each tone download.
this.progress.handle = this.nextHandle();
this.progress.title = "";
this.progress.progress = 0;
this.progress.total = 0;
this.progress.cancelled = false;
this.progress.transferring = true;
this.model.onTone3000DownloadStarted(this.progress.handle, this.progress.title);
}
private onTone3000DownloadProgress(progress: Tone3000DownloadProgress) {
let newProgress = progress.clone();
this.model.onTone3000DownloadProgress(newProgress);
}
private onTone3000DownloadError(errorMessage: string) {
this.progress.transferring = false;
this.model.onTone3000DownloadError(this.progress.handle, errorMessage);
}
private onTone3000DownloadComplete(resultPath: string) {
this.progress.transferring = false;
this.model.onTone3000DownloadComplete(resultPath);
}
private async DownloadModelsFromTone3000(
responseUri: string,
tone3000PckceParams: Tone3000PkceParams,
downloadPath: string,
downloadType: Tone3000DownloadType
): Promise<void> {
try {
if (T3K_DEBUG) {
console.debug("PiPedal responseUri: " + responseUri);
}
this.onTone3000DownloadStarted();
this.progress.title = "Authenticating..."
this.onTone3000DownloadProgress(this.progress);
if (this.checkForCancel()) {
return;
}
if (!await this.throttleRequests()) return;
let tokenResponse = await handleOAuthCallback(
PUBLISHABLE_KEY,
this.redirectUrl(),
responseUri);
if (!tokenResponse.ok) {
if (tokenResponse.canceled === true)
{
this.onTone3000DownloadComplete("");
return;
}
throw new Error(tokenResponse.error);
}
if (tokenResponse.canceled) {
this.onTone3000DownloadComplete("");
return;
}
if (this.checkForCancel()) {
return;
}
this.t3kClient.setTokens(tokenResponse.tokens);
if (tokenResponse.toneId == undefined) {
throw new Error("No tone selected. Please try again.");
}
let toneId = tokenResponse.toneId;
this.progress.title = "Fetching tone information...";
this.onTone3000DownloadProgress(this.progress);
if (!await this.throttleRequests()) return;
let tone: Tone = await this.t3kClient.getTone(tokenResponse.toneId);
if (this.checkForCancel()) {
return;
}
this.progress.title = tone.title;
this.onTone3000DownloadProgress(this.progress);
let page = 1;
let models: Model[] = [];
let architectureFilter: number | undefined = undefined;
// Take A2 models only for NAM downloads, and only if the tone actually has A2 models.
if (downloadType === Tone3000DownloadType.Nam && !!tone.a2_models_count && this.model.tone3000_A2_models) {
architectureFilter = 2; // NAM models only
}
while (true) {
if (!await this.throttleRequests()) return;
if (this.checkForCancel()) {
return;
}
let modelsThisTime: PaginatedResponse<Model> = await this.t3kClient.listModels(toneId, page, 800, architectureFilter);
models.push(...modelsThisTime.data);
if (modelsThisTime.total_pages <= page) {
break;
}
++page;
}
let uploadPath = this.downloadPath;
let extension: string;
switch (tone.platform) {
case Platform.Nam:
extension = ".nam";
break;
case Platform.Ir:
extension = ".wav";
break;
case Platform.Proteus:
extension = ".json";
break;
default:
throw new Error("Unsupported platform: " + tone.platform);
break;
}
let toneUploadPath = uploadPath + "/" + safeFilenameEncode(tone.title) + "/";
this.progress.total = models.length;
this.onTone3000DownloadProgress(this.progress);
let lastUpdateTime = Date.now();
for (const model of models) {
this.progress.title = model.name;;
if (Date.now() - lastUpdateTime > 250) {
this.onTone3000DownloadProgress(this.progress);
lastUpdateTime = Date.now();
}
if (!await this.throttleRequests()) return;
if (!model.model_url) {
throw new Error("Model " + model.name + " does not have a model URL.");
}
let modelUploadPath = toneUploadPath + safeFilenameEncode(model.name) + extension;
let accessToken = await this.t3kClient.getAccessToken();
let modelResult = await fetch(model.model_url,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
duplex: 'half',
} as RequestInit & { duplex: 'half' }
);
if (!modelResult.ok) {
throw new Error(`Model download failed: ${modelResult.status} ${modelResult.statusText}`);
}
if (this.checkForCancel()) {
return;
}
let serverUrl = this.model.varServerUrl + "t3k_uploadAsset?path="
+ encodeURIComponent(modelUploadPath);
// get the content length of modelResult from modelResult headers (modelresult is the return value from fetch())
const strContentLength = modelResult.headers.get('Content-Length');
let contentLength: number = 0;
if (strContentLength) {
contentLength = parseInt(strContentLength);
if (isNaN(contentLength)) {
throw new Error("File download from Tone3000 server failed: Content-Length header is invalid.");
}
}
// POST binary modelResult to serverUrl.
// Prefer streaming request body when available, else fall back to blob().
const uploadBody: Blob =
await modelResult.blob();
const uploadResponse = await fetch(serverUrl, {
method: 'POST',
body: uploadBody,
headers: {
'Content-Type': modelResult.headers.get('Content-Type') ?? 'application/octet-stream',
"Content-Length": contentLength.toString(),
"Transfer-Encoding": "chunked"
},
});
if (!uploadResponse.ok) {
throw new Error(`Upload failed: ${uploadResponse.statusText}`);
}
// discard the response body, if any, to free up memory
let uploadResult: any = await uploadResponse.json();
if (!uploadResult.ok === true) {
throw new Error(`Upload failed: ${uploadResult.error ?? "Unknown error."}`);
}
this.progress.progress++;
}
// the image file and readme.
let toobThubmnailUrl: string = "";
if (tone.images && tone.images.length > 0) {
await this.throttleRequests();
let accessToken = await this.t3kClient.getAccessToken();
let thumbnailUrl = tone.images[0];
let thumbnailResult = await fetch(thumbnailUrl,
{
headers: {
Authorization: `Bearer ${accessToken}`,
},
duplex: 'half',
} as RequestInit & { duplex: 'half' }
);
if (!thumbnailResult.ok) {
throw new Error(`Thumbnail download failed: ${thumbnailResult.status} ${thumbnailResult.statusText}`);
}
if (this.checkForCancel()) {
return;
}
let mediaType = thumbnailResult.headers.get("Content-Type");
if (mediaType == null) {
throw new Error("Thumbnail download failed: Content-Type header is missing.");
}
let extension: string;
switch (mediaType) {
case "image/jpeg":
extension = ".jpeg";
break;
case "image/webp":
extension = ".webp";
break;
case "image/png":
extension = ".png";
break;
default:
throw new Error(`Thumbnail download failed: Unexpected media type '${mediaType}'.`);
}
let blob = await thumbnailResult.blob();
const TONE3000_THUMBNAIL_PATH = "/var/pipedal/tone3000_thumbnails/";
let thumbnailUploadPath = TONE3000_THUMBNAIL_PATH + tone.id + extension;
let serverUrl = this.model.varServerUrl + "t3k_uploadAsset?path="
+ encodeURIComponent(thumbnailUploadPath);
const uploadResponse = await fetch(serverUrl, {
method: 'POST',
body: blob,
headers: {
'Content-Type': mediaType,
"Content-Length": blob.size.toString(),
"Transfer-Encoding": "chunked"
},
});
if (!uploadResponse.ok) {
throw new Error(`Thumbnail upload failed: ${uploadResponse.statusText}`);
}
let uploadResult: any = await uploadResponse.json();
if (!uploadResult.ok === true) {
throw new Error(`Thumbnail upload failed: ${uploadResult.error ?? "Unknown error."}`);
}
toobThubmnailUrl = "/var/t3k_thumbnail?id=" + tone.id;
}
let readmePath = toneUploadPath + "README.md";
this.model.writeTone3000Readme(readmePath, tone, toobThubmnailUrl);
this.onTone3000DownloadComplete(toneUploadPath);
} catch (error) {
let message = getErrorMessage(error);
if (this.progress.progress > 0) {
message += " (" + this.progress.progress.toString() + "/" + this.progress.total.toString() + " models downloaded)";
}
this.onTone3000DownloadError(message);
return;
}
}
private popupWindow: Window | null = null;
private redirectUrl(): string {
let varServerURL = new URL(this.model.varServerUrl);
let serverUrl: string;
serverUrl = varServerURL.origin;
return `${serverUrl}/html/t3k_response.html`;
// return `${serverUrl}/t3k_response.html`; // for a debuggable react version of the page
}
private handleTone3000DownloadComplete() {
if (this.popupWindow) {
if (!this.popupWindow.closed) {
this.popupWindow.close();
}
this.popupWindow = null;
}
this.progress.transferring = false;
this.model.hideTone3000DownloadStatus();
}
closeTone3000Popup() {
this.handleTone3000DownloadComplete();
}
private handleTone3000DownloadError(errorMessage: string) {
this.progress.transferring = false;
this.handleTone3000DownloadComplete();
this.model.showAlert(errorMessage);
}
private downloadPath: string = "";
private downloadType: Tone3000DownloadType = Tone3000DownloadType.Nam;
private checkForCancel(): boolean {
if (this.progress.cancelled) {
this.handleTone3000DownloadComplete();
return true;
}
return false;
}
private launchInstance = 0;
public async launchTone3000Popup(
downloadType: Tone3000DownloadType,
downloadPath: string,
options?: {
userName?: string;
}
): Promise<void> {
if (this.progress.transferring) {
return;
}
this.closeTone3000Dialog();
// online
this.downloadPath = downloadPath;
this.downloadType = downloadType;
let popupWidth = Math.floor(window.innerWidth * 0.8);
let popupHeight = Math.floor(window.innerHeight * 0.8);
if (window.innerWidth < 410) {
popupWidth = 400;
}
this.model.showTone3000DownloadStatus();
let launchInstance = ++this.launchInstance;
let cancelled = () => {
return (launchInstance !== this.launchInstance) || this.popupWindow == null || this.popupWindow.closed;
}
try {
let t3kPceParams: Tone3000PkceParams;
let pkceParams: PkceParams;
if (USE_SERVER_PCKE) {
t3kPceParams = await this.model.makeTone3000Pkce(this.redirectUrl());
pkceParams = {
codeChallenge: t3kPceParams.codeChallenge,
codeVerifier: t3kPceParams.codeVerifier,
state: t3kPceParams.state
}
setServerPkceParams(pkceParams)
} else {
pkceParams = await buildPkceParams();
t3kPceParams = {
codeChallenge: pkceParams.codeChallenge,
codeVerifier: pkceParams.codeVerifier,
state: pkceParams.state,
publishableKey: PUBLISHABLE_KEY,
redirectUrl: this.redirectUrl()
};
// verify that the server matches.
let testcodeChallenge = await this.model.sha256Base64url(pkceParams.codeVerifier);
if (testcodeChallenge !== pkceParams.codeChallenge) {
throw new Error("model.sha256Base64url produces incorrect results. ");
}
}
this.pkceParams = pkceParams;
this.popupWindow = await startSelectFlowPopup(
pkceParams,
PUBLISHABLE_KEY,
this.redirectUrl(),
{
architecture: (downloadType === Tone3000DownloadType.Nam && this.model.tone3000_A2_models) ? 2: undefined,
platform: downloadType === Tone3000DownloadType.CabIr ? Platform.Ir : Platform.Nam,
menubar: true,
width: popupWidth,
height: popupHeight,
userName: options?.userName
},
);
if (!this.popupWindow) {
console.error("Failed to open TONE3000 dialog popup window.");
throw new Error("Cannot open popup window.");
}
// No procedure for detecting 404, so let's ping the server to see if it's reachable, and cancel if it's not.
try {
let response = await fetch(TONE3000_PING_URL, { method: "HEAD", cache: "no-cache" });
if (cancelled()) return;
if (!response.ok) {
// offline
throw new Error("Unable to connect to the TONE3000 server.");
}
} catch (error) {
throw new Error("Not online. Unable to connect to the TONE3000 server. Your client device must have access to the internet to use this feature..");
};
let pipedalServerCanReachTone3000 = false;
// check to see that the PiPedal server can reach TONE3000.
try {
pipedalServerCanReachTone3000 = await this.model.pingTone3000Server();
if (cancelled()) return;
} catch (error) {
}
if (!pipedalServerCanReachTone3000) {
throw new Error("The PiPedal server cannot reach a TONE3000 server. The PiPedal server must have access to the internet to use this feature.");
}
while (true) {
await new Promise(resolve => setTimeout(resolve, 500));
if (launchInstance != this.launchInstance) {
// A new dialog launch has been initiated. Stop monitoring this one.
return;
}
if (this.popupWindow === null || this.popupWindow.closed) {
this.closeTone3000Dialog();
return;
}
}
} catch (error) {
this.handleTone3000DownloadError(getErrorMessage(error));
}
}
cancelDownload(handle: number): void {
this.closeTone3000Dialog();
if (this.progress.handle == handle) {
this.progress.cancelled = true;
}
}
public closeTone3000Dialog(): void {
if (this.popupWindow !== null) {
this.popupWindow.close();
this.popupWindow = null;
if (!this.progress.transferring) {
this.handleTone3000DownloadComplete();
}
}
}
}