Rewrite http utils; properly handle (& retry on) network errors

src/main/utils/http.ts

@@ -5,41 +5,54 @@
  */
 
 import { createWriteStream } from "fs";
-import type { IncomingMessage } from "http";
-import { get, RequestOptions } from "https";
-import { finished } from "stream/promises";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
+import { setTimeout } from "timers/promises";
 
-export async function downloadFile(url: string, file: string, options: RequestOptions = {}) {
-    const res = await simpleReq(url, options);
-    await finished(
-        res.pipe(
-            createWriteStream(file, {
-                autoClose: true
-            })
-        )
+interface FetchieOptions {
+    retryOnNetworkError?: boolean;
+}
+
+export async function downloadFile(url: string, file: string, options: RequestInit = {}, fetchieOpts?: FetchieOptions) {
+    const res = await fetchie(url, options, fetchieOpts);
+    await pipeline(
+        // @ts-expect-error odd type error
+        Readable.fromWeb(res.body!),
+        createWriteStream(file, {
+            autoClose: true
+        })
     );
 }
 
-export function simpleReq(url: string, options: RequestOptions = {}) {
-    return new Promise<IncomingMessage>((resolve, reject) => {
-        get(url, options, res => {
-            const { statusCode, statusMessage, headers } = res;
-            if (statusCode! >= 400) return void reject(`${statusCode}: ${statusMessage} - ${url}`);
-            if (statusCode! >= 300) return simpleReq(headers.location!, options).then(resolve).catch(reject);
+const ONE_MINUTE_MS = 1000 * 60;
 
-            resolve(res);
-        });
-    });
-}
-
-export async function simpleGet(url: string, options: RequestOptions = {}) {
-    const res = await simpleReq(url, options);
-
-    return new Promise<Buffer>((resolve, reject) => {
-        const chunks = [] as Buffer[];
-
-        res.once("error", reject);
-        res.on("data", chunk => chunks.push(chunk));
-        res.once("end", () => resolve(Buffer.concat(chunks)));
-    });
+export async function fetchie(url: string, options?: RequestInit, { retryOnNetworkError }: FetchieOptions = {}) {
+    let res: Response | undefined;
+
+    try {
+        res = await fetch(url, options);
+    } catch (err) {
+        if (retryOnNetworkError) {
+            console.error("Failed to fetch", url + ".", "Gonna retry with backoff.");
+
+            for (let tries = 0, delayMs = 500; tries < 20; tries++, delayMs = Math.min(2 * delayMs, ONE_MINUTE_MS)) {
+                await setTimeout(delayMs);
+                try {
+                    res = await fetch(url, options);
+                    break;
+                } catch {}
+            }
+        }
+
+        if (!res) throw new Error(`Failed to fetch ${url}\n${err}`);
+    }
+
+    if (res.ok) return res;
+
+    let msg = `Got non-OK response for ${url}: ${res.status} ${res.statusText}`;
+
+    const reason = await res.text().catch(() => "");
+    if (reason) msg += `\n${reason}`;
+
+    throw new Error(msg);
 }