feat: enable '*' and 'posix' OS & Arch for source releases + update GitHubishes

.jshintrc

@@ -1,4 +1,7 @@
 {
+  "globals": {
+    "AbortController": false
+  },
   "browser": true,
   "node": true,
   "esversion": 11,

_common/github-source.js

@@ -1,133 +1,40 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubSource = module.exports;
+
+let GitHubishSource = require('./githubish-source.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
- *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
  */
-async function getDistributables(
-  request,
+GitHubSource.getDistributables = async function ({
   owner,
   repo,
-  oses,
-  arches,
   baseurl = 'https://api.github.com',
-) {
-  if (!owner) {
-    return Promise.reject('missing owner for repo');
-  }
-  if (!repo) {
-    return Promise.reject('missing repo name');
-  }
-
-  let req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // TODO make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  for (let release of gHubResp) {
-    // TODO tags aren't always semver / sensical
-    let tag = release['tag_name'];
-    let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-    let channel = 'stable';
-    if (release['prerelease']) {
-      channel = 'beta';
-    }
-    let date = release['published_at'] || '';
-    date = date.replace(/T.*/, '');
-
-    let urls = [release.tarball_url, release.zipball_url];
-    for (let url of urls) {
-      let resp = await request({
-        method: 'HEAD',
-        followRedirect: true,
-        followAllRedirects: true,
-        followOriginalHttpMethod: true,
-        url: url,
-        stream: true,
-      });
-      // Workaround for bug where method changes to GET
-      resp.destroy();
-
-      // content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
-      let name = resp.headers['content-disposition'].replace(
-        /.*filename=([^;]+)(;|$)/,
-        '$1',
-      );
-      all.releases.push({
-        name: name,
-        version: tag,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: resp.request.uri.href,
-      });
-    }
-  }
-
-  if (oses) {
-    return combinate(all, oses, arches);
-  }
-
-  return all;
-}
-
-function combinate(all, oses, arches) {
-  let releases = all.releases;
-  // ex: arches = ['amd64', 'arm64', 'armv7l', 'armv6l', 'x86'];
-  // ex: oses = ['macos', 'linux', 'bsd', 'posix'];
-
-  let combos = [];
-  for (let release of releases) {
-    for (let arch of arches) {
-      for (let os of oses) {
-        let combo = {
-          arch: arch,
-          os: os,
-        };
-        let rel = Object.assign({}, release, combo);
-        combos.push(rel);
-      }
-    }
-  }
-  all.releases = combos;
-
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
+}) {
+  let all = await GitHubishSource.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
-}
-
-module.exports = getDistributables;
+};
 
 if (module === require.main) {
-  getDistributables(
-    require('@root/request'),
-    'BeyondCodeBootcamp',
-    'DuckDNS.sh',
-  ).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
+  GitHubSource.getDistributables(null, 'BeyondCodeBootcamp', 'DuckDNS.sh').then(
+    function (all) {
+      console.info(JSON.stringify(all, null, 2));
+    },
+  );
 }

_common/github.js

@@ -14,7 +14,7 @@ let GitHubish = require('./githubish.js');
  * @param {String} [username]
  * @param {String} [token]
  */
-async function getDistributables(
+module.exports = async function (
   _request,
   owner,
   repo,
@@ -30,12 +30,13 @@ async function getDistributables(
     token,
   });
   return all;
-}
+};
 
-module.exports = getDistributables;
+let GitHub = module.exports;
+GitHub.getDistributables = module.exports;
 
 if (module === require.main) {
-  getDistributables(null, 'BurntSushi', 'ripgrep').then(function (all) {
+  GitHub.getDistributables(null, 'BurntSushi', 'ripgrep').then(function (all) {
     console.info(JSON.stringify(all, null, 2));
   });
 }

_common/githubish-source.js

@@ -0,0 +1,153 @@
+'use strict';
+
+let GitHubishSource = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (source tarball & zip)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubishSource.getDistributables = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp = await fetch(url, opts);
+  if (!resp.ok) {
+    let headers = Array.from(resp.headers);
+    console.error('Bad Resp Headers:', headers);
+    let text = await resp.text();
+    console.error('Bad Resp Body:', text);
+    let msg = `failed to fetch releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  let respText = await resp.text();
+  let gHubResp;
+  try {
+    gHubResp = JSON.parse(respText);
+  } catch (e) {
+    console.error('Bad Resp JSON:', respText);
+    console.error(e.message);
+    let msg = `failed to parse releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  let all = {
+    releases: [],
+    // TODO make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  for (let release of gHubResp) {
+    let dists = GitHubishSource.releaseToDistributables(release);
+    for (let dist of dists) {
+      let updates =
+        await GitHubishSource.followDistributableDownloadAttachment(dist);
+      Object.assign(dist, updates);
+      all.releases.push(dist);
+    }
+  }
+
+  return all;
+};
+
+GitHubishSource.releaseToDistributables = function (ghRelease) {
+  let ghTag = ghRelease['tag_name']; // TODO tags aren't always semver / sensical
+  let lts = /(\b|_)(lts)(\b|_)/.test(ghRelease['tag_name']);
+  let channel = 'stable';
+  if (ghRelease['prerelease']) {
+    channel = 'beta';
+  }
+  let date = ghRelease['published_at'] || '';
+  date = date.replace(/T.*/, '');
+
+  let urls = [ghRelease.tarball_url, ghRelease.zipball_url];
+  let dists = [];
+  for (let url of urls) {
+    dists.push({
+      name: '',
+      version: ghTag,
+      lts: lts,
+      channel: channel,
+      date: date,
+      os: '*',
+      arch: '*',
+      libc: '',
+      ext: '',
+      download: url,
+    });
+  }
+
+  return dists;
+};
+
+GitHubishSource.followDistributableDownloadAttachment = async function (dist) {
+  let abortCtrl = new AbortController();
+  let resp = await fetch(dist.download, {
+    method: 'HEAD',
+    redirect: 'follow',
+    signal: abortCtrl.signal,
+  });
+  let headers = Object.fromEntries(resp.headers);
+
+  // Workaround for bug where METHOD changes to GET
+  abortCtrl.abort();
+  await resp.text().catch(function (err) {
+    if (err.name !== 'AbortError') {
+      throw err;
+    }
+  });
+
+  // ex: content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  //  => BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  let name = headers['content-disposition'].replace(
+    /.*filename=([^;]+)(;|$)/,
+    '$1',
+  );
+  let download = resp.url;
+
+  return { name, download };
+};
+
+if (module === require.main) {
+  GitHubishSource.getDistributables({
+    owner: 'BeyondCodeBootcamp',
+    repo: 'DuckDNS.sh',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_webi/builds-cacher.js

@@ -726,9 +726,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     if (hostTarget.os === 'windows') {
       oses = ['ANYOS', 'windows'];
     } else if (hostTarget.os === 'android') {
-      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
+      oses = ['ANYOS', 'posix_2017', 'posix_2024', 'android', 'linux'];
     } else {
-      oses = ['ANYOS', 'posix_2017', hostTarget.os];
+      oses = ['ANYOS', 'posix_2017', 'posix_2024', hostTarget.os];
     }
 
     let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};

_webi/normalize.js

@@ -211,7 +211,8 @@ function normalize(all) {
       // won't match:
       //   - v1.0beta
       //   - v1.0-beta1b
-      let isBetaRe = /(\b|_)(preview|rc|beta|alpha)(\d+)(\b|_)/;
+      let isBetaRe =
+        /(\b|_)(alpha|beta|dev|developer|prev|preview|rc)(\d+)(\b|_)/;
       let isBeta = isBetaRe.test(rel.name);
       if (isBeta) {
         rel.channel = 'beta';

_webi/serve-installer.js

@@ -132,10 +132,23 @@ InstallerServer.helper = async function ({
     formats: projInfo.formats,
   };
 
+  // TODO .findMatchingPackages() should probably account for this
   let hasOs = projInfo.oses.includes(hostTarget.os);
+  let maybePosix = !hasOs && hostTarget.os !== 'windows';
+  if (maybePosix) {
+    let posixes = ['posix_2017', 'posix_2024'];
+    for (let posixYear of posixes) {
+      let hasPosix = projInfo.oses.includes(posixYear);
+      if (hasPosix) {
+        hasOs = true;
+        break;
+      }
+    }
+  }
   if (!hasOs) {
     hasOs = projInfo.oses.includes('ANYOS');
   }
+
   if (!hasOs) {
     let pkg1 = Object.assign(buildTargetInfo, errPackage);
     return [pkg1, tmplParams];