community-repo/scripts/test-providers/channels.js

const fs = require("fs");
const axios = require("axios");
const moment = require("moment");
const {
  ch,
  time,
  sleep,
  loadResults,
  headAsset,
  sendResult,
  printFailed,
  writeTable,
  query,
} = require("./util");

// config
const QN = `https://ipfs.joystreamstats.live/graphql`;
const resultsUrl = `https://joystreamstats.live/api/v1/bags/status`;
const resultsFile = `./bag_availability.json`;
const getUrl = (metadata) => metadata?.nodeEndpoint;
const testProviders = async (channelId, object, sUrls, dUrls) => {
  const storage = await Promise.all(
    sUrls
      .filter((url) => url)
      .map((url) =>
        headAsset(object.id, url, url + `api/v1/files/${object.id}`)
      )
  );
  const distribution = await Promise.all(
    dUrls.map((url) =>
      headAsset(object.id, url, url + `api/v1/assets/${object.id}`)
    )
  );
  return [storage, distribution];
};
const testBags = async (bags, objects = [], rapid = false) => {
  console.debug(
    time(),
    `Starting rapid test`,
    objects.length && `for ${objects}`
  );
  const start = new Date();
  let results = [];
  for (const bag of bags.sort((a, b) => ch(b.id) - ch(a.id))) {
    if (!bag.objects) continue;
    const channelId = ch(bag.id);
    const sUrls = bag.storageBuckets.map((sb) => getUrl(sb.operatorMetadata));
    const dUrls = bag.distributionBuckets.map((db) =>
      getUrl(db.operators[0]?.metadata)
    );
    for (const object of bag.objects) {
      //if (results.length) continue; // TODO
      if (objects.length && !objects.includes(object.id)) continue;
      const [storage, distribution] = await testProviders(
        ch(bag.id),
        object,
        sUrls,
        dUrls
      );
      const sFailed = storage.filter((b) => b.status !== `success`);
      const dFailed = distribution.filter((b) => b.status !== `success`);
      const SP =
        `${storage.length - sFailed.length}/${storage.length} ` +
        (sFailed.length ? `( ` + printFailed(sFailed) + ` )` : "OK");
      const DP =
        `${distribution.length - dFailed.length}/${distribution.length} ` +
        (dFailed.length ? `( ` + printFailed(dFailed) + ` )` : "OK");
      console.log(`${time()} ${channelId} ${object.id} SP:${SP} DP:${DP}`);
      const result = { channelId, storage, distribution };
      sendResult(result, resultsUrl);
      results.push(result);
    }
    if (!rapid) await sleep(Math.randon * 10.0);
  }
  const duration = (moment().diff(start) / 1000).toFixed(3) + `s`;
  console.log(time(), `Finished rapid test in `, duration);
  if (!objects.length) {
    fs.writeFileSync(resultsFile, JSON.stringify(results));
    console.log(time(), `Wrote results to ${resultsFile} `);
  }
};

// start
const bagIds = process.argv.slice(2);
if (bagIds.length) console.log(`selected bags`, bagIds);
const old = loadResults(resultsFile);
console.debug(time(), `Fetching bags with buckets and objects\n`, query);
axios
  .post(QN, { query })
  .then(({ data }) => {
    const bags = data.data.storageBags;
    if (!bags.length)
      return console.error(time(), `No bags received.`, data.error);
    console.log(time(), `Received list with ${bags.length} bags.`);
    //bags.forEach((b) => printBagBuckets(b));
    let selected = bags;
    if (bagIds.length) {
      console.log(time(), `Selecting bags with id`, bagIds);
      selected = bags.filter(
        (b) => !bagIds.length || bagIds.find((id) => id === ch(b.id))
      );
    }
    if (old.length) {
      console.log(time(), `Testing formerly failed assets`);
      const ids = old.reduce(
        (ids, bag) =>
          ids.concat(
            ...bag.storage.map((r) => r.objectId),
            ...bag.distribution.map((r) => r.objectId)
          ),
        []
      );
      console.log(ids);
      testBags(selected, ids);
    } else testBags(selected);
  })
  .catch((e) => console.error(e.message, e.response?.data));