|
@@ -0,0 +1,117 @@
|
|
|
+const fs = require("fs");
|
|
|
+const axios = require("axios");
|
|
|
+const moment = require("moment");
|
|
|
+const {
|
|
|
+ ch,
|
|
|
+ time,
|
|
|
+ sleep,
|
|
|
+ loadResults,
|
|
|
+ headAsset,
|
|
|
+ sendResult,
|
|
|
+ printFailed,
|
|
|
+ writeTable,
|
|
|
+ query,
|
|
|
+} = require("./util");
|
|
|
+
|
|
|
+// config
|
|
|
+const QN = `https://ipfs.joystreamstats.live/graphql`;
|
|
|
+const resultsUrl = `https://joystreamstats.live/api/v1/bags/status`;
|
|
|
+const resultsFile = `./bag_availability.json`;
|
|
|
+const getUrl = (metadata) => metadata?.nodeEndpoint;
|
|
|
+const testProviders = async (channelId, object, sUrls, dUrls) => {
|
|
|
+ const storage = await Promise.all(
|
|
|
+ sUrls
|
|
|
+ .filter((url) => url)
|
|
|
+ .map((url) =>
|
|
|
+ headAsset(object.id, url, url + `api/v1/files/${object.id}`)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ const distribution = await Promise.all(
|
|
|
+ dUrls.map((url) =>
|
|
|
+ headAsset(object.id, url, url + `api/v1/assets/${object.id}`)
|
|
|
+ )
|
|
|
+ );
|
|
|
+ return [storage, distribution];
|
|
|
+};
|
|
|
+const testBags = async (bags, objects = [], rapid = false) => {
|
|
|
+ console.debug(
|
|
|
+ time(),
|
|
|
+ `Starting rapid test`,
|
|
|
+ objects.length && `for ${objects}`
|
|
|
+ );
|
|
|
+ const start = new Date();
|
|
|
+ let results = [];
|
|
|
+ for (const bag of bags.sort((a, b) => ch(b.id) - ch(a.id))) {
|
|
|
+ if (!bag.objects) continue;
|
|
|
+ const channelId = ch(bag.id);
|
|
|
+ const sUrls = bag.storageBuckets.map((sb) => getUrl(sb.operatorMetadata));
|
|
|
+ const dUrls = bag.distributionBuckets.map((db) =>
|
|
|
+ getUrl(db.operators[0]?.metadata)
|
|
|
+ );
|
|
|
+ for (const object of bag.objects) {
|
|
|
+ //if (results.length) continue; // TODO
|
|
|
+ if (objects.length && !objects.includes(object.id)) continue;
|
|
|
+ const [storage, distribution] = await testProviders(
|
|
|
+ ch(bag.id),
|
|
|
+ object,
|
|
|
+ sUrls,
|
|
|
+ dUrls
|
|
|
+ );
|
|
|
+ const sFailed = storage.filter((b) => b.status !== `success`);
|
|
|
+ const dFailed = distribution.filter((b) => b.status !== `success`);
|
|
|
+ const SP =
|
|
|
+ `${storage.length - sFailed.length}/${storage.length} ` +
|
|
|
+ (sFailed.length ? `( ` + printFailed(sFailed) + ` )` : "OK");
|
|
|
+ const DP =
|
|
|
+ `${distribution.length - dFailed.length}/${distribution.length} ` +
|
|
|
+ (dFailed.length ? `( ` + printFailed(dFailed) + ` )` : "OK");
|
|
|
+ console.log(`${time()} ${channelId} ${object.id} SP:${SP} DP:${DP}`);
|
|
|
+ const result = { channelId, storage, distribution };
|
|
|
+ sendResult(result, resultsUrl);
|
|
|
+ results.push(result);
|
|
|
+ }
|
|
|
+ if (!rapid) await sleep(Math.randon * 10.0);
|
|
|
+ }
|
|
|
+ const duration = (moment().diff(start) / 1000).toFixed(3) + `s`;
|
|
|
+ console.log(time(), `Finished rapid test in `, duration);
|
|
|
+ if (!objects.length) {
|
|
|
+ fs.writeFileSync(resultsFile, JSON.stringify(results));
|
|
|
+ console.log(time(), `Wrote results to ${resultsFile} `);
|
|
|
+ }
|
|
|
+};
|
|
|
+
|
|
|
+// start
|
|
|
+const bagIds = process.argv.slice(2);
|
|
|
+if (bagIds.length) console.log(`selected bags`, bagIds);
|
|
|
+const old = loadResults(resultsFile);
|
|
|
+console.debug(time(), `Fetching bags with buckets and objects\n`, query);
|
|
|
+axios
|
|
|
+ .post(QN, { query })
|
|
|
+ .then(({ data }) => {
|
|
|
+ const bags = data.data.storageBags;
|
|
|
+ if (!bags.length)
|
|
|
+ return console.error(time(), `No bags received.`, data.error);
|
|
|
+ console.log(time(), `Received list with ${bags.length} bags.`);
|
|
|
+ //bags.forEach((b) => printBagBuckets(b));
|
|
|
+ let selected = bags;
|
|
|
+ if (bagIds.length) {
|
|
|
+ console.log(time(), `Selecting bags with id`, bagIds);
|
|
|
+ selected = bags.filter(
|
|
|
+ (b) => !bagIds.length || bagIds.find((id) => id === ch(b.id))
|
|
|
+ );
|
|
|
+ }
|
|
|
+ if (old.length) {
|
|
|
+ console.log(time(), `Testing formerly failed assets`);
|
|
|
+ const ids = old.reduce(
|
|
|
+ (ids, bag) =>
|
|
|
+ ids.concat(
|
|
|
+ ...bag.storage.map((r) => r.objectId),
|
|
|
+ ...bag.distribution.map((r) => r.objectId)
|
|
|
+ ),
|
|
|
+ []
|
|
|
+ );
|
|
|
+ console.log(ids);
|
|
|
+ testBags(selected, ids);
|
|
|
+ } else testBags(selected);
|
|
|
+ })
|
|
|
+ .catch((e) => console.error(e.message, e.response?.data));
|