report generator fixes

contributions/tech/report-generator/report-template.md

@@ -26,11 +26,11 @@ This is a report which explains the current state of the Joystream network in nu
 ### 2.2 Fiat Pool
 | Property            | Start Block, USD | End Block, USD | % Change |
 |---------------------|--------------|--------------|----------|
-| USD Pool |  {startDollarPool} | {endDollarPool} | {dollarPoolPctChange} |
+| USD Pool | {startDollarPool} | {endDollarPool} | {dollarPoolPctChange} |
 
 {dollarPoolRefills}
 
-### 2.3 Mints 
+### 2.3 Mints
 | Property                    | Start Block           | End Block | % Change |
 |-----------------------------|-----------------------|--------------|----------|
 | Council Mint Total Minted   | {startCouncilMinted}  | {endCouncilMinted} |{percNewCouncilMinted} |
@@ -64,10 +64,10 @@ Negative value indicates deflation
 ### 4.1 Validator Information
 * Block generation time (average): {avgBlockProduction}
 
-| Property                    | Start Block | End Block | % Change |
-|-----------------------------|--------------|--------------|----------|
-| Number of Validators       |  {startValidators} | {endValidators} | {percValidators} |
-| Validator Total Stake       | {startValidatorsStake} | {endValidatorsStake} | {percNewValidatorsStake} |
+| Property                   | Start Block | End Block | % Change |
+|----------------------------|--------------|--------------|----------|
+| Number of Validators       | {startValidators} | {endValidators} | {percValidators} |
+| Validator Total Stake      | {startValidatorsStake} | {endValidatorsStake} | {percNewValidatorsStake} |
 
 
 ### 4.2 Storage Role
@@ -89,7 +89,7 @@ Negative value indicates deflation
 | Property                | Start Block | End Block | % Change |
 |-------------------------|--------------|--------------|----------|
 | Number of Operations Workers      | {startOperationsWorkers} | {endOperationsWorkers} | {percNewOperationsWorkers} |
-| Total Operations Stake (workers + lead)  | {startOperationsStake} |  {endOperationsStake} | {percNewOperationstake} |
+| Total Operations Stake (workers + lead) | {startOperationsStake} | {endOperationsStake} | {percNewOperationstake} |
 
 {operations}
 
@@ -103,7 +103,7 @@ Negative value indicates deflation
 | Property                | Start Block | End Block | % Change |
 |-------------------------|--------------|--------------|----------|
 | Number of uploads       | {startMedia} | {endMedia}  |  {percNewMedia} |
-| Size of content (MB)        |  {startUsedSpace} |  {endUsedSpace} | {percNewUsedSpace} |
+| Size of content (MB)    |  {startUsedSpace} |  {endUsedSpace} | {percNewUsedSpace} |
 | Number of channels      |  {startChannels} | {endChannels} | {percNewChannels} |
 
 ### 5.3 Forum Activity

contributions/tech/report-generator/src/StatisticsCollector.ts

@@ -45,6 +45,7 @@ import {
   connectApi,
   getBlock,
   getBlockHash,
+  getHead,
   getTimestamp,
   getIssuance,
   getEra,
@@ -144,9 +145,15 @@ export class StatisticsCollector {
     endBlock: number
   ): Promise<Statistics> {
     this.api = await connectApi(PROVIDER_URL);
+    const diff = endBlock - Number(await getHead(this.api));
+    if (diff > 0) {
+      console.log(`End Block is greater than Head, wait ${diff} blocks.`);
+      return this.statistics;
+    }
 
     let startHash: Hash = await getBlockHash(this.api, startBlock);
     let endHash: Hash = await getBlockHash(this.api, endBlock);
+
     let dateStart = momentToString(await getTimestamp(this.api, startHash));
     let dateEnd = momentToString(await getTimestamp(this.api, endHash));
     this.saveStats({
@@ -158,62 +165,65 @@ export class StatisticsCollector {
       percNewBlocks: getPercent(startBlock, endBlock),
     });
 
-    await this.buildBlocksEventCache(startBlock, endBlock);
-    eventStats(this.blocksEventsCache);
-    await this.fillTokenGenerationInfo(
-      startBlock,
-      endBlock,
-      startHash,
-      endHash
-    );
-    await this.fillMintsInfo(startHash, endHash);
-    await this.fillCouncilInfo(startHash, endHash);
-    await this.fillCouncilElectionInfo(startBlock);
-    await this.fillValidatorInfo(startHash, endHash);
-    await this.fillStorageProviderInfo(
-      startBlock,
-      endBlock,
-      startHash,
-      endHash
-    );
-    await this.fillCuratorInfo(startHash, endHash);
-    await this.fillOperationsInfo(startBlock, endBlock, startHash, endHash);
-    await this.fillMembershipInfo(startHash, endHash);
-    await this.fillMediaUploadInfo(startHash, endHash);
-    await this.fillForumInfo(startHash, endHash);
-    await this.getFiatEvents(startBlock, endBlock);
-
+    // run long running tasks in parallel first
+    await Promise.all([
+      this.buildBlocksEventCache(startBlock, endBlock).then(() =>
+        this.fillStats(startBlock, endBlock, startHash, endHash)
+      ),
+      this.getFiatEvents(startBlock, endBlock),
+      this.fillMediaUploadInfo(startHash, endHash),
+    ]);
     this.api.disconnect();
     return this.statistics;
   }
 
+  fillStats(
+    startBlock: number,
+    endBlock: number,
+    startHash: Hash,
+    endHash: Hash
+  ): Promise<void[]> {
+    eventStats(this.blocksEventsCache); // print event stats
+    return Promise.all([
+      this.fillTokenGenerationInfo(startBlock, endBlock, startHash, endHash),
+      this.fillMintsInfo(startHash, endHash),
+      this.fillCouncilInfo(startHash, endHash),
+      this.fillCouncilElectionInfo(startBlock),
+      this.fillValidatorInfo(startHash, endHash),
+      this.fillStorageProviderInfo(startBlock, endBlock, startHash, endHash),
+      this.fillCuratorInfo(startHash, endHash),
+      this.fillOperationsInfo(startBlock, endBlock, startHash, endHash),
+      this.fillMembershipInfo(startHash, endHash),
+      this.fillForumInfo(startHash, endHash),
+    ]);
+  }
+
   async getApprovedBounties(): Promise<Bounty[]> {
     try {
       await fs.access(SPENDING_PROPOSALS_CATEGORIES_FILE, constants.R_OK);
     } catch {
-      console.warn("File with the spending proposal categories not found");
+      console.warn("File with spending proposal categories not found.");
     }
-
     const fileContent = await fs.readFile(SPENDING_PROPOSALS_CATEGORIES_FILE);
-    let rawBounties = parse(fileContent);
-    rawBounties.shift();
-    rawBounties = rawBounties.filter((line: string[]) => line[8] == "Bounties");
-
-    let bounties = rawBounties.map((rawBounty: any) => {
-      return new Bounty(
-        rawBounty[0],
-        rawBounty[1],
-        rawBounty[2],
-        rawBounty[3],
-        rawBounty[4],
-        rawBounty[5]
-      );
-    });
-
-    return bounties.filter(
-      (bounty: Bounty) =>
-        bounty.status == "Approved" && bounty.testnet == "Antioch"
-    );
+    const proposals = parse(fileContent).slice(1);
+    console.log(`Loaded ${proposals.length} proposals.`);
+    return proposals
+      .filter(
+        (line: string[]) =>
+          line[0] === "Antioch" &&
+          line[3] === "Approved" &&
+          line[8] === "Bounties"
+      )
+      .map((bounty: string[]) => {
+        return new Bounty(
+          bounty[0],
+          Number(bounty[1]),
+          bounty[2],
+          bounty[3],
+          Number(bounty[4]),
+          Number(bounty[5])
+        );
+      });
   }
 
   fillSudoSetBalance() {
@@ -251,29 +261,27 @@ export class StatisticsCollector {
     const blocks = this.filterCache(filterMethods.finalizedSpendingProposals);
     const spendingProposals: SpendingProposal[] =
       await getFinalizedSpendingProposals(this.api, blocks);
+
     let bountiesTotalPaid = 0;
-    if (bounties) {
-      for (let bounty of bounties) {
-        const bountySpendingProposal = spendingProposals.find(
-          (spendingProposal) => spendingProposal.id == bounty.proposalId
-        );
-        if (bountySpendingProposal)
-          bountiesTotalPaid += bountySpendingProposal.amount;
-      }
-      this.saveStats({ bountiesTotalPaid });
+    for (let bounty of bounties) {
+      const bountySpendingProposal = spendingProposals.find(
+        (spendingProposal) => spendingProposal.id == bounty.proposalId
+      );
+      if (bountySpendingProposal)
+        bountiesTotalPaid += bountySpendingProposal.amount;
     }
 
     if (!bountiesTotalPaid) {
       console.warn(
-        `No bounties found in ${SPENDING_PROPOSALS_CATEGORIES_FILE}, trying to find spending proposals of bounties, please check the values!...`
+        `No bounties in selected period. Need to update ${SPENDING_PROPOSALS_CATEGORIES_FILE}?\nLooking for spending proposals titled "bounty":`
       );
-      for (const spendingProposal of spendingProposals) {
-        if (spendingProposal.title.toLowerCase().includes("bounty")) {
-          bountiesTotalPaid += spendingProposal.amount;
-        }
+      for (const { title, amount } of spendingProposals) {
+        if (!title.toLowerCase().includes("bounty")) continue;
+        bountiesTotalPaid += amount;
+        console.log(` - ${title}: ${amount}`);
       }
-      this.saveStats({ bountiesTotalPaid });
     }
+    this.saveStats({ bountiesTotalPaid });
 
     let roundNrBlocks = endBlock - startBlock;
     const spendingProposalsTotal = spendingProposals.reduce(
@@ -388,7 +396,6 @@ export class StatisticsCollector {
       if (hired) earnedBefore = hired.reward.total_reward_received.toNumber();
       workers += getWorkerRow(worker, earnedBefore);
     });
-    const header = `| # | Member | Status | tJOY / Block | M tJOY Term | M tJOY total |\n|--|--|--|--|--|--|\n`;
     const groupTag =
       workingGroup === `storage`
         ? `storageProviders`
@@ -397,7 +404,10 @@ export class StatisticsCollector {
         : workingGroup === `operations`
         ? `operations`
         : ``;
-    this.saveStats({ [groupTag]: header + workers });
+    if (workers.length) {
+      const header = `| # | Member | Status | tJOY / Block | M tJOY Term | M tJOY total |\n|--|--|--|--|--|--|\n`;
+      this.saveStats({ [groupTag]: header + workers });
+    } else this.saveStats({ [groupTag]: `` });
 
     info.rewards = await this.computeReward(
       roundNrBlocks,
@@ -683,38 +693,41 @@ export class StatisticsCollector {
   }
 
   async fillMediaUploadInfo(startHash: Hash, endHash: Hash): Promise<void> {
-    const startMedia = await getNextVideo(this.api, startHash);
-    const endMedia = await getNextVideo(this.api, endHash);
-    const startChannels = await getNextChannel(this.api, startHash);
-    const endChannels = await getNextChannel(this.api, endHash);
+    console.log(`Collecting Media stats`);
+    const startMedia = Number(await getNextVideo(this.api, startHash));
+    const endMedia = Number(await getNextVideo(this.api, endHash));
+    const startChannels = Number(await getNextChannel(this.api, startHash));
+    const endChannels = Number(await getNextChannel(this.api, endHash));
 
     // count size
     let startUsedSpace = 0;
     let endUsedSpace = 0;
     const startBlock = await getBlock(this.api, startHash);
     const endBlock = await getBlock(this.api, endHash);
-    const dataObjects: Map<ContentId, DataObject> = await getDataObjects(
-      this.api
-    );
-    for (let [key, dataObject] of dataObjects) {
-      const added = dataObject.added_at.block.toNumber();
-      const start = startBlock.block.header.number.toNumber();
-      const end = endBlock.block.header.number.toNumber();
-      if (added < start)
-        startUsedSpace += dataObject.size_in_bytes.toNumber() / 1024 / 1024;
-      if (added < end)
-        endUsedSpace += dataObject.size_in_bytes.toNumber() / 1024 / 1024;
-    }
-    this.saveStats({
-      startMedia,
-      endMedia,
-      percNewMedia: getPercent(startMedia, endMedia),
-      startChannels,
-      endChannels,
-      percNewChannels: getPercent(startChannels, endChannels),
-      startUsedSpace: Number(startUsedSpace.toFixed(2)),
-      endUsedSpace: Number(endUsedSpace.toFixed(2)),
-      percNewUsedSpace: getPercent(startUsedSpace, endUsedSpace),
+    getDataObjects(this.api).then((dataObjects: Map<ContentId, DataObject>) => {
+      for (let [key, dataObject] of dataObjects) {
+        const added = dataObject.added_at.block.toNumber();
+        const start = startBlock.block.header.number.toNumber();
+        const end = endBlock.block.header.number.toNumber();
+
+        if (added < start)
+          startUsedSpace += dataObject.size_in_bytes.toNumber() / 1024 / 1024;
+        if (added < end)
+          endUsedSpace += dataObject.size_in_bytes.toNumber() / 1024 / 1024;
+      }
+      if (!startUsedSpace || !endUsedSpace)
+        console.log(`space start, end`, startUsedSpace, endUsedSpace);
+      this.saveStats({
+        startMedia,
+        endMedia,
+        percNewMedia: getPercent(startMedia, endMedia),
+        startChannels,
+        endChannels,
+        percNewChannels: getPercent(startChannels, endChannels),
+        startUsedSpace: Number(startUsedSpace.toFixed(2)),
+        endUsedSpace: Number(endUsedSpace.toFixed(2)),
+        percNewUsedSpace: getPercent(startUsedSpace, endUsedSpace),
+      });
     });
   }
 
@@ -894,7 +907,6 @@ export class StatisticsCollector {
       console.log("Cache file found, loading it...");
       let fileData = await fs.readFile(cacheFile);
       this.blocksEventsCache = new Map(JSON.parse(fileData));
-      console.log("Cache file loaded...");
     }
   }
 }

contributions/tech/report-generator/src/generator.ts

@@ -1,45 +1,91 @@
+import fs from "fs";
+const exec = require("util").promisify(require("child_process").exec);
 import { StatisticsCollector } from "./StatisticsCollector";
+import { connectApi, getHead, getCouncils } from "./lib/api";
+import { Round } from "./lib/types";
 
-const fs = require("fs").promises;
+const TEMPLATE_FILE = __dirname + "/../report-template.md";
+const PROVIDER_URL = "ws://127.0.0.1:9944";
 
 async function main() {
   const args = process.argv.slice(2);
 
-  if (args.length != 2) {
-    console.error("Usage: [start bock number] [end block number]");
-    process.exit(1);
-  }
+  if (args.length < 2) return updateReports(Number(args[0]));
 
   const startBlock = Number(args[0]);
   const endBlock = Number(args[1]);
 
   if (isNaN(startBlock) || isNaN(endBlock) || startBlock >= endBlock) {
-    console.error("Invalid block range");
+    console.error("Invalid block range.");
     process.exit(1);
-  }
+  } else generateReport(startBlock, endBlock, TEMPLATE_FILE);
+}
 
-  try {
-    let fileData = await fs.readFile(__dirname + "/../report-template.md", {
-      encoding: "utf8",
-    });
-    console.log("Getting report info...");
-    let staticCollecttor = new StatisticsCollector();
-    let statistics = await staticCollecttor.getStatistics(startBlock, endBlock);
-    console.log("Writing info in the report...");
+const generateReport = (
+  startBlock: number,
+  endBlock: number,
+  templateFile: string
+): Promise<any> => {
+  let fileData = fs.readFileSync(templateFile, "utf8");
+  let staticCollecttor = new StatisticsCollector();
+  console.log(`-> Collecting stats from ${startBlock} to ${endBlock}`);
+  return staticCollecttor.getStatistics(startBlock, endBlock).then((stats) => {
+    console.log(stats);
+    if (!stats.dateStart) return false;
+    const round = stats.councilRound || 1;
 
-    let entries = Object.entries(statistics);
+    const fileName = `Council_Round${round}_${startBlock}-${endBlock}_Tokenomics_Report.md`;
+    // antioch was updated to sumer at 717987
+    const version = startBlock < 717987 ? "antioch-3" : "sumer-4";
+    const dir = __dirname + `/../../../../council/tokenomics/${version}`;
+
+    const reports = fs.readdirSync(dir);
+    const exists = reports.find(
+      (file: string) => file.indexOf(`_${endBlock + 1}_T`) !== -1
+    );
+    if (exists && exists !== fileName) {
+      console.log(`INFO renaming ${exists} to ${fileName}`);
+      try {
+        exec(`git mv ${dir}/${exists} ${dir}/${fileName}`);
+      } catch (e) {}
+    }
 
-    for (let entry of entries) {
-      let regex = new RegExp("{" + entry[0] + "}", "g");
+    console.log(`-> Writing report to ${fileName}`);
+    for (const entry of Object.entries(stats)) {
+      const regex = new RegExp("{" + entry[0] + "}", "g");
       fileData = fileData.replace(regex, entry[1].toString());
     }
+    fs.writeFileSync(`${dir}/${fileName}`, fileData);
+    return true;
+  });
+};
 
-    await fs.writeFile("report.md", fileData);
-    console.log("Report generated!");
-    process.exit(0);
-  } catch (e) {
-    console.error(e);
-  }
-}
+const updateReports = async (round?: number) => {
+  console.debug(`Connecting to ${PROVIDER_URL}`);
+  const api = await connectApi(PROVIDER_URL);
+  await api.isReady;
+
+  console.log(`-> Fetching councils`);
+  const head = await getHead(api)
+  getCouncils(api, +head).then(async (councils: Round[]) => {
+    api.disconnect();
+    if (round !== null) {
+      const council = councils.find((c) => c.round === round);
+      if (!council) return console.warn(`Round ${round} not found:`, councils);
+      console.log(
+        `-> Updating round ${round} (${council.start}-${council.end})`
+      );
+      await generateReport(council.start, council.end, TEMPLATE_FILE);
+    } else {
+      console.log(`-> Updating reports`);
+      await Promise.all(
+        councils.map((council) =>
+          generateReport(council.start, council.end, TEMPLATE_FILE)
+        )
+      );
+    }
+    process.exit();
+  });
+};
 
 main();

contributions/tech/report-generator/src/lib

@@ -1 +1 @@
-Subproject commit e3e2f0aca938668d557c003133763f2bc186233f
+Subproject commit 7a0df3177dbb47836c7968af0e12e3f722425d5b