dynamic groups, improved syncing

src/App.tsx

@@ -52,11 +52,10 @@ class App extends React.Component<IProps, IState> {
   async updateStatus(api: ApiPromise, id: number): Promise<Status> {
     console.debug(`#${id}: Updating status`);
     this.updateActiveProposals();
-    getMints(api, [2, 3, 4]).then((mints) => this.save(`mints`, mints));
+    getMints(api).then((mints) => this.save(`mints`, mints));
     getTokenomics().then((tokenomics) => this.save(`tokenomics`, tokenomics));
 
     let { status, councils } = this.state;
-    status.era = await this.updateEra(api);
     status.election = await updateElection(api);
     if (status.election?.stage) this.getElectionStatus(api);
     councils.forEach((c) => {
@@ -74,9 +73,12 @@ class App extends React.Component<IProps, IState> {
     status.threads = await get.currentThreadId(api);
     status.categories = await get.currentCategoryId(api);
     status.proposalPosts = await api.query.proposalsDiscussion.postCount();
-    status.lastReward = await getLastReward(api, status.era);
-    status.validatorStake = await getTotalStake(api, status.era);
-    this.save("status", status);
+    await this.updateEra(api, status.era).then(async (era) => {
+      status.era = era.toNumber();
+      status.lastReward = await getLastReward(api, era);
+      status.validatorStake = await getTotalStake(api, era);
+      this.save("status", status);
+    });
     return status;
   }
 
@@ -114,9 +116,10 @@ class App extends React.Component<IProps, IState> {
     });
   }
 
-  async updateEra(api: Api) {
+  async updateEra(api: Api, old: number) {
     const { status, validators } = this.state;
     const era = Number(await api.query.staking.currentEra());
+    if (era === old) return era;
     this.updateWorkingGroups(api);
     this.updateValidatorPoints(api, status.era);
     if (era > status.era || !validators.length) this.updateValidators(api);
@@ -125,12 +128,12 @@ class App extends React.Component<IProps, IState> {
 
   async updateWorkingGroups(api: ApiPromise) {
     const { members, openings, workers } = this.state;
-    await updateOpenings(api, openings, members).then((openings) =>
-      this.save("openings", openings)
-    );
-    await updateWorkers(api, workers, members).then((workers) =>
-      this.save("workers", workers)
-    );
+    updateWorkers(api, workers, members).then((workers) => {
+      this.save("workers", workers);
+      updateOpenings(api, openings, members).then((openings) =>
+        this.save("openings", openings)
+      );
+    });
     return this.save("council", await api.query.council.activeCouncil());
   }
 
@@ -255,6 +258,7 @@ class App extends React.Component<IProps, IState> {
       await api.isReady;
       console.log(`Connected to ${wsLocation}`);
       this.setState({ connected: true });
+      this.updateWorkingGroups(api);
 
       api.rpc.chain.subscribeNewHeads(async (header: Header) => {
         let { blocks, status } = this.state;

src/components/Tokenomics/Burns.tsx

@@ -20,7 +20,7 @@ const Burns = (props: {
   });
   const pctRounded = (100 * percent).toFixed(2);
   return (
-    <div className="p-5">
+    <div className="p-5 w-100 overflow-hidden">
       <Chart
         data={data}
         x="time"

src/components/Tokenomics/Groups.tsx

@@ -2,52 +2,53 @@ import { Button, Table } from "react-bootstrap";
 import { Link } from "react-router-dom";
 import Loading from "../Loading";
 
-const mintTags = [``, ``, `Storage`, `Content`, `Operations`];
 const mJoy = (joy: number) => (joy ? (joy / 1000000).toFixed(3) : ``);
 
 const Groups = (props: { price: nubmer; workers: {}; mints: number[] }) => {
   const { mints, workers, price } = props;
-  if (!mints[4]) return <div />;
+  if (!workers) return <Loading target="workers" />;
+  const groups = Object.keys(workers).filter((k) => workers[k]?.length);
+  if (!groups.length) return <Loading target="workers" />;
   return (
     <div className="d-flex flex-wrap">
-      <Group
-        workers={workers?.storage}
-        mint={mints[2]}
-        tag={mintTags[2]}
-        price={price}
-      />
-      <Group
-        workers={workers?.content}
-        mint={mints[3]}
-        tag={mintTags[3]}
-        price={price}
-      />
-      <Group
-        workers={workers?.operations}
-        mint={mints[4]}
-        tag={mintTags[4]}
-        price={price}
-      />
+      {groups.map((group) => (
+        <Group
+          key={group}
+          group={group}
+          workers={workers[group]}
+          mint={mints.find((m) => m?.group === group)}
+          price={price}
+        />
+      ))}
     </div>
   );
 };
 
+const Head = (props: { group: string; mint: any }) => {
+  const { group, mint } = props;
+  return (
+    <h2 className="m-3 text-center">
+      {group} <Mint mint={mint?.content} />
+    </h2>
+  );
+};
+
 const Group = (props: {
+  group: string;
   price: number;
   workers: any[];
   mint: {};
-  tag: string;
 }) => {
-  const { workers, mint, tag, price } = props;
+  const { group, workers, mint, price } = props;
+  if (!group) return <div />;
   if (!workers)
     return (
-      <div className="p-3 col-lg-4">
-        <h2 className="m-3 text-center">
-          {tag} <Mint mint={mint} />
-        </h2>
+      <div className="p-12 col-lg-6">
+        <Head group={group} mint={mint} />
         <Loading target="workers" />
       </div>
     );
+  if (!workers.length) return <div />;
 
   const rewards = workers.reduce(
     (sum, { reward }) => sum + (reward?.amount_per_payout || 0),
@@ -60,11 +61,8 @@ const Group = (props: {
   );
 
   return (
-    <div className="p-3 col-lg-4">
-      <h2 className="m-3 text-center">
-        {tag} <Mint mint={mint} />
-      </h2>
-
+    <div className="p-12 col-lg-6">
+      <Head group={group} mint={mint} />
       <Table className="text-light">
         <thead>
           <tr>
@@ -79,7 +77,7 @@ const Group = (props: {
             <td></td>
             <td>Total</td>
             {td(rewards * 28)}
-            {td(mint.total_minted)}
+            {td(mint?.content?.total_minted)}
             {td(stakes)}
           </tr>
 
@@ -103,9 +101,9 @@ const Group = (props: {
 };
 
 const Mint = (props: { mint: { capacity: number; total_minted: number } }) => {
-  const { mint } = props;
-  if (!mint) return <div />;
-  const current = (mint.capacity / 1000000).toFixed(1);
+  if (!props.mint) return <div />;
+  const { capacity } = props.mint;
+  const current = (capacity / 1000000).toFixed(1);
   const color = current < 1 ? `danger` : current < 2 ? `warning` : `success`;
   return (
     <Button className="p-1" variant={color} title="Mint Capacity">

src/components/Tokenomics/Spending.tsx

@@ -7,7 +7,7 @@ const percent = (joy, total) => ((100 * joy) / total).toFixed(2);
 
 const Spending = (props: { roles: RoleSpending[] }) => {
   const { groups, price } = props;
-  if (!groups.length) return <div />;
+  if (!groups?.length) return <div />;
 
   const minted = groups.reduce((sum, { earning }) => earning + sum, 0);
   const staked = groups.reduce((sum, { stake }) => stake + sum, 0);

src/components/Tokenomics/lib.ts

@@ -1,3 +1,5 @@
+import { groups } from "../../lib/groups";
+
 export const groupsMinting = (council, workers, validators) => {
   if (!council || !workers || !validators) return [];
 
@@ -15,7 +17,7 @@ export const groupsMinting = (council, workers, validators) => {
   const leadReward = ({ reward }): number => termReward(reward);
   const leadStake = ({ stake }): number => stake || 0;
 
-  const groups = [
+  let nonGroups = [
     {
       id: "validators",
       name: "Validators & Nominators",
@@ -26,7 +28,9 @@ export const groupsMinting = (council, workers, validators) => {
         0
       ),
     },
-    {
+  ];
+  if (council)
+    nonGroups.push({
       id: "council",
       name: "Council members",
       actors: council?.length,
@@ -36,50 +40,28 @@ export const groupsMinting = (council, workers, validators) => {
           sum + +stake + backers.reduce((s, voter) => s + +voter.stake, 0),
         0
       ),
+    });
+  return Object.values(groups).reduce(
+    (updated: { [key: string]: any }, id: string) => {
+      if (!workers[id] || !workers[id].length) return updated;
+      const add = [
+        {
+          id,
+          name: id,
+          actors: workers[id].filter((w) => !w.isLead).length,
+          earning: workerRewards(workers[id].filter((w) => !w.isLead)),
+          stake: workerStakes(workers[id].filter((w) => !w.isLead)),
+        },
+        {
+          id: id + "-lead",
+          name: id + " lead",
+          actors: 1,
+          earning: leadReward(workers[id].find((w) => w.isLead)),
+          stake: leadStake(workers[id].find((w) => w.isLead)),
+        },
+      ];
+      return updated.concat(add);
     },
-    {
-      id: "storage",
-      name: "Storage Providers",
-      actors: workers.storage.filter((w) => !w.isLead).length,
-      earning: workerRewards(workers.storage.filter((w) => !w.isLead)),
-      stake: workerStakes(workers.storage.filter((w) => !w.isLead)),
-    },
-    {
-      id: "storage-lead",
-      name: "Storage lead",
-      actors: 1,
-      earning: leadReward(workers.storage.find((w) => w.isLead)),
-      stake: leadStake(workers.storage.find((w) => w.isLead)),
-    },
-    {
-      id: "curators",
-      name: "Content Curators",
-      actors: workers.content.filter((w) => !w.isLead).length,
-      earning: workerRewards(workers.content.filter((w) => !w.isLead)),
-      stake: workerStakes(workers.content.filter((w) => !w.isLead)),
-    },
-    {
-      id: "curators-lead",
-      name: "Curators lead",
-      actors: 1,
-      earning: leadReward(workers.content.find((w) => w.isLead)),
-      stake: leadStake(workers.content.find((w) => w.isLead)),
-    },
-    {
-      id: "operations",
-      name: "Operations",
-      actors: workers.operations.filter((w) => !w.isLead).length,
-      earning: workerRewards(workers.operations.filter((w) => !w.isLead)),
-      stake: workerStakes(workers.operations.filter((w) => !w.isLead)),
-    },
-    {
-      id: "operations-lead",
-      name: "Operations lead",
-      actors: 1,
-      earning: leadReward(workers.operations.find((w) => w.isLead)),
-      stake: leadStake(workers.operations.find((w) => w.isLead)),
-    },
-  ];
-  //this.save("groups", groups);
-  return groups;
+    nonGroups
+  );
 };

src/lib/groups.ts

@@ -3,7 +3,7 @@ import { Openings } from "./types";
 import { Mint } from "@joystream/types/mint";
 
 // mapping: key = pioneer route, value: chain section
-const groups = {
+export const groups = {
   curators: "contentWorkingGroup",
   storageProviders: "storageWorkingGroup",
   distribution: "distributionWorkingGroup",
@@ -12,14 +12,17 @@ const groups = {
   operationsGroupGamma: "operationsWorkingGroupGamma",
 };
 
-export const getMints = async (api: Api, ids: number[]): Promise<Mint[]> => {
+export const getMints = async (api: Api): Promise<Mint[]> => {
   console.debug(`Fetching mints`);
   const getMint = (id: number) => api.query.minting.mints(id);
-  const mints: Mint[] = [];
-  await Promise.all(
-    ids.map(async (id) => (mints[id] = (await getMint(id)).toJSON()))
+  const promises = Object.values(groups).map((group) =>
+    api.query[group].mint().then((mintId) =>
+      getMint(mintId).then((content) => {
+        return { group, mintId: mintId.toNumber(), content };
+      })
+    )
   );
-  return mints;
+  return await Promise.all(promises);
 };
 
 export const updateWorkers = async (
@@ -28,14 +31,20 @@ export const updateWorkers = async (
   members: Member[]
 ) => {
   const lastUpdate = workers?.timestamp;
-  if (lastUpdate && moment() < moment(lastUpdate).add(1, `hour`))
+  if (
+    lastUpdate &&
+    Object.keys(workers).length > 1 &&
+    moment() < moment(lastUpdate).add(1, `hour`)
+  )
     return workers;
+  console.log(`Fetching workers of ${Object.keys(groups).length} groups`);
   let updated: { [key: string]: any[] } = {};
-  Object.values(groups).map(
-    async (group: string) =>
-      (groups[group] = await getGroupWorkers(api, group, members))
+  const promises = Object.values(groups).map((group: string) =>
+    getGroupWorkers(api, group, members).then((w) => (updated[group] = w))
   );
-  await Promise.all(Object.keys(updated));
+  await Promise.all(promises);
+  console.log(`Collected all workers`, updated);
+  updated.timestamp = moment().valueOf();
   return updated;
 };
 
@@ -45,7 +54,7 @@ const getGroupWorkers = async (
   members: Member[]
 ) => {
   if (!api.query[group]) {
-    console.debug(`Skipping outdated group`, group);
+    console.debug(`getGroupWorkers: Skipping outdated group`, group);
     return [];
   }
 
@@ -54,7 +63,7 @@ const getGroupWorkers = async (
     (await api.query[group].nextWorkerId()) as WorkerId
   ).toNumber();
   const lead = await api.query[group].currentLead();
-  console.debug(`Fetching ${count} ${group} workers`);
+  console.debug(` - Fetching ${count} ${group} workers`);
   for (let id = 0; id < count; ++id) {
     const isLead = id === +lead;
     const worker: WorkerOf = await api.query[group].workerById(id);
@@ -94,18 +103,18 @@ const getGroupWorkers = async (
 
 export const updateOpenings = async (
   api: ApiPromise,
-  outdated: any,
+  outdated: { [key: string]: Opening[] },
   members: Member[]
 ) => {
   const lastUpdate = outdated?.timestamp;
   if (lastUpdate && moment() < moment(lastUpdate).add(1, `hour`))
     return outdated;
-  console.debug(`Updating openings`);
+  console.debug(`Updating openings of ${Object.keys(groups).length} groups`);
 
   let updated: Openings = {};
   await Promise.all(
-    Object.keys(groups).map((group) =>
-      updateGroupOpenings(api, groups[group], outdated[group], members).then(
+    Object.values(groups).map((group) =>
+      updateGroupOpenings(api, group, outdated[group], members).then(
         (openings) => (updated[group] = openings)
       )
     )
@@ -121,7 +130,7 @@ export const updateGroupOpenings = async (
   members: Member[]
 ) => {
   if (!api.query[group]) {
-    console.debug(`Skipping outdated group`, group);
+    console.debug(`updateGroupOpenings: Skipping outdated group`, group);
     return [];
   }
   const count = (
@@ -165,7 +174,7 @@ export const getApplications = (
   members: Member[]
 ) => {
   if (!api.query[group]) {
-    console.debug(`Skipping outdated group`, group);
+    console.debug(`getApplications: Skipping outdated group`, group);
     return [];
   }
   return Promise.all(

src/lib/storage.ts

@@ -3,25 +3,7 @@ import { hydraLocation } from "../config";
 
 export const getAssets = async () => {
   const query = {
-    query: `\nquery {
-  videos (limit:1000000, orderBy:createdAt_DESC){
-    id
-    title
-    updatedAt
-    createdAt
-    createdInBlock
-    mediaDataObject {
-      joystreamContentId
-      liaisonJudgement
-      ipfsContentId
-      liaison {
-        workerId
-        metadata
-        isActive
-      }
-    }
-  }
-}\n`,
+    query: `query { videos(limit: 1000000, orderBy: createdAt_DESC) { id updatedAt createdAt createdInBlock title description licenseId isCensored isPublic isExplicit categoryId thumbnailPhoto { storageBagId storageBag { distributionBuckets { operators { workerId status distributionBucketId } } } } channelId mediaId mediaMetadata { size encoding { videomediametadataencoding { pixelWidth } } } } }`,
   };
   console.debug(`Fetching data IDs from ${hydraLocation}`);
   return axios.post(hydraLocation, query).then(({ data }) => data.data.videos);

src/types.ts

@@ -125,7 +125,7 @@ export interface IState {
   domain: string;
   proposalPosts: any[];
   members: Member[];
-  mints: number[];
+  mints: { group: string; mintId: number; content: number }[];
   tokenomics?: Tokenomics;
   reports: { [key: string]: string };
   [key: string]: any;