refactor App: outsource requests

src/App.tsx

@@ -4,6 +4,8 @@ import "./index.css";
 import { Modals, Routes, Loading, Footer, Status } from "./components";
 
 import * as get from "./lib/getters";
+import { bootstrap, getTokenomics, queryJstats } from "./lib/queries";
+import { getMints, updateOpenings, updateWorkers } from "./lib/groups";
 import {
   updateElection,
   getCouncilApplicants,
@@ -11,15 +13,19 @@ import {
   getVotes,
 } from "./lib/election";
 import {
+  getStashes,
+  getNominators,
   getValidators,
+  getValidatorStakes,
   getEraRewardPoints,
   getLastReward,
   getTotalStake,
 } from "./lib/validators";
-import { domain, apiLocation, wsLocation, historyDepth } from "./config";
+import { apiLocation, wsLocation, historyDepth } from "./config";
+import { initialState } from "./state";
 import axios from "axios";
-import moment from "moment";
 
+// types
 import { Api, IState } from "./types";
 import { types } from "@joystream/types";
 import { ApiPromise, WsProvider } from "@polkadot/api";
@@ -27,46 +33,6 @@ import { Header } from "@polkadot/types/interfaces";
 
 interface IProps {}
 
-const initialState = {
-  assets: [],
-  connected: false,
-  faq: [],
-  fetching: "",
-  tasks: 0,
-  blocks: [],
-  nominators: [],
-  validators: [],
-  mints: [],
-  channels: [],
-  posts: [],
-  councils: [],
-  election: {
-    applicants: [],
-    votes: [],
-    councilSize: 20,
-  },
-  categories: [],
-  threads: [],
-  proposals: [],
-  domain,
-  members: [],
-  providers: [],
-  reports: {},
-  stakes: {},
-  stashes: [],
-  stars: {},
-  hideFooter: true,
-  showStatus: false,
-  editKpi: false,
-  status: { era: 0, block: { id: 0, era: 0, timestamp: 0, duration: 6 } },
-  groups: [],
-  rewardPoints: {
-    total: 0,
-    eraTotals: {},
-    validators: {},
-  },
-};
-
 class App extends React.Component<IProps, IState> {
   initializeSocket() {
     socket.on("disconnect", () => setTimeout(this.initializeSocket, 1000));
@@ -81,107 +47,14 @@ class App extends React.Component<IProps, IState> {
     });
   }
 
-  async handleApi(api: ApiPromise) {
-    this.fetchFromApi();
-    api.rpc.chain.subscribeNewHeads((head: Header) =>
-      this.handleBlock(api, head)
-    );
-    this.updateStatus(api);
-    this.fetchMints(api, [2, 3, 4]);
-  }
-
-  async fetchMints(api: Api, ids: number[]) {
-    console.debug(`Fetching mints`);
-    let mints = [];
-    return Promise.all(
-      ids.map(
-        async (id) => (mints[id] = (await api.query.minting.mints(id)).toJSON())
-      )
-    ).then(() => this.save(`mints`, mints));
-  }
-
-  async fetchAssets() {
-    const url = "https://hydra.joystream.org/graphql";
-    const request = {
-      query: "query {\n dataObjects(where: {}) { joystreamContentId }\n}",
-    };
-    console.debug(`Fetching data IDs (from ${url})`);
-    const { data } = await axios.post(url, request);
-    let assets = [];
-    data.data.dataObjects.forEach((p) => assets.push(p.joystreamContentId));
-    //console.log(`assets`, data);
-    this.save(`assets`, assets);
-  }
-
-  async fetchStorageProviders() {
-    const url = "https://hydra.joystream.org/graphql";
-    const request = {
-      query:
-        'query {\n  workers(where: {metadata_contains: "http", isActive_eq: true, type_eq: STORAGE}){\n    metadata\n  }\n}',
-    };
-    console.debug(`Fetching storage providers (from ${url})`);
-    const { data } = await axios.post(url, request);
-    const providers = data.data.workers.map((p) => {
-      return {
-        url: p.metadata,
-      };
-    });
-    this.save(`providers`, providers);
-  }
-
-  async getStorageProviders(api: Api) {
-    console.debug(`Fetching storage providers (from chain)`);
-    let providers = [];
-    const worker = await api.query.storageWorkingGroup.nextWorkerId();
-    console.log(`next provider: ${worker}`);
-
-    for (let i = 0; i < Number(worker); ++i) {
-      let storageProvider = (await api.query.storageWorkingGroup.workerById(
-        i
-      )) as WorkerOf;
-      if (storageProvider.is_active) {
-        const storage = (await api.query.storageWorkingGroup.workerStorage(
-          i
-        )) as Bytes;
-        const url = Buffer.from(storage.toString().substr(2), "hex").toString();
-
-        let membership = (await api.query.members.membershipById(
-          storageProvider.member_id
-        )) as Membership;
-
-        providers[i] = {
-          owner: membership.handle,
-          account: membership.root_account,
-          storage,
-          url,
-        };
-      }
-      this.save(`providers`, providers);
-    }
-  }
-
-  async handleBlock(api, header: Header) {
-    let { blocks, status } = this.state;
-    const id = header.number.toNumber();
-    if (blocks.find((b) => b.id === id)) return;
-    const timestamp = (await api.query.timestamp.now()).toNumber();
-    const duration = status.block ? timestamp - status.block.timestamp : 6000;
-
-    status.block = { id, timestamp, duration };
-    this.save("status", status);
+  // sync via joystream-api
 
-    blocks = this.addOrReplace(blocks, status.block);
-    this.setState({ blocks });
-
-    if (id / 50 === Math.floor(id / 50)) {
-      this.updateStatus(api, id);
-      this.fetchTokenomics();
-      this.updateActiveProposals();
-    }
-  }
+  async updateStatus(api: ApiPromise, id: number): Promise<Status> {
+    console.debug(`#${id}: Updating status`);
+    this.updateActiveProposals();
+    getMints(api, [2, 3, 4]).then((mints) => this.save(`mints`, mints));
+    getTokenomics().then((tokenomics) => this.save(`tokenomics`, tokenomics));
 
-  async updateStatus(api: ApiPromise, id = 0): Promise<Status> {
-    console.debug(`Updating status for block ${id}`);
     let { status, councils } = this.state;
     status.era = await this.updateEra(api);
     status.election = await updateElection(api);
@@ -253,222 +126,32 @@ class App extends React.Component<IProps, IState> {
     return era;
   }
 
-  async fetchCouncils() {
-    const { data } = await axios.get(`${apiLocation}/v1/councils`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`councils`, data);
-    this.save("councils", data);
-
-    // TODO OPTIMIZE find max round
-    let council = { round: 0 };
-    data.forEach((c) => {
-      if (c.round > council.round) council = c;
-    });
-    let { status } = this.state;
-    status.council = council;
-    this.save("status", status);
-  }
-
-  async fetchProposals() {
-    const { data } = await axios.get(`${apiLocation}/v2/proposals`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    const proposals = data.filter((p) => p.created);
-    console.debug(`proposals`, proposals);
-    this.save("proposals", proposals);
-  }
-
   async updateWorkingGroups(api: ApiPromise) {
-    const openingsUpdated = this.state.openings?._lastUpdate;
-    if (
-      !openingsUpdated ||
-      moment().valueOf() < moment(openingsUpdated).add(1, `hour`).valueOf()
-    ) {
-      const openings = {
-        curators: await this.fetchOpenings(api, "contentDirectory"),
-        storageProviders: await this.fetchOpenings(api, "storage"),
-        operationsGroup: await this.fetchOpenings(api, "operations"),
-        _lastUpdate: moment().valueOf(),
-      };
-      this.save("openings", openings);
-    }
-
-    const lastUpdate = this.state.workers?._lastUpdate;
-    if (lastUpdate && moment() < moment(lastUpdate).add(1, `hour`)) return;
-    const workers = {
-      content: await this.fetchWorkers(api, "contentDirectory"),
-      storage: await this.fetchWorkers(api, "storage"),
-      operations: await this.fetchWorkers(api, "operations"),
-      _lastUpdate: moment().valueOf(),
-    };
-    this.save("workers", workers);
-    const council = await api.query.council.activeCouncil();
-    this.save("council", council);
-    return workers;
-  }
-
-  async fetchOpenings(api: ApiPromise, wg: string) {
-    const group = wg + "WorkingGroup";
-    const count = (
-      (await api.query[group].nextOpeningId()) as OpeningId
-    ).toNumber();
-    console.debug(`Fetching ${count} ${wg} openings`);
-    let openings = [];
-    for (let wgOpeningId = 0; wgOpeningId < count; ++wgOpeningId) {
-      const wgOpening: OpeningOf = (
-        await api.query[group].openingById(wgOpeningId)
-      ).toJSON();
-      const openingId = wgOpening.hiring_opening_id;
-      const opening = (await api.query.hiring.openingById(openingId)).toJSON();
-      openings.push({
-        ...opening,
-        openingId,
-        wgOpeningId,
-        type: Object.keys(wgOpening.opening_type)[0],
-        applications: await this.fetchApplications(
-          api,
-          group,
-          wgOpening.applications
-        ),
-        policy: wgOpening.policy_commitment,
-      });
-    }
-    console.debug(`${group} openings`, openings);
-    return openings;
-  }
-
-  async fetchApplications(api: ApiPromise, group: string, ids: number[]) {
-    const { members } = this.state;
-    return Promise.all(
-      ids.map(async (wgApplicationId) => {
-        const wgApplication: ApplicationOf = (
-          await api.query[group].applicationById(wgApplicationId)
-        ).toJSON();
-        let application = {};
-        application.account = wgApplication.role_account_id;
-        application.openingId = +wgApplication.opening_id;
-        application.memberId = +wgApplication.member_id;
-        const member = members.find((m) => +m.id === application.memberId);
-        const handle = member ? member.handle : null;
-        application.member = { handle };
-        application.id = +wgApplication.application_id;
-        application.application = (
-          await api.query.hiring.applicationById(application.id)
-        ).toJSON();
-        return application;
-      })
+    const { members, openings, workers } = this.state;
+    await updateOpenings(api, openings, members).then((openings) =>
+      this.save("openings", openings)
     );
-  }
-
-  async fetchWorkers(api: ApiPromise, wg: string) {
-    const group = wg + "WorkingGroup";
-    const { members } = this.state;
-    let workers = [];
-    const count = (
-      (await api.query[group].nextWorkerId()) as WorkerId
-    ).toNumber();
-    const lead = await api.query[group].currentLead();
-    console.debug(`Fetching ${count} ${wg} workers`);
-    for (let id = 0; id < count; ++id) {
-      const isLead = id === +lead;
-      const worker: WorkerOf = await api.query[group].workerById(id);
-      if (!worker.is_active) continue;
-      const memberId = worker.member_id.toJSON();
-      const member: Membership = members.find((m) => m.id === memberId);
-      const handle = member?.handle;
-      let stake: Stake;
-      let reward: RewardRelationship;
-
-      if (worker.role_stake_profile.isSome) {
-        const roleStakeProfile = worker.role_stake_profile.unwrap();
-        const stakeId = roleStakeProfile.stake_id;
-        const { staking_status } = (
-          await api.query.stake.stakes(stakeId)
-        ).toJSON();
-        stake = staking_status?.staked?.staked_amount;
-      }
-
-      if (worker.reward_relationship.isSome) {
-        const rewardId = worker.reward_relationship.unwrap();
-        reward = (
-          await api.query.recurringRewards.rewardRelationships(rewardId)
-        ).toJSON();
-      }
-      workers.push({
-        id,
-        memberId,
-        handle,
-        stake,
-        reward,
-        isLead,
-      });
-    }
-    return workers;
-  }
-
-  // forum
-  updateForum() {
-    this.fetchPosts();
-    this.fetchThreads();
-    this.fetchCategories();
-  }
-  async fetchPosts() {
-    const { data } = await axios.get(`${apiLocation}/v1/posts`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`posts`, data);
-    this.save("posts", data);
-  }
-
-  async fetchThreads() {
-    const { data } = await axios.get(`${apiLocation}/v1/threads`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`threads`, data);
-    this.save("threads", data);
-  }
-
-  async fetchCategories() {
-    const { data } = await axios.get(`${apiLocation}/v1/categories`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`categories`, data);
-    this.save("categories", data);
-  }
-
-  async fetchMembers() {
-    const { data } = await axios.get(`${apiLocation}/v1/members`);
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`members`, data);
-    this.save("members", data);
-  }
-
-  async fetchFAQ() {
-    const { data } = await axios.get(
-      `https://joystreamstats.live/static/faq.json`
+    await updateWorkers(api, workers, members).then((workers) =>
+      this.save("workers", workers)
     );
-    if (!data || data.error) return console.error(`failed to fetch from API`);
-    console.debug(`faq`, data);
-    this.save("faq", data);
-  }
-
-  addOrReplace(array, item) {
-    return array.filter((i) => i.id !== item.id).concat(item);
-  }
-
-  async fetchTokenomics() {
-    const now = new Date();
-    if (this.state.tokenomics?.timestamp + 300000 > now) return;
-    console.debug(`Updating tokenomics`);
-    let { data } = await axios.get("https://status.joystream.org/status");
-    if (!data || data.error) return;
-    data.timestamp = now;
-    this.save("tokenomics", data);
-  }
-
-  async updateValidators(api: ApiPromise) {
-    this.save("validators", await getValidators(api));
-    this.save("nominators", await getNominators(api));
-    const stashes = await getStashes(api);
-    this.save("stashes", stashes);
-    const { members } = this.state;
-    this.save("stakes", await getValidatorStakes(api, era, stashes, members));
+    return this.save("council", await api.query.council.activeCouncil());
+  }
+
+  updateValidators(api: ApiPromise) {
+    getValidators(api).then((validators) => {
+      this.save("validators", validators);
+      getNominators(api).then((nominators) => {
+        this.save("nominators", nominators);
+        getStashes(api).then((stashes) => {
+          this.save("stashes", stashes);
+          const { status, members } = this.state;
+          const { era } = status;
+          getValidatorStakes(api, era, stashes, members, this.save).then(
+            (stakes) => this.save("stakes", stakes)
+          );
+        });
+      });
+    });
   }
 
   async updateValidatorPoints(api: ApiPromise, currentEra: number) {
@@ -495,109 +178,27 @@ class App extends React.Component<IProps, IState> {
     }
   }
 
-  toggleStar(account: string) {
-    let { stars } = this.state;
-    stars[account] = !stars[account];
-    this.save("stars", stars);
-  }
-
-  // Reports
-  async fetchReports() {
-    const domain = `https://raw.githubusercontent.com/Joystream/community-repo/master/council-reports`;
-    const apiBase = `https://api.github.com/repos/joystream/community-repo/contents/council-reports`;
-
-    const urls: { [key: string]: string } = {
-      alexandria: `${apiBase}/alexandria-testnet`,
-      archive: `${apiBase}/archived-reports`,
-      template: `${domain}/templates/council_report_template_v1.md`,
-    };
-
-    ["alexandria", "archive"].map((folder) =>
-      this.fetchGithubDir(urls[folder])
-    );
-
-    // template
-    this.fetchGithubFile(urls.template);
-  }
-
-  async saveReport(name: string, content: Promise<string>) {
-    const { reports } = this.state;
-    reports[name] = await content;
-    this.save("reports", reports);
-  }
-
-  async fetchGithubFile(url: string): Promise<string> {
-    const { data } = await axios.get(url);
-    return data;
-  }
-  async fetchGithubDir(url: string) {
-    const { data } = await axios.get(url);
-
-    data.forEach(
-      async (o: {
-        name: string;
-        type: string;
-        url: string;
-        download_url: string;
-      }) => {
-        const match = o.name.match(/^(.+)\.md$/);
-        const name = match ? match[1] : o.name;
-        if (o.type === "file")
-          this.saveReport(name, this.fetchGithubFile(o.download_url));
-        else this.fetchGithubDir(o.url);
-      }
-    );
-  }
-
-  getMember(handle: string) {
-    const { members } = this.state;
-    const member = members.find((m) => m.handle === handle);
-    if (member) return member;
-    return members.find((m) => m.rootKey === handle);
-  }
-
-  loadMembers() {
-    const members = this.load("members");
-    if (!members) return;
-    this.setState({ members });
-  }
+  async updateCouncils() {
+    queryJstats(`v1/councils`).then((councils) => {
+      this.save(`councils`, councils);
 
-  loadPosts() {
-    const posts: Post[] = this.load("posts");
-    posts.forEach(({ id, text }) => {
-      if (text && text.length > 500)
-        console.debug(`post ${id}: ${(text.length / 1000).toFixed(1)} KB`);
+      // TODO OPTIMIZE find max round
+      let council = { round: 0 };
+      councils.forEach((c) => {
+        if (c.round > council.round) council = c;
+      });
+      let { status } = this.state;
+      status.council = council; // needed by dashboard
+      this.save("status", status);
     });
-    if (posts) this.setState({ posts });
   }
 
-  async loadData() {
-    console.debug(`Loading data`);
-    "status members assets providers councils council election workers categories channels proposals posts threads  mints openings tokenomics transactions reports validators nominators staches stakes rewardPoints stars"
-      .split(" ")
-      .map((key) => this.load(key));
-  }
+  // interface interactions
 
-  load(key: string) {
-    try {
-      const data = localStorage.getItem(key);
-      if (!data) return;
-      const size = data.length;
-      if (size > 10240)
-        console.debug(` -${key}: ${(size / 1024).toFixed(1)} KB`);
-      this.setState({ [key]: JSON.parse(data) });
-      return JSON.parse(data);
-    } catch (e) {
-      console.warn(`Failed to load ${key}`, e);
-    }
-  }
-  save(key: string, data: any) {
-    this.setState({ [key]: data });
-    try {
-      localStorage.setItem(key, JSON.stringify(data));
-    } catch (e) {
-      console.warn(`Failed to save ${key} (${data.length}KB)`, e);
-    }
+  toggleStar(account: string) {
+    let { stars } = this.state;
+    stars[account] = !stars[account];
+    this.save("stars", stars);
   }
 
   toggleEditKpi(editKpi) {
@@ -610,6 +211,13 @@ class App extends React.Component<IProps, IState> {
     this.setState({ hideFooter: !this.state.hideFooter });
   }
 
+  getMember(handle: string) {
+    const { members } = this.state;
+    const member = members.find((m) => m.handle === handle);
+    if (member) return member;
+    return members.find((m) => m.rootKey === handle);
+  }
+
   render() {
     const { connected, fetching, loading, hideFooter } = this.state;
     if (loading) return <Loading />;
@@ -621,8 +229,6 @@ class App extends React.Component<IProps, IState> {
           toggleFooter={this.toggleFooter}
           toggleStar={this.toggleStar}
           getMember={this.getMember}
-          fetchProposals={this.fetchProposals}
-          updateForum={this.updateForum}
           {...this.state}
         />
 
@@ -643,6 +249,8 @@ class App extends React.Component<IProps, IState> {
     );
   }
 
+  // startup from bottom up
+
   joyApi() {
     console.debug(`Connecting to ${wsLocation}`);
     const provider = new WsProvider(wsLocation);
@@ -650,39 +258,78 @@ class App extends React.Component<IProps, IState> {
       await api.isReady;
       console.log(`Connected to ${wsLocation}`);
       this.setState({ connected: true });
-      this.handleApi(api);
+
+      api.rpc.chain.subscribeNewHeads(async (header: Header) => {
+        let { blocks, status } = this.state;
+        const id = header.number.toNumber();
+        if (blocks.find((b) => b.id === id)) return;
+        const timestamp = (await api.query.timestamp.now()).toNumber();
+        const duration = status.block
+          ? timestamp - status.block.timestamp
+          : 6000;
+        status.block = { id, timestamp, duration };
+        this.save("status", status);
+
+        blocks = blocks.filter((i) => i.id !== id).concat(status.block);
+        this.setState({ blocks });
+
+        const isEven = id / 50 === Math.floor(id / 50);
+        if (isEven || status.block?.id + 50 < id) this.updateStatus(api, id);
+      });
     });
   }
 
-  async fetchFromApi() {
-    this.fetchProposals();
-    this.updateForum();
-    this.fetchMembers();
-    this.fetchCouncils();
-    this.fetchStorageProviders();
-    this.fetchAssets();
-    //this.fetchFAQ();
+  save(key: string, data: any) {
+    this.setState({ [key]: data });
+    const value = JSON.stringify(data);
+    try {
+      localStorage.setItem(key, value);
+    } catch (e) {
+      const size = value.length / 1024;
+      console.warn(`Failed to save ${key} (${size.toFixed()} KB)`, e.message);
+    }
+    return data;
+  }
+
+  load(key: string) {
+    try {
+      const data = localStorage.getItem(key);
+      if (!data) return;
+      const size = data.length;
+      if (size > 10240)
+        console.debug(` -${key}: ${(size / 1024).toFixed(1)} KB`);
+      this.setState({ [key]: JSON.parse(data) });
+      return JSON.parse(data);
+    } catch (e) {
+      console.warn(`Failed to load ${key}`, e);
+    }
+  }
+
+  async loadData() {
+    console.debug(`Loading data`);
+    "status members assets providers councils council election workers categories channels proposals posts threads mints openings tokenomics transactions reports validators nominators staches stakes rewardPoints stars"
+      .split(" ")
+      .map((key) => this.load(key));
+    bootstrap(this.save); // axios requests
+    this.updateCouncils();
   }
 
   componentDidMount() {
-    this.joyApi();
-    this.loadData();
-    setTimeout(() => this.fetchTokenomics(), 30000);
-    //this.initializeSocket();
+    this.loadData(); // local storage + bootstrap
+    this.joyApi(); // joystream rpc connection
+    //this.initializeSocket() // jsstats socket.io
   }
-  componentWillUnmount() {}
+
   constructor(props: IProps) {
     super(props);
     this.state = initialState;
-    this.fetchTokenomics = this.fetchTokenomics.bind(this);
+    this.save = this.save.bind(this);
     this.load = this.load.bind(this);
     this.toggleEditKpi = this.toggleEditKpi.bind(this);
     this.toggleStar = this.toggleStar.bind(this);
     this.toggleFooter = this.toggleFooter.bind(this);
     this.toggleShowStatus = this.toggleShowStatus.bind(this);
     this.getMember = this.getMember.bind(this);
-    this.fetchProposals = this.fetchProposals.bind(this);
-    this.updateForum = this.updateForum.bind(this);
   }
 }
 

src/components/Dashboard/Forum.tsx

@@ -1,6 +1,5 @@
 import React from "react";
 import { Link } from "react-router-dom";
-import { RefreshCw } from "react-feather";
 import { LatestPost, Spinner } from "..";
 
 import { Post, Thread } from "../../types";
@@ -57,7 +56,6 @@ const Forum = (props: { posts: Post[]; threads: Thread[] }) => {
               <Link style={{ color: "#fff" }} to={"/forum"}>
                 Posts
               </Link>
-              <RefreshCw className="ml-2" onClick={props.updateForum} />
             </Typography>
           </Toolbar>
         </AppBar>

src/components/Dashboard/Proposals.tsx

@@ -1,5 +1,4 @@
 import { Link } from "react-router-dom";
-import { RefreshCw } from "react-feather";
 import { ProposalTable } from "..";
 import {
   createStyles,
@@ -24,7 +23,7 @@ const useStyles = makeStyles((theme: Theme) =>
       color: "#fff",
       minHeight: 600,
       maxHeight: 600,
-      overflow: "auto",
+      overflow: `hidden`,
     },
   })
 );
@@ -40,6 +39,7 @@ const Proposals = (props: {
   status: { council: Council };
 }) => {
   const classes = useStyles();
+
   const { proposals, validators, councils, members, posts, block, status } =
     props;
   const pending = proposals.filter((p) => p && p.result === "Pending");
@@ -51,7 +51,6 @@ const Proposals = (props: {
           <Toolbar>
             <Typography variant="h6" className={classes.title}>
               Active Proposals: {pending.length}
-              <RefreshCw className="ml-2" onClick={props.fetchProposals} />
             </Typography>
             <Link className="m-3 text-light" to={"/proposals"}>
               All
@@ -64,17 +63,19 @@ const Proposals = (props: {
             </Link>
           </Toolbar>
         </AppBar>
-        <ProposalTable
-          block={block}
-          hideNav={true}
-          proposals={pending}
-          members={members}
-          council={status.council}
-          councils={councils}
-          posts={posts}
-          status={status}
-          validators={validators}
-        />
+        <div className="h-100 overflow-auto">
+          <ProposalTable
+            block={block}
+            hideNav={true}
+            proposals={pending}
+            members={members}
+            council={status.council}
+            councils={councils}
+            posts={posts}
+            status={status}
+            validators={validators}
+          />
+        </div>
       </Paper>
     </Grid>
   );

src/components/Openings/Applications.tsx

@@ -18,7 +18,7 @@ const Applications = (props: { applications: Application[] }) => {
             <Details object={JSON.parse(a.application.human_readable_text)} />
           }
         >
-          <span className="ml-1">{a.member.handle || a.memberId}</span>
+          <span className="ml-1">{a.author || a.memberId}</span>
         </InfoTooltip>
       ))}
     </>

src/components/Openings/Group.tsx

@@ -1,15 +1,23 @@
 import GroupOpening from "./Opening";
+import { Member, Opening } from "./types";
 
-import { Opening } from "./types";
-
-const GroupOpenings = (props: { group: string; openings: Opening[] }) => {
-  const { group, openings } = props;
+const GroupOpenings = (props: {
+  members: Member[];
+  group: string;
+  openings: Opening[];
+}) => {
+  const { members, group, openings } = props;
   if (!openings?.length) return <div />;
   return (
     <div className="p-3">
       <h2>{group}</h2>
       {openings.map((opening) => (
-        <GroupOpening key={opening.openingId} group={group} opening={opening} />
+        <GroupOpening
+          key={opening.openingId}
+          members={members}
+          group={group}
+          opening={opening}
+        />
       ))}
     </div>
   );

src/components/Openings/Opening.tsx

@@ -6,7 +6,7 @@ import { domain } from "../../config";
 import { Opening } from "./types";
 
 const GroupOpening = (props: { group: string; opening: Opening }) => {
-  const { group, opening } = props;
+  const { group, members,opening } = props;
   const details = JSON.parse(opening.human_readable_text);
   return (
     <div>
@@ -20,7 +20,7 @@ const GroupOpening = (props: { group: string; opening: Opening }) => {
           {opening.type} opening ({opening.wgOpeningId})
         </a>
       </InfoTooltip>
-      <Applications applications={opening.applications} />
+      <Applications applications={opening.applications} members={members} />
     </div>
   );
 };

src/components/Openings/index.tsx

@@ -28,22 +28,23 @@ const useStyles = makeStyles((theme: Theme) =>
   })
 );
 
-const activeOpenings = (openings: Opening[]) =>
-  openings.filter(
+const activeOpenings = (openings: Opening[]): Opening[] => {
+  if (!openings?.length) return [];
+  return openings.filter(
     (o) => Object.keys(o.stage["active"].stage)[0] === "acceptingApplications"
   );
+};
 
 const Openings = (props: { openings: {} }) => {
   const classes = useStyles();
-  const { openings } = props;
+  const { members, openings } = props;
   if (!openings) return <div />;
-  const groups = Object.keys(openings).filter((g) => g !== "_lastUpdate");
-  const active = groups.reduce(
-    (sum, group) => sum + activeOpenings(openings[group]),
-    0
-  );
-  if (!active.length) return <div />;
+  const groups = Object.keys(openings).filter((g) => g !== "timestamp");
+  const active = groups
+    .map((group) => activeOpenings(openings[group]))
+    .reduce((sum: number, a: Opening[]) => sum + a.length, 0);
 
+  //if (!active.length) return <div />;
   return (
     <Grid className={classes.grid} item lg={6}>
       <Paper className={classes.paper}>
@@ -54,10 +55,15 @@ const Openings = (props: { openings: {} }) => {
             </Typography>
           </Toolbar>
         </AppBar>
+        <div className="m-3 text-center">
+          {active || `No`} active openings in {groups.length} group
+          {groups.length !== 1 ? `s` : ``}.
+        </div>
         <div>
           {groups.map((group) => (
             <GroupOpenings
               key={`${group}-openings`}
+              members={members}
               group={group}
               openings={activeOpenings(openings[group])}
             />

src/components/Proposals/ProposalTable.tsx

@@ -175,10 +175,10 @@ class ProposalTable extends React.Component<IProps, IState> {
           proposals={proposals.length}
         />
 
-        <div className="d-flex flex-column overflow-auto p-2">
+        <div className="d-flex flex-column p-2">
           {!proposals.length ? (
             <Spinner />
-          ) : (
+          ) : (	  
             proposals
               .slice((page - 1) * perPage, page * perPage)
               .map((p) => (

src/components/Validators/Leaderboard/config.ts

@@ -1,2 +1,2 @@
 export const maxEras = 8;
-export const maxValidators = 50;
+export const maxValidators = 100;

src/components/Validators/MinMax.tsx

@@ -1,6 +1,7 @@
 import { Table } from "react-bootstrap";
 import { Link } from "react-router-dom";
 import { Stakes } from "../../types";
+import Loading from "../Loading";
 
 const dollar = (d: number) => (d > 0 ? `$ ${d.toFixed(2)}` : "");
 
@@ -16,7 +17,7 @@ const MinMax = (props: {
   price: number;
 }) => {
   const { issued, stakes, validators, waiting, reward, price } = props;
-  if (!stakes || !Object.values(stakes).length) return <span />;
+  if (!stakes || !Object.values(stakes).length) return <Loading target={"stakes"} />
 
   let sum = 0;
   let minStake: number = 10000000;

src/lib/groups.ts

@@ -0,0 +1,171 @@
+import moment from "moment";
+import { Openings } from "./types";
+import { Mint } from "@joystream/types/mint";
+
+export const getMints = async (api: Api, ids: number[]): Promise<Mint[]> => {
+  console.debug(`Fetching mints`);
+  let mints: Mint[] = [];
+  const getMint = (id: number) => api.query.minting.mints(id);
+  ids.forEach(async (id) => (await getMint(id)).toJSON() as Mint);
+  return Promise.all(mints);
+};
+
+export const updateWorkers = async (
+  api: ApiPromise,
+  workers,
+  members: Member[]
+) => {
+  const lastUpdate = workers?.timestamp;
+  if (lastUpdate && moment() < moment(lastUpdate).add(1, `hour`))
+    return workers;
+  return {
+    content: await getGroupWorkers(api, "contentDirectory", members),
+    storage: await getGroupWorkers(api, "storage", members),
+    operations: await getGroupWorkers(api, "operations", members),
+    timestamp: moment().valueOf(),
+  };
+};
+
+const getGroupWorkers = async (
+  api: ApiPromise,
+  wg: string,
+  members: Member[]
+) => {
+  const group = wg + "WorkingGroup";
+  let workers = [];
+  const count = (
+    (await api.query[group].nextWorkerId()) as WorkerId
+  ).toNumber();
+  const lead = await api.query[group].currentLead();
+  console.debug(`Fetching ${count} ${wg} workers`);
+  for (let id = 0; id < count; ++id) {
+    const isLead = id === +lead;
+    const worker: WorkerOf = await api.query[group].workerById(id);
+    if (!worker.is_active) continue;
+    const memberId = worker.member_id.toJSON();
+    const member: Membership = members.find((m) => m.id === memberId);
+    const handle = member?.handle;
+    let stake: Stake;
+    let reward: RewardRelationship;
+
+    if (worker.role_stake_profile.isSome) {
+      const roleStakeProfile = worker.role_stake_profile.unwrap();
+      const stakeId = roleStakeProfile.stake_id;
+      const { staking_status } = (
+        await api.query.stake.stakes(stakeId)
+      ).toJSON();
+      stake = staking_status?.staked?.staked_amount;
+    }
+
+    if (worker.reward_relationship.isSome) {
+      const rewardId = worker.reward_relationship.unwrap();
+      reward = (
+        await api.query.recurringRewards.rewardRelationships(rewardId)
+      ).toJSON();
+    }
+    workers.push({
+      id,
+      memberId,
+      handle,
+      stake,
+      reward,
+      isLead,
+    });
+  }
+  return workers;
+};
+
+export const updateOpenings = async (
+  api: ApiPromise,
+  outdated: any,
+  members: Member[]
+) => {
+  const lastUpdate = outdated?.timestamp;
+  if (lastUpdate && moment() < moment(lastUpdate).add(1, `hour`))
+    return outdated;
+  console.debug(`Updating openings`);
+
+  // mapping: key = pioneer route, value: chain section
+  const groups = {
+    curators: "contentDirectory",
+    storageProviders: "storage",
+    operationsGroup: "operations",
+  };
+  let updated: Openings = {};
+  await Promise.all(
+    Object.keys(groups).map((group) =>
+      updateGroupOpenings(api, groups[group], outdated[group], members).then(
+        (openings) => (updated[group] = openings)
+      )
+    )
+  );
+  updated.timestamp = moment().valueOf();
+  return updated;
+};
+
+export const updateGroupOpenings = async (
+  api: ApiPromise,
+  wg: string,
+  outdated: Opening[],
+  members: Member[]
+) => {
+  const group = wg + "WorkingGroup";
+  const count = (
+    (await api.query[group].nextOpeningId()) as OpeningId
+  ).toNumber();
+  console.debug(` - Fetching ${count} ${wg} openings`);
+
+  const isActive = (opening: Opening) =>
+    Object.keys(opening.stage["active"].stage)[0] === "acceptingApplications";
+
+  let updated = [];
+  for (let wgOpeningId = 0; wgOpeningId < count; ++wgOpeningId) {
+    const old = outdated?.find((o) => o.wgOpeningId === wgOpeningId);
+    if (old && !isActive(old)) {
+      updated.push(old);
+      continue;
+    }
+    const wgOpening: OpeningOf = (
+      await api.query[group].openingById(wgOpeningId)
+    ).toJSON();
+    const ids = wgOpening.applications;
+    const openingId = wgOpening.hiring_opening_id;
+    const opening = (await api.query.hiring.openingById(openingId)).toJSON();
+    updated.push({
+      ...opening,
+      openingId,
+      wgOpeningId,
+      type: Object.keys(wgOpening.opening_type)[0],
+      applications: await getApplications(api, group, ids, members),
+      policy: wgOpening.policy_commitment,
+    });
+  }
+  console.debug(`${group} openings`, updated);
+  return updated;
+};
+
+export const getApplications = (
+  api: ApiPromise,
+  group: string,
+  ids: number[],
+  members: Member[]
+) => {
+  return Promise.all(
+    ids.map(async (wgApplicationId) => {
+      const wgApplication: ApplicationOf = (
+        await api.query[group].applicationById(wgApplicationId)
+      ).toJSON();
+      let application = {};
+      application.account = wgApplication.role_account_id;
+      application.openingId = +wgApplication.opening_id;
+      application.memberId = +wgApplication.member_id;
+      const member = members.find((m) => m.id === application.memberId);
+      if (member) application.author = member.handle;
+      application.id = +wgApplication.application_id;
+      application.application = (
+        await api.query.hiring.applicationById(application.id)
+      ).toJSON();
+      return application;
+    })
+  );
+};

src/lib/queries.ts

@@ -0,0 +1,92 @@
+import { apiLocation } from "../config";
+import { Tokenomics } from "../types";
+import { getAssets, getStorageProviders } from "./storage";
+import axios from "axios";
+
+export const queryJstats = (route: string) => {
+  const url = `${apiLocation}/${route}`;
+  return axios.get(url).then(({ data }) => {
+    if (data && !data.error) return data;
+    return console.error(`Jstats query failed: ${route}`, data);
+  });
+};
+
+export const getTokenomics = async (old?: Tokenomics) => {
+  if (old?.timestamp + 300000 > new Date()) return;
+  console.debug(`Updating tokenomics`);
+  let { data } = await axios.get("https://status.joystream.org/status");
+  if (!data || data.error) return;
+  data.timestamp = new Date();
+  return data;
+};
+
+export const getFAQ = async () => {
+  const { data } = await axios.get(
+    `https://joystreamstats.live/static/faq.json`
+  );
+  if (!data || data.error) return console.error(`failed to fetch from API`);
+  return data;
+};
+
+// Reports
+export const getReports = async () => {
+  const domain = `https://raw.githubusercontent.com/Joystream/community-repo/master/council-reports`;
+  const apiBase = `https://api.github.com/repos/joystream/community-repo/contents/council-reports`;
+
+  const urls: { [key: string]: string } = {
+    alexandria: `${apiBase}/alexandria-testnet`,
+    archive: `${apiBase}/archived-reports`,
+    template: `${domain}/templates/council_report_template_v1.md`,
+  };
+
+  ["alexandria", "archive"].map((folder) => getGithubDir(urls[folder]));
+
+  // template
+  getGithubFile(urls.template);
+};
+
+const getGithubFile = async (url: string): Promise<string> => {
+  const { data } = await axios.get(url);
+  return data;
+};
+
+const getGithubDir = async (url: string) => {
+  const { data } = await axios.get(url);
+
+  data.forEach(
+    async (o: {
+      name: string;
+      type: string;
+      url: string;
+      download_url: string;
+    }) => {
+      const match = o.name.match(/^(.+)\.md$/);
+      const name = match ? match[1] : o.name;
+      if (o.type === "file")
+        this.saveReport(name, this.fetchGithubFile(o.download_url));
+      else this.fetchGithubDir(o.url);
+    }
+  );
+};
+
+export const bootstrap = (save: (key: string, data: any) => {}) => {
+  [
+    { tokenomics: () => getTokenomics() },
+    //{ faq: () => getFAQ() },
+    //{ reports: () => getReports() },
+    { members: () => queryJstats(`/v2/members`) },
+    { proposals: () => queryJstats(`/v2/proposals`) },
+    { posts: () => queryJstats(`/v1/posts`) },
+    { threads: () => queryJstats(`/v1/threads`) },
+    { categories: () => queryJstats(`/v1/categories`) },
+    { proposals: () => queryJstats(`/v2/proposals`) },
+    { providers: () => getStorageProviders() },
+    { assets: () => getAssets() },
+  ].reduce(async (promise, request) => {
+    //promise.then(async () => {
+    const key = Object.keys(request)[0];
+    console.debug(`Requesting ${key}`);
+    return save(key, await request[key]());
+    //}, new Promise((res) => res))
+  });
+};

src/lib/storage.ts

@@ -0,0 +1,60 @@
+import axios from "axios";
+
+export const getAssets = async () => {
+  const url = "https://hydra.joystream.org/graphql";
+  const request = {
+    query: "query {\n dataObjects(where: {}) { joystreamContentId }\n}",
+  };
+  console.debug(`Fetching data IDs from ${url}`);
+  const { data } = await axios.post(url, request);
+  let assets = [];
+  data.data.dataObjects.forEach((p) => assets.push(p.joystreamContentId));
+  return data;
+};
+
+export const getStorageProviders = async () => {
+  const url = "https://hydra.joystream.org/graphql";
+  const request = {
+    query:
+      'query {\n  workers(where: {metadata_contains: "http", isActive_eq: true, type_eq: STORAGE}){\n    metadata\n  }\n}',
+  };
+  console.debug(`Fetching storage providers from ${url}`);
+  const { data } = await axios.post(url, request);
+  const providers = data.data.workers.map((p) => {
+    return {
+      url: p.metadata,
+    };
+  });
+  return providers;
+};
+
+export const getStorageProvidersFromApi = async (api: Api) => {
+  console.debug(`Fetching storage providers (from chain)`);
+  let providers = [];
+  const worker = await api.query.storageWorkingGroup.nextWorkerId();
+  console.log(`next provider: ${worker}`);
+
+  for (let i = 0; i < Number(worker); ++i) {
+    let storageProvider = (await api.query.storageWorkingGroup.workerById(
+      i
+    )) as WorkerOf;
+    if (storageProvider.is_active) {
+      const storage = (await api.query.storageWorkingGroup.workerStorage(
+        i
+      )) as Bytes;
+      const url = Buffer.from(storage.toString().substr(2), "hex").toString();
+
+      let membership = (await api.query.members.membershipById(
+        storageProvider.member_id
+      )) as Membership;
+
+      providers[i] = {
+        owner: membership.handle,
+        account: membership.root_account,
+        storage,
+        url,
+      };
+    }
+    return providers;
+  }
+};

src/lib/validators.ts

@@ -25,9 +25,12 @@ export const getValidatorStakes = async (
   api: ApiPromise,
   era: number,
   stashes: string[],
-  members: Member[]
+  members: Member[],
+  save: (key: string, data: any) => {}
 ) => {
-  return stashes.reduce(async (allAtakes, validator: string) => {
+  console.log(`Updating stakes`);
+  let stakes = {};
+  for (const validator of stashes) {
     const prefs = await api.query.staking.erasValidatorPrefs(era, validator);
     const commission = Number(prefs.commission) / 10000000;
 
@@ -35,11 +38,13 @@ export const getValidatorStakes = async (
     let { total, own, others } = data.toJSON();
 
     others = others.map(({ who, value }) => {
-      const member = members.find((m) => m.rootKey === who);
+      const member = members.find((m) => m.rootKey === String(who));
       return { who, value, member };
     });
-    return allStakes.concat({ total, own, others, commission });
-  }, []);
+    stakes[validator] = { total, own, others, commission };
+    save(`stakes`, stakes);
+  }
+  return stakes;
 };
 
 export const getEraRewardPoints = async (api: Api, era: EraId | number) =>

src/state.ts

@@ -0,0 +1,42 @@
+import { domain } from "./config";
+
+export const initialState = {
+  assets: [],
+  connected: false,
+  faq: [],
+  fetching: "",
+  tasks: 0,
+  blocks: [],
+  nominators: [],
+  validators: [],
+  mints: [],
+  channels: [],
+  posts: [],
+  councils: [],
+  election: {
+    applicants: [],
+    votes: [],
+    councilSize: 20,
+  },
+  categories: [],
+  threads: [],
+  proposals: [],
+  domain,
+  members: [],
+  providers: [],
+  reports: {},
+  stakes: {},
+  stashes: [],
+  stars: {},
+  hideFooter: true,
+  showStatus: false,
+  editKpi: false,
+  status: { era: 0, block: { id: 0, era: 0, timestamp: 0, duration: 6 } },
+  groups: [],
+  openings: {},
+  rewardPoints: {
+    total: 0,
+    eraTotals: {},
+    validators: {},
+  },
+};