merged nicaea

.github/workflows/network-tests.yml

@@ -0,0 +1,37 @@
+name: network-tests
+on: [pull_request, push]
+
+jobs:
+  network_build_ubuntu:
+    name: Ubuntu Checks
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: checks
+      run: |
+        yarn install --frozen-lockfile
+        yarn workspace joystream-testing checks
+
+  network_build_osx:
+    name: MacOS Checks
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: checks
+      run: |
+        yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace joystream-testing checks

.github/workflows/storage-node.yml

@@ -0,0 +1,37 @@
+name: storage-node
+on: [pull_request, push]
+
+jobs:
+  storage_node_build_ubuntu:
+    name: Ubuntu Checks
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: checks
+      run: |
+        yarn install --frozen-lockfile
+        yarn workspace storage-node checks
+
+  storage_node_build_osx:
+    name: MacOS Checks
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        node-version: [12.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: checks
+      run: |
+        yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace storage-node checks

.travis.yml

@@ -17,6 +17,18 @@ matrix:
     - os: linux
       env: TARGET=x86_64-unknown-linux-gnu
 
+# Skip Rust build in a pull request if no rust project files were modified
+before_install:
+  - |
+    if [ "$TRAVIS_PULL_REQUEST" != "false" ]
+      then
+      if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qE "(.rs|Cargo.(lock|toml))$"
+        then
+        echo "No changes to Rust or Cargo Files, CI not running."
+        travis_terminate 0
+      fi
+    fi
+
 install:
   - rustup install nightly-2020-05-23
   - rustup target add wasm32-unknown-unknown --toolchain nightly-2020-05-23

cli/package.json

@@ -8,7 +8,7 @@
   },
   "bugs": "https://github.com/Joystream/substrate-runtime-joystream/issues",
   "dependencies": {
-    "@joystream/types": "^0.11.0",
+    "@joystream/types": "^0.12.0",
     "@oclif/command": "^1.5.19",
     "@oclif/config": "^1.14.0",
     "@oclif/plugin-help": "^2.2.3",

pioneer/package.json

@@ -82,6 +82,6 @@
     "node-sass": "^4.13.0",
     "sass-loader": "^8.0.0",
     "style-loader": "^1.0.0",
-    "@joystream/types": "./types"
+    "@joystream/types": "link:../types"
   }
 }

pioneer/packages/joy-proposals/src/Proposal/Body.tsx

@@ -3,7 +3,6 @@ import { Link } from 'react-router-dom';
 import { Card, Header, Button, Icon, Message } from 'semantic-ui-react';
 import { ProposalType } from '@polkadot/joy-utils/types/proposals';
 import { bytesToString } from '@polkadot/joy-utils/functions/misc';
-import { blake2AsHex } from '@polkadot/util-crypto';
 import styled from 'styled-components';
 import AddressMini from '@polkadot/react-components/AddressMiniJoy';
 import TxButton from '@polkadot/joy-utils/TxButton';
@@ -110,13 +109,10 @@ const paramParsers: { [x in ProposalType]: (params: any[]) => ParsedParam[]} = {
       true
     )
   ],
-  RuntimeUpgrade: ([wasm]) => {
-    const buffer: Buffer = Buffer.from(wasm.replace('0x', ''), 'hex');
-    return [
-      new ParsedParam('Blake2b256 hash of WASM code', blake2AsHex(buffer, 256), true),
-      new ParsedParam('File size', buffer.length + ' bytes')
-    ];
-  },
+  RuntimeUpgrade: ([hash, filesize]) => [
+    new ParsedParam('Blake2b256 hash of WASM code', hash, true),
+    new ParsedParam('File size', filesize + ' bytes')
+  ],
   SetElectionParameters: ([params]) => [
     new ParsedParam('Announcing period', params.announcing_period + ' blocks'),
     new ParsedParam('Voting period', params.voting_period + ' blocks'),

pioneer/packages/joy-proposals/src/Proposal/ProposalFromId.tsx

@@ -3,6 +3,7 @@ import { RouteComponentProps } from 'react-router-dom';
 import ProposalDetails from './ProposalDetails';
 import { useProposalSubscription } from '@polkadot/joy-utils/react/hooks';
 import { PromiseComponent } from '@polkadot/joy-utils/react/components';
+import { ProposalId } from '@joystream/types/proposals';
 
 export default function ProposalFromId (props: RouteComponentProps<any>) {
   const {
@@ -11,7 +12,7 @@ export default function ProposalFromId (props: RouteComponentProps<any>) {
     }
   } = props;
 
-  const { proposal: proposalState, votes: votesState } = useProposalSubscription(id);
+  const { proposal: proposalState, votes: votesState } = useProposalSubscription(new ProposalId(id));
 
   return (
     <PromiseComponent

pioneer/packages/joy-utils/src/react/context/transport.tsx

@@ -1,4 +1,4 @@
-import React, { createContext, useContext } from 'react';
+import React, { createContext, useContext, useState } from 'react';
 import { ApiContext } from '@polkadot/react-api';
 import { ApiProps } from '@polkadot/react-api/types';
 
@@ -15,7 +15,8 @@ export function TransportProvider ({ children }: { children: React.PropsWithChil
     throw new Error('Cannot create Transport: The Substrate API is not ready yet.');
   }
 
-  const transport = new Transport(api.api);
+  // Preserve transport instance in state to allow more effective caching
+  const [transportInstance] = useState<Transport>(() => new Transport(api.api));
 
-  return <TransportContext.Provider value={transport}>{children}</TransportContext.Provider>;
+  return <TransportContext.Provider value={transportInstance}>{children}</TransportContext.Provider>;
 }

pioneer/packages/joy-utils/src/transport/base.ts

@@ -1,59 +1,62 @@
 import { ApiPromise } from '@polkadot/api';
 import { Codec } from '@polkadot/types/types';
+import { APIQueryCache } from '../APIQueryCache';
 
 export default abstract class BaseTransport {
   protected api: ApiPromise;
+  protected cacheApi: APIQueryCache;
 
-  constructor (api: ApiPromise) {
+  constructor (api: ApiPromise, cacheApi: APIQueryCache) {
     this.api = api;
+    this.cacheApi = cacheApi;
   }
 
   protected get proposalsEngine () {
-    return this.api.query.proposalsEngine;
+    return this.cacheApi.query.proposalsEngine;
   }
 
   protected get proposalsCodex () {
-    return this.api.query.proposalsCodex;
+    return this.cacheApi.query.proposalsCodex;
   }
 
   protected get proposalsDiscussion () {
-    return this.api.query.proposalsDiscussion;
+    return this.cacheApi.query.proposalsDiscussion;
   }
 
   protected get members () {
-    return this.api.query.members;
+    return this.cacheApi.query.members;
   }
 
   protected get council () {
-    return this.api.query.council;
+    return this.cacheApi.query.council;
   }
 
   protected get councilElection () {
-    return this.api.query.councilElection;
+    return this.cacheApi.query.councilElection;
   }
 
   protected get actors () {
-    return this.api.query.actors;
+    return this.cacheApi.query.actors;
   }
 
   protected get contentWorkingGroup () {
-    return this.api.query.contentWorkingGroup;
+    return this.cacheApi.query.contentWorkingGroup;
   }
 
   protected get minting () {
-    return this.api.query.minting;
+    return this.cacheApi.query.minting;
   }
 
   protected get hiring () {
-    return this.api.query.hiring;
+    return this.cacheApi.query.hiring;
   }
 
   protected get stake () {
-    return this.api.query.stake;
+    return this.cacheApi.query.stake;
   }
 
   protected get recurringRewards () {
-    return this.api.query.recurringRewards;
+    return this.cacheApi.query.recurringRewards;
   }
 
   protected queryMethodByName (name: string) {

pioneer/packages/joy-utils/src/transport/contentWorkingGroup.ts

@@ -5,12 +5,13 @@ import { MintId, Mint } from '@joystream/types/mint';
 import { LeadId } from '@joystream/types/content-working-group';
 import { ApiPromise } from '@polkadot/api';
 import MembersTransport from './members';
+import { APIQueryCache } from '../APIQueryCache';
 
 export default class ContentWorkingGroupTransport extends BaseTransport {
   private membersT: MembersTransport;
 
-  constructor (api: ApiPromise, membersTransport: MembersTransport) {
-    super(api);
+  constructor (api: ApiPromise, cacheApi: APIQueryCache, membersTransport: MembersTransport) {
+    super(api, cacheApi);
     this.membersT = membersTransport;
   }
 

pioneer/packages/joy-utils/src/transport/council.ts

@@ -8,13 +8,14 @@ import { FIRST_MEMBER_ID } from '../consts/members';
 import { ApiPromise } from '@polkadot/api';
 import MembersTransport from './members';
 import ChainTransport from './chain';
+import { APIQueryCache } from '../APIQueryCache';
 
 export default class CouncilTransport extends BaseTransport {
   private membersT: MembersTransport;
   private chainT: ChainTransport;
 
-  constructor (api: ApiPromise, membersTransport: MembersTransport, chainTransport: ChainTransport) {
-    super(api);
+  constructor (api: ApiPromise, cacheApi: APIQueryCache, membersTransport: MembersTransport, chainTransport: ChainTransport) {
+    super(api, cacheApi);
     this.membersT = membersTransport;
     this.chainT = chainTransport;
   }
@@ -22,7 +23,7 @@ export default class CouncilTransport extends BaseTransport {
   async councilMembersLength (atBlock?: number): Promise<number> {
     if (atBlock) {
       const blockHash = await this.chainT.blockHash(atBlock);
-      return ((await this.council.activeCouncil.at(blockHash)) as Seats).length;
+      return ((await this.api.query.council.activeCouncil.at(blockHash)) as Seats).length;
     }
 
     return ((await this.council.activeCouncil()) as Seats).length;

pioneer/packages/joy-utils/src/transport/index.ts

@@ -7,9 +7,11 @@ import CouncilTransport from './council';
 import StorageProvidersTransport from './storageProviders';
 import ValidatorsTransport from './validators';
 import WorkingGroupsTransport from './workingGroups';
+import { APIQueryCache } from '../APIQueryCache';
 
 export default class Transport {
   protected api: ApiPromise;
+  protected cacheApi: APIQueryCache;
   // Specific transports
   public chain: ChainTransport;
   public members: MembersTransport;
@@ -22,13 +24,14 @@ export default class Transport {
 
   constructor (api: ApiPromise) {
     this.api = api;
-    this.chain = new ChainTransport(api);
-    this.members = new MembersTransport(api);
-    this.storageProviders = new StorageProvidersTransport(api);
-    this.validators = new ValidatorsTransport(api);
-    this.council = new CouncilTransport(api, this.members, this.chain);
-    this.contentWorkingGroup = new ContentWorkingGroupTransport(api, this.members);
-    this.proposals = new ProposalsTransport(api, this.members, this.chain, this.council);
-    this.workingGroups = new WorkingGroupsTransport(api, this.members);
+    this.cacheApi = new APIQueryCache(api);
+    this.chain = new ChainTransport(api, this.cacheApi);
+    this.members = new MembersTransport(api, this.cacheApi);
+    this.storageProviders = new StorageProvidersTransport(api, this.cacheApi);
+    this.validators = new ValidatorsTransport(api, this.cacheApi);
+    this.council = new CouncilTransport(api, this.cacheApi, this.members, this.chain);
+    this.contentWorkingGroup = new ContentWorkingGroupTransport(api, this.cacheApi, this.members);
+    this.proposals = new ProposalsTransport(api, this.cacheApi, this.members, this.chain, this.council);
+    this.workingGroups = new WorkingGroupsTransport(api, this.cacheApi, this.members);
   }
 }

pioneer/packages/joy-utils/src/transport/proposals.ts

@@ -13,9 +13,9 @@ import { ParsedMember } from '../types/members';
 import BaseTransport from './base';
 
 import { ThreadId, PostId } from '@joystream/types/common';
-import { Proposal, ProposalId, VoteKind, DiscussionThread, DiscussionPost } from '@joystream/types/proposals';
+import { Proposal, ProposalId, VoteKind, DiscussionThread, DiscussionPost, ProposalDetails } from '@joystream/types/proposals';
 import { MemberId } from '@joystream/types/members';
-import { u32, u64 } from '@polkadot/types/';
+import { u32, u64, Bytes, Vec } from '@polkadot/types/';
 import { BalanceOf } from '@polkadot/types/interfaces';
 
 import { bytesToString } from '../functions/misc';
@@ -28,43 +28,80 @@ import MembersTransport from './members';
 import ChainTransport from './chain';
 import CouncilTransport from './council';
 
+import { blake2AsHex } from '@polkadot/util-crypto';
+import { APIQueryCache } from '../APIQueryCache';
+
+type ProposalDetailsCacheEntry = {
+  type: ProposalType;
+  details: any[];
+}
+type ProposalDetailsCache = {
+  [id: number]: ProposalDetailsCacheEntry | undefined;
+}
+
 export default class ProposalsTransport extends BaseTransport {
   private membersT: MembersTransport;
   private chainT: ChainTransport;
   private councilT: CouncilTransport;
+  private proposalDetailsCache: ProposalDetailsCache = {};
 
   constructor (
     api: ApiPromise,
+    cacheApi: APIQueryCache,
     membersTransport: MembersTransport,
     chainTransport: ChainTransport,
     councilTransport: CouncilTransport
   ) {
-    super(api);
+    super(api, cacheApi);
     this.membersT = membersTransport;
     this.chainT = chainTransport;
     this.councilT = councilTransport;
   }
 
   proposalCount () {
-    return this.proposalsEngine.proposalCount<u32>();
+    return this.proposalsEngine.proposalCount() as Promise<u32>;
   }
 
   rawProposalById (id: ProposalId) {
-    return this.proposalsEngine.proposals<Proposal>(id);
+    return this.proposalsEngine.proposals(id) as Promise<Proposal>;
   }
 
-  proposalDetailsById (id: ProposalId) {
-    return this.proposalsCodex.proposalDetailsByProposalId(id);
+  rawProposalDetails (id: ProposalId) {
+    return this.proposalsCodex.proposalDetailsByProposalId(id) as Promise<ProposalDetails>;
   }
 
   cancellationFee (): number {
     return (this.api.consts.proposalsEngine.cancellationFee as BalanceOf).toNumber();
   }
 
+  async typeAndDetails (id: ProposalId) {
+    const cachedProposalDetails = this.proposalDetailsCache[id.toNumber()];
+    // Avoid fetching runtime upgrade proposal details if we already have them cached
+    if (cachedProposalDetails) {
+      return cachedProposalDetails;
+    } else {
+      // TODO: The right typesafe handling with JoyEnum would be very useful here
+      const rawDetails = await this.rawProposalDetails(id);
+      const type = rawDetails.type as ProposalType;
+      let details: any[];
+      if (type === 'RuntimeUpgrade') {
+        // In case of RuntimeUpgrade proposal we override details to just contain the hash and filesize
+        // (instead of full WASM bytecode)
+        const wasm = rawDetails.value as Bytes;
+        details = [blake2AsHex(wasm, 256), wasm.length];
+      } else {
+        const detailsJSON = rawDetails.value.toJSON();
+        details = Array.isArray(detailsJSON) ? detailsJSON : [detailsJSON];
+      }
+      // Save entry in cache
+      this.proposalDetailsCache[id.toNumber()] = { type, details };
+
+      return { type, details };
+    }
+  }
+
   async proposalById (id: ProposalId): Promise<ParsedProposal> {
-    const rawDetails = (await this.proposalDetailsById(id)).toJSON() as { [k: string]: any };
-    const type = Object.keys(rawDetails)[0] as ProposalType;
-    const details = Array.isArray(rawDetails[type]) ? rawDetails[type] : [rawDetails[type]];
+    const { type, details } = await this.typeAndDetails(id);
     const rawProposal = await this.rawProposalById(id);
     const proposer = (await this.membersT.memberProfile(rawProposal.proposerId)).toJSON() as ParsedMember;
     const proposal = rawProposal.toJSON() as {
@@ -102,7 +139,7 @@ export default class ProposalsTransport extends BaseTransport {
   }
 
   async activeProposals () {
-    const activeProposalIds = await this.proposalsEngine.activeProposalIds<ProposalId[]>();
+    const activeProposalIds = (await this.proposalsEngine.activeProposalIds()) as Vec<ProposalId>;
 
     return Promise.all(activeProposalIds.map(id => this.proposalById(id)));
   }
@@ -112,13 +149,9 @@ export default class ProposalsTransport extends BaseTransport {
     return proposals.filter(({ proposerId }) => member.eq(proposerId));
   }
 
-  async proposalDetails (id: ProposalId) {
-    return this.proposalsCodex.proposalDetailsByProposalId(id);
-  }
-
   async voteByProposalAndMember (proposalId: ProposalId, voterId: MemberId): Promise<VoteKind | null> {
-    const vote = await this.proposalsEngine.voteExistsByProposalByVoter<VoteKind>(proposalId, voterId);
-    const hasVoted = (await this.proposalsEngine.voteExistsByProposalByVoter.size(proposalId, voterId)).toNumber();
+    const vote = (await this.proposalsEngine.voteExistsByProposalByVoter(proposalId, voterId)) as VoteKind;
+    const hasVoted = (await this.api.query.proposalsEngine.voteExistsByProposalByVoter.size(proposalId, voterId)).toNumber();
     return hasVoted ? vote : null;
   }
 
@@ -177,7 +210,7 @@ export default class ProposalsTransport extends BaseTransport {
   }
 
   async subscribeProposal (id: number|ProposalId, callback: () => void) {
-    return this.proposalsEngine.proposals(id, callback);
+    return this.api.query.proposalsEngine.proposals(id, callback);
   }
 
   async discussion (id: number|ProposalId): Promise<ParsedDiscussion | null> {

pioneer/packages/joy-utils/src/transport/workingGroups.ts

@@ -12,18 +12,19 @@ import { OpeningId, ApplicationId, Opening, Application, ActiveOpeningStageKey }
 import { MultipleLinkedMapEntry } from '../LinkedMapEntry';
 import { Stake, StakeId } from '@joystream/types/stake';
 import { RewardRelationshipId, RewardRelationship } from '@joystream/types/recurring-rewards';
+import { APIQueryCache } from '../APIQueryCache';
 
 export default class WorkingGroupsTransport extends BaseTransport {
   private membersT: MembersTransport;
 
-  constructor (api: ApiPromise, membersTransport: MembersTransport) {
-    super(api);
+  constructor (api: ApiPromise, cacheApi: APIQueryCache, membersTransport: MembersTransport) {
+    super(api, cacheApi);
     this.membersT = membersTransport;
   }
 
   protected queryByGroup (group: WorkingGroupKey) {
     const module = apiModuleByGroup[group];
-    return this.api.query[module];
+    return this.cacheApi.query[module];
   }
 
   public async groupMemberById (group: WorkingGroupKey, workerId: number): Promise<WorkerData | null> {

storage-node/.eslintrc.js

@@ -13,6 +13,7 @@ module.exports = {
     'no-console': 'off', // we use console in the project
     '@typescript-eslint/no-var-requires': 'warn',
     '@typescript-eslint/camelcase': 'warn',
+    '@typescript-eslint/naming-convention': 'off',
   },
   overrides: [
     {

storage-node/package.json

@@ -32,7 +32,7 @@
   ],
   "scripts": {
     "test": "wsrun --serial test",
-    "lint": "eslint --ignore-path .gitignore .",
+    "lint": "eslint --ext .js,.ts --ignore-path .gitignore .",
     "build": "yarn workspace @joystream/storage-cli run build",
     "checks": "yarn lint && prettier . --check",
     "format": "prettier . --write"

storage-node/packages/cli/package.json

@@ -28,7 +28,7 @@
   },
   "scripts": {
     "test": "mocha 'dist/test/**/*.js'",
-    "lint": "eslint --ext .ts,.tsx . && tsc --noEmit --pretty",
+    "lint": "eslint --ext .js,.ts . && tsc --noEmit --pretty",
     "build": "tsc --build"
   },
   "bin": {
@@ -44,7 +44,7 @@
     "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/service-discovery": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
-    "@joystream/types": "^0.11.0",
+    "@joystream/types": "^0.12.0",
     "axios": "^0.19.2",
     "chalk": "^2.4.2",
     "lodash": "^4.17.11",

storage-node/packages/runtime-api/package.json

@@ -45,7 +45,7 @@
     "temp": "^0.9.0"
   },
   "dependencies": {
-    "@joystream/types": "^0.11.0",
+    "@joystream/types": "^0.12.0",
     "@polkadot/api": "^0.96.1",
     "async-lock": "^1.2.0",
     "lodash": "^4.17.11",

types/README.md

@@ -31,7 +31,7 @@ async function main () {
   registerJoystreamTypes();
 
   // Create the API and wait until ready
-  const api = await ApiPromise.create(provider);
+  const api = await ApiPromise.create({ provider });
 
   // Retrieve the chain & node information information via RPC calls
   const [chain, nodeName, nodeVersion] = await Promise.all([
@@ -46,13 +46,6 @@ async function main () {
 main();
 ```
 
-## Publishing Instructions
-The package is published with typescipt sources pre-compiled, automated with prepublish hook in package.json
-
-```
-npm publish
-```
-
-## Users
+## Examples
 
 See [joystream-api-examples](https://github.com/Joystream/joystream-api-examples) for some additional examples on usage.

types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joystream/types",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Types for Joystream Substrate Runtime - nicaea release",
   "main": "index.js",
   "types": "index.d.ts",

yarn.lock

@@ -1828,8 +1828,8 @@
     "@types/istanbul-reports" "^1.1.1"
     "@types/yargs" "^13.0.0"
 
-"@joystream/types@./types":
-  version "0.11.0"
+"@joystream/types@link:types", "@nicaea/types@link:types":
+  version "0.12.0"
   dependencies:
     "@polkadot/keyring" "^1.7.0-beta.5"
     "@polkadot/types" "^0.96.1"
@@ -2584,10 +2584,6 @@
     call-me-maybe "^1.0.1"
     glob-to-regexp "^0.3.0"
 
-"@nicaea/types@link:types":
-  version "0.0.0"
-  uid ""
-
 "@nodelib/fs.scandir@2.1.3":
   version "2.1.3"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz#3a582bdb53804c6ba6d146579c46e52130cf4a3b"