Merge branch 'babylon' into babylon-types

Cargo.lock

@@ -711,12 +711,12 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ee0cc8804d5393478d743b035099520087a5186f3b93fa58cec08fa62407b6"
+checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
 dependencies = [
- "cfg-if",
  "crossbeam-utils",
+ "maybe-uninit",
 ]
 
 [[package]]
@@ -1993,7 +1993,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node"
-version = "3.1.0"
+version = "3.3.1"
 dependencies = [
  "frame-benchmarking",
  "frame-benchmarking-cli",
@@ -2053,7 +2053,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node-runtime"
-version = "7.3.0"
+version = "7.5.1"
 dependencies = [
  "frame-benchmarking",
  "frame-executive",

README.md

@@ -94,7 +94,7 @@ You can also run your our own joystream-node:
 ```sh
 git checkout master
 WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
-./target/release/joystream-node -- --pruning archive --chain testnets/rome.json
+./target/release/joystream-node -- --pruning archive --chain testnets/joy-testnet-4.json
 ```
 
 Wait for the node to sync to the latest block, then change pioneer settings "remote node" option to "Local Node", or follow the link below:

cli/package.json

@@ -15,7 +15,7 @@
     "@oclif/plugin-help": "^2.2.3",
     "@oclif/plugin-not-found": "^1.2.4",
     "@oclif/plugin-warn-if-update-available": "^1.7.0",
-    "@polkadot/api": "^0.96.1",
+    "@polkadot/api": "1.26.1",
     "@types/inquirer": "^6.5.0",
     "@types/proper-lockfile": "^4.1.1",
     "@types/slug": "^0.9.1",

devops/dockerfiles/node-and-runtime/Dockerfile

@@ -3,14 +3,15 @@ LABEL description="Compiles all workspace artifacts"
 WORKDIR /joystream
 COPY . /joystream
 
-# Build joystream-node and its dependencies - runtime
-RUN WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release -p joystream-node
+# Build all cargo crates
+RUN WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
 
 FROM debian:stretch
 LABEL description="Joystream node"
 WORKDIR /joystream
 COPY --from=builder /joystream/target/release/joystream-node /joystream/node
 COPY --from=builder /joystream/target/release/wbuild/joystream-node-runtime/joystream_node_runtime.compact.wasm /joystream/runtime.compact.wasm
+COPY --from=builder /joystream/target/release/chain-spec-builder /joystream/chain-spec-builder
 
 # confirm it works
 RUN /joystream/node --version

node/Cargo.toml

@@ -3,7 +3,7 @@ authors = ['Joystream contributors']
 build = 'build.rs'
 edition = '2018'
 name = 'joystream-node'
-version = '3.1.0'
+version = '3.3.1'
 default-run = "joystream-node"
 
 [[bin]]

node/README.md

@@ -49,7 +49,7 @@ this script will build and run a fresh new local development chain (purging exis
 Use the `--chain` argument, and specify the path to the genesis `chain.json` file for that public network. The JSON "chain spec" files for Joystream public networks can be found in [../testnets/](../testnets/).
 
 ```bash
-./target/release/joystream-node --chain testnets/rome.json
+./target/release/joystream-node --chain testnets/joy-testnet-4.json
 ```
 
 ### Tests and code quality
@@ -57,7 +57,7 @@ Use the `--chain` argument, and specify the path to the genesis `chain.json` fil
 Running unit tests:
 
 ```bash
-cargo test --all
+cargo test --release --all
 ```
 
 Running full suite of checks, tests, formatting and linting:
@@ -75,7 +75,7 @@ cargo fmt --all
 ### Integration tests
 
 ```bash
-./scripts/run-test-chain.sh
+./scripts/run-dev-chain.sh
 yarn workspace joystream-testing test
 ```
 
@@ -88,11 +88,11 @@ If you are building a tagged release from `master` branch and want to install th
 This will install the executable `joystream-node` to your `~/.cargo/bin` folder, which you would normally have in your `$PATH` environment.
 
 ```bash
-cargo install joystream-node --path node/
+WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo install joystream-node --path node/ --locked
 ```
 
-Now you can run and connect to the Rome testnet:
+Now you can run and connect to the testnet:
 
 ```bash
-joystream-node --chain testnets/rome.json
+joystream-node --chain testnets/joy-testnet-4.json
 ```

node/src/chain_spec/mod.rs

@@ -357,9 +357,9 @@ pub fn testnet_genesis(
             fill_working_group_leader_opening_proposal_grace_period: cpcp
                 .fill_working_group_leader_opening_proposal_grace_period,
             set_working_group_mint_capacity_proposal_voting_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_voting_period,
+                .set_working_group_mint_capacity_proposal_voting_period,
             set_working_group_mint_capacity_proposal_grace_period: cpcp
-                .set_content_working_group_mint_capacity_proposal_grace_period,
+                .set_working_group_mint_capacity_proposal_grace_period,
             decrease_working_group_leader_stake_proposal_voting_period: cpcp
                 .decrease_working_group_leader_stake_proposal_voting_period,
             decrease_working_group_leader_stake_proposal_grace_period: cpcp

pioneer/packages/apps/public/index.html

@@ -18,17 +18,104 @@
       </script>
     <% } %>
     <script type="text/javascript" src="/env-config.js"></script>
+    <% if (!htmlWebpackPlugin.options.IS_LIVE) { %>
+      <meta name="viewport" content="width=device-width">
+      <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;700&display=swap" rel="stylesheet">
+      <style>
+        body {
+          background: #000 !important;
+          color: #fff !important;
+          min-height: 100vh !important;
+          margin: 0 !important;
+          padding: 0 !important;
+          padding-top: 20vh !important;
+          box-sizing: border-box;
+          font-family: 'Inter', sans-serif !important;
+        }
+        header {
+          width: 100%;
+          background: #261EE4;
+        }
+        .info-flexbox {
+          display: flex;
+          width: 80%;
+          max-width: 800px;
+          margin: 0 auto;
+          align-items: flex-end;
+          padding: 20px;
+        }
+        h2 {
+          font-weight: normal !important;
+          margin-bottom: 20px;
+        }
+        .blog-link {
+          margin-left: auto;
+          color: #000 !important;
+          background: #6C6CFF;
+          padding: 10px 20px;
+          font-size: 16px;
+          text-decoration: none;
+          transition: background-color 0.2s;
+        }
+        .blog-link:hover {
+          background: #fff;
+        }
+        .box-content {
+          display: flex;
+          flex-direction: column;
+          justify-content: space-between;
+          margin-top: 20px;
+        }
+        .logo-alexandria {
+          width: 30%;
+          flex-shrink: 0;
+          margin-right: 30px;
+        }
+        @media screen and (max-width: 800px) {
+          h2 {
+            font-size: 20px !important;
+          }
+        }
+        @media screen and (max-width: 500px) {
+          body {
+            padding: 50px 0 !important;
+          }
+          .info-flexbox {
+            flex-wrap: wrap;
+          }
+          .info-flexbox > * {
+            width: 100% !important;
+          }
+        }
+      </style>
+    <% } %>
   </head>
   <body>
-    <noscript>
-      You need to enable JavaScript to run this app.
-    </noscript>
-    <div id="root"></div>
-    <div id="tooltips"></div>
-    <script>
-      if (window.self !== window.top) {
-        window.top.location.href = window.location.href;
-      }
-    </script>
+    <% if (htmlWebpackPlugin.options.IS_LIVE) { %>
+      <noscript>
+        You need to enable JavaScript to run this app.
+      </noscript>
+      <div id="root"></div>
+      <div id="tooltips"></div>
+      <script>
+        if (window.self !== window.top) {
+          window.top.location.href = window.location.href;
+        }
+      </script>
+    <% } else { %>
+      <header>
+        <div class="info-flexbox">
+          <img src="./Logo_Alexandrial.svg" class="logo-alexandria">
+          <div class="box-content">
+            <h2>
+              Our new testnet - Alexandria will be launched
+              around&nbsp;09:00&nbsp;UTC on&nbsp;Monday the&nbsp;21st
+              of&nbsp;September&nbsp;2020.
+            </h2>
+            <a href="https://blog.joystream.org/announcing-alexandria/" target="_blank" class="blog-link">Find out more</a>
+          </div>
+        </div>
+      </header>
+    <% } %>
   </body>
 </html>

pioneer/packages/apps/public/locales/en/index.json

@@ -24,6 +24,7 @@
   "joy-election.json",
   "joy-media.json",
   "joy-members.json",
+  "joy-proposals.json",
   "joy-roles.json",
   "joy-utils.json",
   "react-components.json",

pioneer/packages/apps/public/locales/en/joy-proposals.json

@@ -1 +1,4 @@
-{}
+{
+  "Current": "Current",
+  "Historical": "Historical"
+}

pioneer/packages/apps/public/locales/en/translation.json

@@ -125,6 +125,7 @@
   "Create and backup account": "",
   "Crypto not detected": "",
   "Cryptography used to create this signature. It is auto-detected on valid signatures.": "",
+  "Current": "",
   "Current account nonce: {{accountNonce}}": "",
   "Current prime member, default voting": "",
   "Current society head, exempt": "",
@@ -184,6 +185,7 @@
   "Genesis Hash refers to initial state of the chain, it cannot be changed once the chain is launched": "",
   "Grandpa": "",
   "Hash data": "",
+  "Historical": "",
   "I'm Online": "",
   "If the recipient account is new, the balance needs to be more than the existential deposit. Likewise if the sending account balance drops below the same value, the account will be removed from the state.": "",
   "If this proposal is passed, the changes will be applied via dispatch and the deposit returned.": "",

pioneer/packages/apps/src/notLive.ts

@@ -0,0 +1 @@
+// Empty entry point for webpack if not in live mode to speed up the build

pioneer/packages/apps/webpack.base.config.js

@@ -27,6 +27,7 @@ function mapChunks (name, regs, inc) {
 function createWebpack (ENV, context) {
   const pkgJson = require(path.join(context, 'package.json'));
   const isProd = ENV === 'production';
+  const isLive = !(process.env.IS_LIVE === 'false' || process.env.IS_LIVE === false);
   const hasPublic = fs.existsSync(path.join(context, 'public'));
   const plugins = hasPublic
     ? [new CopyWebpackPlugin({ patterns: [{ from: 'public' }] })]
@@ -53,7 +54,8 @@ function createWebpack (ENV, context) {
 
   return {
     context,
-    entry: ['@babel/polyfill', './src/index.tsx'],
+    // Make it quicker if we're not in a LIVE mode
+    entry: !isLive ? './src/notLive.ts' : ['@babel/polyfill', './src/index.tsx'],
     mode: ENV,
     module: {
       rules: [

pioneer/packages/apps/webpack.config.js

@@ -11,6 +11,7 @@ const baseConfig = require('./webpack.base.config');
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 
 const ENV = process.env.NODE_ENV || 'development';
+const IS_LIVE = !(process.env.IS_LIVE === false || process.env.IS_LIVE === 'false');
 const context = __dirname;
 const hasPublic = fs.existsSync(path.join(context, 'public'));
 
@@ -21,6 +22,7 @@ module.exports = merge(
     plugins: [
       new HtmlWebpackPlugin({
         IS_PROD: ENV === 'production',
+        IS_LIVE,
         PAGE_TITLE: 'Joystream Network Portal',
         inject: true,
         template: path.join(context, `${hasPublic ? 'public/' : ''}index.html`)

pioneer/packages/joy-media/src/DiscoveryProvider.tsx

@@ -44,7 +44,11 @@ type ProviderStats = {
 function newDiscoveryProvider ({ bootstrapNodes }: BootstrapNodes): DiscoveryProvider {
   const stats = new Map<string, ProviderStats>();
 
-  const resolveAssetEndpoint = async (storageProvider: StorageProviderId, contentId?: string, cancelToken?: CancelToken) => {
+  const resolveAssetEndpoint = async (
+    storageProvider: StorageProviderId,
+    contentId?: string,
+    cancelToken?: CancelToken
+  ) => {
     const providerKey = storageProvider.toString();
 
     let stat = stats.get(providerKey);

pioneer/packages/joy-proposals/src/Proposal/Body.tsx

@@ -33,6 +33,7 @@ type BodyProps = {
   proposerId: number | MemberId;
   isCancellable: boolean;
   cancellationFee: number;
+  historical?: boolean;
 };
 
 function ProposedAddress (props: { accountId?: AccountId }) {
@@ -97,7 +98,7 @@ class ParsedParam {
 }
 
 // The methods for parsing params by Proposal type.
-const paramParsers: { [k in ProposalType]: (params: SpecificProposalDetails<k>) => ParsedParam[]} = {
+const paramParsers: { [k in ProposalType]: (params: SpecificProposalDetails<k>, historical?: boolean) => ParsedParam[]} = {
   Text: (content) => [
     new ParsedParam(
       'Content',
@@ -237,12 +238,14 @@ const paramParsers: { [k in ProposalType]: (params: SpecificProposalDetails<k>)
     new ParsedParam('Working group', (group as WorkingGroup).type),
     new ParsedParam('Mint capacity', formatBalance((capacity as Balance)))
   ],
-  BeginReviewWorkingGroupLeaderApplication: ([id, group]) => [
+  BeginReviewWorkingGroupLeaderApplication: ([id, group], historical) => [
     new ParsedParam('Working group', (group as WorkingGroup).type),
     // TODO: Adjust the link to work with multiple groups after working-groups are normalized!
     new ParsedParam(
       'Opening id',
-      <Link to={`/working-groups/opportunities/storageProviders/${id.toString()}`}>#{id.toString()}</Link>
+      historical
+        ? `#${id.toString()}`
+        : <Link to={`/working-groups/opportunities/storageProviders/${id.toString()}`}>#{id.toString()}</Link>
     )
   ],
   FillWorkingGroupLeaderOpening: ({
@@ -250,46 +253,56 @@ const paramParsers: { [k in ProposalType]: (params: SpecificProposalDetails<k>)
     successful_application_id: succesfulApplicationId,
     reward_policy: rewardPolicy,
     working_group: workingGroup
-  }) => [
+  }, historical) => [
     new ParsedParam('Working group', workingGroup.type),
     // TODO: Adjust the link to work with multiple groups after working-groups are normalized!
     new ParsedParam(
       'Opening id',
-      <Link to={`/working-groups/opportunities/storageProviders/${openingId.toString()}`}>#{openingId.toString()}</Link>),
+      historical
+        ? `#${openingId.toString()}`
+        : <Link to={`/working-groups/opportunities/storageProviders/${openingId.toString()}`}>#{openingId.toString()}</Link>),
     new ParsedParam('Reward policy', rewardPolicy.isSome ? formatReward(rewardPolicy.unwrap(), true) : 'NONE'),
     new ParsedParam(
       'Result',
-      <ApplicationsDetailsByOpening
-        openingId={openingId.toNumber()}
-        acceptedIds={[succesfulApplicationId.toNumber()]}
-        group={workingGroup.type}/>,
+      historical
+        ? `Accepted application ID: ${succesfulApplicationId.toNumber()}`
+        : <ApplicationsDetailsByOpening
+          openingId={openingId.toNumber()}
+          acceptedIds={[succesfulApplicationId.toNumber()]}
+          group={workingGroup.type}/>,
       true
     )
   ],
-  SlashWorkingGroupLeaderStake: ([leadId, amount, group]) => [
+  SlashWorkingGroupLeaderStake: ([leadId, amount, group], historical) => [
     new ParsedParam('Working group', (group as WorkingGroup).type),
     new ParsedParam('Slash amount', formatBalance(amount as Balance)),
     new ParsedParam(
       'Lead',
-      <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
+      historical
+        ? `#${(leadId as WorkerId).toNumber()}`
+        : <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
       true
     )
   ],
-  DecreaseWorkingGroupLeaderStake: ([leadId, amount, group]) => [
+  DecreaseWorkingGroupLeaderStake: ([leadId, amount, group], historical) => [
     new ParsedParam('Working group', (group as WorkingGroup).type),
     new ParsedParam('Decrease amount', formatBalance(amount as Balance)),
     new ParsedParam(
       'Lead',
-      <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
+      historical
+        ? `#${(leadId as WorkerId).toNumber()}`
+        : <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
       true
     )
   ],
-  SetWorkingGroupLeaderReward: ([leadId, amount, group]) => [
+  SetWorkingGroupLeaderReward: ([leadId, amount, group], historical) => [
     new ParsedParam('Working group', (group as WorkingGroup).type),
     new ParsedParam('New reward amount', formatBalance(amount as Balance)),
     new ParsedParam(
       'Lead',
-      <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
+      historical
+        ? `#${(leadId as WorkerId).toNumber()}`
+        : <LeadInfoFromId group={(group as WorkingGroup).type} leadId={(leadId as WorkerId).toNumber()}/>,
       true
     )
   ],
@@ -298,12 +311,19 @@ const paramParsers: { [k in ProposalType]: (params: SpecificProposalDetails<k>)
     rationale,
     worker_id: leadId,
     slash
-  }) => {
+  },
+  historical) => {
     return [
       new ParsedParam('Working group', workingGroup.type),
       new ParsedParam('Rationale', bytesToString(rationale), true),
       new ParsedParam('Slash stake', slash.isTrue ? 'YES' : 'NO'),
-      new ParsedParam('Lead', <LeadInfoFromId group={workingGroup.type} leadId={leadId.toNumber()}/>, true)
+      new ParsedParam(
+        'Lead',
+        historical
+          ? `#${leadId.toNumber()}`
+          : <LeadInfoFromId group={workingGroup.type} leadId={leadId.toNumber()}/>,
+        true
+      )
     ];
   }
 };
@@ -364,14 +384,16 @@ export default function Body ({
   proposalId,
   proposerId,
   isCancellable,
-  cancellationFee
+  cancellationFee,
+  historical
 }: BodyProps) {
   // Assert more generic type (since TypeScript cannot possibly know the value of "type" here yet)
-  const parseParams = paramParsers[type] as (params: SpecificProposalDetails<ProposalType>) => ParsedParam[];
+  const parseParams = paramParsers[type] as (params: SpecificProposalDetails<ProposalType>, historical?: boolean) => ParsedParam[];
   const parsedParams = parseParams(
     type === 'RuntimeUpgrade'
       ? params as RuntimeUpgradeProposalDetails
-      : (params as ProposalDetails).asType(type)
+      : (params as ProposalDetails).asType(type),
+    historical
   );
 
   return (

pioneer/packages/joy-proposals/src/Proposal/ProposalDetails.tsx

@@ -56,7 +56,7 @@ export type ExtendedProposalStatus = {
   executionFailReason: string | null;
 }
 
-export function getExtendedStatus (proposal: ParsedProposal, bestNumber: BlockNumber | undefined): ExtendedProposalStatus {
+export function getExtendedStatus (proposal: ParsedProposal, bestNumber?: BlockNumber): ExtendedProposalStatus {
   const basicStatus: BasicProposalStatus = proposal.status.type;
   let expiresIn: number | null = null;
 
@@ -120,6 +120,7 @@ type ProposalDetailsProps = MyAccountProps & {
   proposalId: ProposalId;
   bestNumber?: BlockNumber;
   council?: Seat[];
+  historical?: boolean;
 };
 
 function ProposalDetails ({
@@ -129,11 +130,12 @@ function ProposalDetails ({
   myMemberId,
   iAmMember,
   council,
-  bestNumber
+  bestNumber,
+  historical
 }: ProposalDetailsProps) {
   const iAmCouncilMember = Boolean(iAmMember && council && council.some((seat) => seat.member.toString() === myAddress));
   const iAmProposer = Boolean(iAmMember && myMemberId !== undefined && proposal.proposerId === myMemberId.toNumber());
-  const extendedStatus = getExtendedStatus(proposal, bestNumber);
+  const extendedStatus = getExtendedStatus(proposal, historical ? undefined : bestNumber);
   const isVotingPeriod = extendedStatus.periodStatus === 'Voting period';
 
   return (
@@ -150,21 +152,23 @@ function ProposalDetails ({
           proposerId={ proposal.proposerId }
           isCancellable={ isVotingPeriod }
           cancellationFee={ proposal.cancellationFee }
+          historical={historical}
         />
         <ProposalDetailsVoting>
-          { iAmCouncilMember && (
+          { (iAmCouncilMember && !historical) && (
             <VotingSection
               proposalId={proposalId}
               memberId={ myMemberId as MemberId }
               isVotingPeriod={ isVotingPeriod }/>
           ) }
-          <Votes proposal={proposal}/>
+          <Votes proposal={proposal} historical={historical}/>
         </ProposalDetailsVoting>
       </ProposalDetailsMain>
       <ProposalDetailsDiscussion>
         <ProposalDiscussion
           proposalId={proposalId}
-          memberId={ iAmMember ? myMemberId : undefined }/>
+          memberId={ iAmMember ? myMemberId : undefined }
+          historical={historical}/>
       </ProposalDetailsDiscussion>
     </div>
   );

pioneer/packages/joy-proposals/src/Proposal/ProposalFromId.tsx

@@ -1,12 +1,37 @@
 import React from 'react';
 import { RouteComponentProps } from 'react-router-dom';
 import ProposalDetails from './ProposalDetails';
-import { useProposalSubscription } from '@polkadot/joy-utils/react/hooks';
+import { usePromise, useProposalSubscription, useTransport } from '@polkadot/joy-utils/react/hooks';
 import PromiseComponent from '@polkadot/joy-utils/react/components/PromiseComponent';
 import { useApi } from '@polkadot/react-hooks';
+import { ParsedProposal } from '@polkadot/joy-utils/types/proposals';
 
 type RouteParams = { id?: string | undefined };
 
+export function HistoricalProposalFromId (props: RouteComponentProps<RouteParams>) {
+  const {
+    match: {
+      params: { id }
+    }
+  } = props;
+  const { api } = useApi();
+
+  const transport = useTransport();
+  const [proposal, error, loading] = usePromise(
+    () => transport.proposals.historicalProposalById(api.createType('ProposalId', id)),
+    null as ParsedProposal | null
+  );
+
+  return (
+    <PromiseComponent
+      error={error}
+      loading={loading}
+      message={'Fetching proposal...'}>
+      <ProposalDetails proposal={ proposal } proposalId={ id } historical/>
+    </PromiseComponent>
+  );
+}
+
 export default function ProposalFromId (props: RouteComponentProps<RouteParams>) {
   const {
     match: {

pioneer/packages/joy-proposals/src/Proposal/ProposalPreview.tsx

@@ -25,15 +25,16 @@ const ProposalDesc = styled.div`
 export type ProposalPreviewProps = {
   proposal: ParsedProposal;
   bestNumber?: BlockNumber;
+  historical?: boolean;
 };
 
-export default function ProposalPreview ({ proposal, bestNumber }: ProposalPreviewProps) {
-  const extendedStatus = getExtendedStatus(proposal, bestNumber);
+export default function ProposalPreview ({ proposal, bestNumber, historical }: ProposalPreviewProps) {
+  const extendedStatus = getExtendedStatus(proposal, historical ? undefined : bestNumber);
 
   return (
     <Card
       fluid
-      href={`#/proposals/${proposal.id.toString()}`}>
+      href={`#/proposals/${historical ? 'historical/' : ''}${proposal.id.toString()}`}>
       <ProposalIdBox>{ `#${proposal.id.toString()}` }</ProposalIdBox>
       <Card.Content>
         <Card.Header>

pioneer/packages/joy-proposals/src/Proposal/ProposalPreviewList.tsx

@@ -13,6 +13,7 @@ import { Dropdown } from '@polkadot/react-components';
 
 type ProposalPreviewListProps = {
   bestNumber?: BlockNumber;
+  historical?: boolean;
 };
 
 const FilterContainer = styled.div`
@@ -22,6 +23,7 @@ const FilterContainer = styled.div`
   margin-bottom: 1.75rem;
 `;
 const StyledDropdown = styled(Dropdown)`
+  margin-left: auto;
   .dropdown {
     width: 200px;
   }
@@ -30,15 +32,17 @@ const PaginationBox = styled.div`
   margin-bottom: 1em;
 `;
 
-function ProposalPreviewList ({ bestNumber }: ProposalPreviewListProps) {
+function ProposalPreviewList ({ bestNumber, historical }: ProposalPreviewListProps) {
   const { pathname } = useLocation();
   const transport = useTransport();
   const [activeFilter, setActiveFilter] = useState<ProposalStatusFilter>('All');
   const [currentPage, setCurrentPage] = useState<number>(1);
   const [proposalsBatch, error, loading] = usePromise<ProposalsBatch | undefined>(
-    () => transport.proposals.proposalsBatch(activeFilter, currentPage),
+    () => historical
+      ? transport.proposals.historicalProposalsBatch(activeFilter, currentPage)
+      : transport.proposals.proposalsBatch(activeFilter, currentPage),
     undefined,
-    [activeFilter, currentPage]
+    [activeFilter, currentPage, historical]
   );
 
   const filterOptions = proposalStatusFilters.map((filter) => ({
@@ -54,10 +58,12 @@ function ProposalPreviewList ({ bestNumber }: ProposalPreviewListProps) {
   return (
     <Container className='Proposal' fluid>
       <FilterContainer>
-        <Button primary as={Link} to={`${pathname}/new`}>
-          <Icon name='add' />
-          New proposal
-        </Button>
+        { !historical && (
+          <Button primary as={Link} to={`${pathname}/new`}>
+            <Icon name='add' />
+            New proposal
+          </Button>
+        ) }
         <StyledDropdown
           label='Proposal state'
           options={filterOptions}
@@ -85,7 +91,7 @@ function ProposalPreviewList ({ bestNumber }: ProposalPreviewListProps) {
             ? (
               <Card.Group>
                 {proposalsBatch.proposals.map((prop: ParsedProposal, idx: number) => (
-                  <ProposalPreview key={`${prop.title}-${idx}`} proposal={prop} bestNumber={bestNumber} />
+                  <ProposalPreview key={`${prop.title}-${idx}`} proposal={prop} bestNumber={bestNumber} historical={historical}/>
                 ))}
               </Card.Group>
             )

pioneer/packages/joy-proposals/src/Proposal/Votes.tsx

@@ -10,14 +10,17 @@ import PromiseComponent from '@polkadot/joy-utils/react/components/PromiseCompon
 
 type VotesProps = {
   proposal: ParsedProposal;
+  historical?: boolean;
 };
 
-export default function Votes ({ proposal: { id, votingResults } }: VotesProps) {
+export default function Votes ({ proposal: { id, votingResults }, historical }: VotesProps) {
   const transport = useTransport();
   const [votes, error, loading] = usePromise<ProposalVotes | null>(
-    () => transport.proposals.votes(id),
+    () => historical
+      ? transport.proposals.historicalVotes(id)
+      : transport.proposals.votes(id),
     null,
-    [votingResults]
+    [votingResults, historical]
   );
 
   return (

pioneer/packages/joy-proposals/src/Proposal/discussion/DiscussionPost.tsx

@@ -45,17 +45,20 @@ type ProposalDiscussionPostProps = {
   post: ParsedPost;
   memberId?: MemberId;
   refreshDiscussion: () => void;
+  historical?: boolean;
 }
 
 export default function DiscussionPost ({
   post,
   memberId,
-  refreshDiscussion
+  refreshDiscussion,
+  historical
 }: ProposalDiscussionPostProps) {
   const { author, authorId, text, createdAt, editsCount } = post;
   const [editing, setEditing] = useState(false);
   const constraints = useTransport().proposals.discussionContraints();
   const canEdit = (
+    !historical &&
     memberId &&
     post.postId &&
     authorId.toNumber() === memberId.toNumber() &&

pioneer/packages/joy-proposals/src/Proposal/discussion/ProposalDiscussion.tsx

@@ -11,16 +11,21 @@ import { MemberId } from '@joystream/types/members';
 type ProposalDiscussionProps = {
   proposalId: ProposalId;
   memberId?: MemberId;
+  historical?: boolean;
 };
 
 export default function ProposalDiscussion ({
   proposalId,
-  memberId
+  memberId,
+  historical
 }: ProposalDiscussionProps) {
   const transport = useTransport();
   const [discussion, error, loading, refreshDiscussion] = usePromise<ParsedDiscussion | null | undefined>(
-    () => transport.proposals.discussion(proposalId),
-    undefined
+    () => historical
+      ? transport.proposals.historicalDiscussion(proposalId)
+      : transport.proposals.discussion(proposalId),
+    undefined,
+    [historical]
   );
   const constraints = transport.proposals.discussionContraints();
 
@@ -36,14 +41,15 @@ export default function ProposalDiscussion ({
                 key={post.postId ? post.postId.toNumber() : `k-${key}`}
                 post={post}
                 memberId={memberId}
-                refreshDiscussion={refreshDiscussion}/>
+                refreshDiscussion={refreshDiscussion}
+                historical={historical}/>
             ))
           )
             : (
               <Header as='h4' style={{ margin: '1rem 0' }}>Nothing has been posted here yet!</Header>
             )
           }
-          { memberId && (
+          { (memberId && !historical) && (
             <DiscussionPostForm
               threadId={discussion.threadId}
               memberId={memberId}

pioneer/packages/joy-proposals/src/index.tsx

@@ -1,8 +1,9 @@
 import React from 'react';
 import { Route, Switch, RouteComponentProps } from 'react-router';
+import Tabs from '@polkadot/react-components/Tabs';
 import { Link } from 'react-router-dom';
 import styled from 'styled-components';
-import { Breadcrumb } from 'semantic-ui-react';
+import { Breadcrumb, Message } from 'semantic-ui-react';
 
 import { I18nProps } from '@polkadot/react-components/types';
 import { ProposalPreviewList, ProposalFromId, ChooseProposalType } from './Proposal';
@@ -27,6 +28,7 @@ import { SignalForm,
   TerminateWorkingGroupLeaderForm } from './forms';
 import { RouteProps as AppMainRouteProps } from '@polkadot/apps-routing/types';
 import style from './style';
+import { HistoricalProposalFromId } from './Proposal/ProposalFromId';
 
 const ProposalsMain = styled.main`${style}`;
 
@@ -42,11 +44,27 @@ const StyledHeader = styled.header`
 `;
 
 function App (props: Props): React.ReactElement<Props> {
-  const { basePath } = props;
+  const { basePath, t } = props;
+  const tabs = [
+    {
+      isRoot: true,
+      name: 'current',
+      text: t('Current')
+    },
+    {
+      name: 'historical',
+      text: t('Historical'),
+      hasParams: true
+    }
+  ];
 
   return (
     <ProposalsMain className='proposal--App'>
       <StyledHeader>
+        <Tabs
+          basePath={basePath}
+          items={tabs}
+        />
         <Breadcrumb>
           <Switch>
             <Route path={`${basePath}/new/:type`} render={(props: RouteComponentProps<{ type?: string }>) => (
@@ -63,11 +81,25 @@ function App (props: Props): React.ReactElement<Props> {
               <Breadcrumb.Divider icon='right angle' />
               <Breadcrumb.Section active>New proposal</Breadcrumb.Section>
             </Route>
+            <Route path={`${basePath}/historical`}>
+              <Breadcrumb.Section active>Historical Proposals</Breadcrumb.Section>
+            </Route>
             <Route>
               <Breadcrumb.Section active>Proposals</Breadcrumb.Section>
             </Route>
           </Switch>
         </Breadcrumb>
+        <Switch>
+          <Route path={`${basePath}/historical`}>
+            <Message warning active>
+              <Message.Header>{"You're in a historical proposals view."}</Message.Header>
+              <Message.Content>
+                The data presented here comes from previous Joystream testnet chain, which
+                means all proposals are read-only and can no longer be interacted with!
+              </Message.Content>
+            </Message>
+          </Route>
+        </Switch>
       </StyledHeader>
       <Switch>
         <Route exact path={`${basePath}/new`} component={ChooseProposalType} />
@@ -92,7 +124,9 @@ function App (props: Props): React.ReactElement<Props> {
         <Route exact path={`${basePath}/new/terminate-working-group-leader-role`} component={TerminateWorkingGroupLeaderForm} />
         <Route exact path={`${basePath}/active`} component={NotDone} />
         <Route exact path={`${basePath}/finalized`} component={NotDone} />
-        <Route exact path={`${basePath}/:id`} component={ProposalFromId} />
+        <Route exact path={`${basePath}/historical`} render={() => <ProposalPreviewList historical={true}/>} />
+        <Route exact path={`${basePath}/historical/:id`} component={HistoricalProposalFromId}/>
+        <Route exact path={`${basePath}/:id`} component={ProposalFromId}/>
         <Route component={ProposalPreviewList} />
       </Switch>
     </ProposalsMain>

pioneer/packages/joy-utils/src/transport/proposals.ts

@@ -8,13 +8,15 @@ import { ParsedProposal,
   DiscussionContraints,
   ProposalStatusFilter,
   ProposalsBatch,
-  ParsedProposalDetails } from '../types/proposals';
+  ParsedProposalDetails,
+  RuntimeUpgradeProposalDetails,
+  HistoricalProposalData } from '../types/proposals';
 import { ParsedMember } from '../types/members';
 
 import BaseTransport from './base';
 
 import { ThreadId, PostId } from '@joystream/types/common';
-import { Proposal, ProposalId, VoteKind, DiscussionThread, DiscussionPost, ProposalDetails } from '@joystream/types/proposals';
+import { Proposal, ProposalId, VoteKind, DiscussionThread, DiscussionPost, ProposalDetails, ProposalStatus } from '@joystream/types/proposals';
 import { MemberId } from '@joystream/types/members';
 import { u32, Bytes, Null } from '@polkadot/types/';
 import { BalanceOf } from '@polkadot/types/interfaces';
@@ -31,6 +33,8 @@ import CouncilTransport from './council';
 import { blake2AsHex } from '@polkadot/util-crypto';
 import { APIQueryCache } from './APIQueryCache';
 
+import HISTORICAL_PROPOSALS from './static/historical-proposals.json';
+
 type ProposalDetailsCacheEntry = {
   type: ProposalType;
   details: ParsedProposalDetails;
@@ -143,21 +147,31 @@ export default class ProposalsTransport extends BaseTransport {
     return result.map(([proposalId]) => proposalId);
   }
 
-  async proposalsBatch (status: ProposalStatusFilter, batchNumber = 1, batchSize = 5): Promise<ProposalsBatch> {
+  private checkProposalStatusFilter (status: ProposalStatus, filter: ProposalStatusFilter) {
+    if (filter === 'All') {
+      return true;
+    }
+
+    if (filter === 'Active' && status.isOfType('Active')) {
+      return true;
+    }
+
+    if (!status.isOfType('Finalized')) {
+      return false;
+    }
+
+    return status.asType('Finalized').proposalStatus.type === filter;
+  }
+
+  async proposalsBatch (statusFilter: ProposalStatusFilter, batchNumber = 1, batchSize = 5): Promise<ProposalsBatch> {
     const ids = (status === 'Active' ? await this.activeProposalsIds() : await this.proposalsIds())
       .sort((id1, id2) => id2.cmp(id1)); // Sort by newest
     let rawProposalsWithIds = (await Promise.all(ids.map((id) => this.rawProposalById(id))))
       .map((proposal, index) => ({ id: ids[index], proposal }));
 
-    if (status !== 'All' && status !== 'Active') {
-      rawProposalsWithIds = rawProposalsWithIds.filter(({ proposal }) => {
-        if (!proposal.status.isOfType('Finalized')) {
-          return false;
-        }
-
-        return proposal.status.asType('Finalized').proposalStatus.type === status;
-      });
-    }
+    rawProposalsWithIds = rawProposalsWithIds.filter(({ proposal }) => (
+      this.checkProposalStatusFilter(proposal.status, statusFilter)
+    ));
 
     const totalBatches = Math.ceil(rawProposalsWithIds.length / batchSize);
 
@@ -283,4 +297,135 @@ export default class ProposalsTransport extends BaseTransport {
       maxPostLength: (this.api.consts.proposalsDiscussion.postLengthLimit as u32).toNumber()
     };
   }
+
+  private replaceHistoricalProposalLinks (text: string) {
+    return text.replace(
+      /testnet\.joystream\.org\/#\/proposals\/([0-9]+)/g,
+      'testnet.joystream.org/#/proposals/historical/$1'
+    );
+  }
+
+  private parseHistoricalProposalDetails (proposal: HistoricalProposalData['proposal']): ParsedProposalDetails {
+    const { type, details } = proposal;
+
+    if (type === 'RuntimeUpgrade') {
+      return details as RuntimeUpgradeProposalDetails;
+    }
+
+    if (type === 'Text') {
+      details[0] = this.replaceHistoricalProposalLinks(details[0] as string);
+    }
+
+    return this.api.createType('ProposalDetails', {
+      [type]: details.length > 1 ? details : details[0]
+    });
+  }
+
+  // Historical proposals methods
+  private parseHistoricalProposal ({ proposal }: HistoricalProposalData): ParsedProposal {
+    return {
+      ...proposal,
+      id: this.api.createType('ProposalId', proposal.id),
+      type: proposal.type as ProposalType,
+      status: this.api.createType('ProposalStatus', proposal.status),
+      createdAt: new Date(proposal.createdAt),
+      parameters: this.api.createType('ProposalParameters', proposal.parameters),
+      votingResults: this.api.createType('VotingResults', proposal.votingResults),
+      details: this.parseHistoricalProposalDetails(proposal),
+      description: this.replaceHistoricalProposalLinks(proposal.description)
+    };
+  }
+
+  historicalProposalById (id: ProposalId): Promise<ParsedProposal> {
+    return new Promise((resolve, reject) => {
+      const proposalData = HISTORICAL_PROPOSALS.find(({ proposal }) => proposal.id === id.toNumber());
+
+      if (!proposalData) {
+        reject(new Error('Historical proposal not found!'));
+      } else {
+        resolve(this.parseHistoricalProposal(proposalData));
+      }
+    });
+  }
+
+  private parseHistoricalProposalDiscussion (proposalData: HistoricalProposalData): ParsedDiscussion {
+    const { discussion } = proposalData;
+
+    return {
+      ...discussion,
+      threadId: this.api.createType('ThreadId', discussion.threadId),
+      posts: discussion.posts.map((post) => ({
+        ...post,
+        postId: this.api.createType('PostId', post.postId),
+        threadId: this.api.createType('ThreadId', post.threadId),
+        createdAt: new Date(post.createdAt),
+        updatedAt: new Date(post.updatedAt),
+        author: this.api.createType('Membership', post.author),
+        authorId: this.api.createType('MemberId', post.authorId),
+        text: this.replaceHistoricalProposalLinks(post.text)
+      }))
+    };
+  }
+
+  historicalProposalsBatch (statusFilter: ProposalStatusFilter, batchNumber = 1, batchSize = 5): Promise<ProposalsBatch> {
+    return new Promise((resolve, reject) => {
+      const filteredProposalsData = HISTORICAL_PROPOSALS
+        .sort((a, b) => b.proposal.id - a.proposal.id)
+        .filter(({ proposal }) => (
+          this.checkProposalStatusFilter(this.api.createType('ProposalStatus', proposal.status), statusFilter)
+        ));
+
+      const totalBatches = Math.ceil(filteredProposalsData.length / batchSize);
+      const proposalsInBatchData = filteredProposalsData.slice((batchNumber - 1) * batchSize, batchNumber * batchSize);
+      const parsedProposals: ParsedProposal[] = proposalsInBatchData
+        .map((proposalData) => this.parseHistoricalProposal(proposalData));
+
+      resolve({
+        batchNumber,
+        batchSize: parsedProposals.length,
+        totalBatches,
+        proposals: parsedProposals
+      });
+    });
+  }
+
+  historicalDiscussion (id: number|ProposalId): Promise<ParsedDiscussion | null> {
+    return new Promise((resolve, reject) => {
+      const proposalData = HISTORICAL_PROPOSALS.find(({ proposal }) => proposal.id.toString() === id.toString());
+
+      if (!proposalData) {
+        reject(new Error('Historical proposal not found!'));
+      } else {
+        resolve(this.parseHistoricalProposalDiscussion(proposalData));
+      }
+    });
+  }
+
+  private parseHistoricalVotes (proposalData: HistoricalProposalData): ProposalVotes {
+    const { votes } = proposalData;
+
+    return {
+      ...votes,
+      votes: votes.votes.map((vote) => ({
+        ...vote,
+        vote: this.api.createType('VoteKind', vote.vote),
+        member: {
+          ...vote.member,
+          memberId: this.api.createType('MemberId', vote.member.memberId)
+        }
+      }))
+    };
+  }
+
+  historicalVotes (proposalId: ProposalId): Promise<ProposalVotes> {
+    return new Promise((resolve, reject) => {
+      const proposalData = HISTORICAL_PROPOSALS.find(({ proposal }) => proposal.id.toString() === proposalId.toString());
+
+      if (!proposalData) {
+        reject(new Error('Historical proposal not found!'));
+      } else {
+        resolve(this.parseHistoricalVotes(proposalData));
+      }
+    });
+  }
 }

pioneer/packages/joy-utils/src/types/proposals.ts

@@ -118,3 +118,46 @@ export type DiscussionContraints = {
   maxPostLength: number;
   maxPostEdits: number;
 }
+
+export type HistoricalParsedPost = {
+  postId: number;
+  threadId: number;
+  text: string;
+  createdAt: string;
+  createdAtBlock: number;
+  updatedAt: string;
+  updatedAtBlock: number;
+  author: ParsedMember;
+  authorId: number;
+  editsCount: number;
+}
+
+export type HistoricalProposalData = {
+  proposal: {
+    id: number,
+    parameters: unknown, // JSON of ProposalParameters
+    proposerId: number,
+    title: string,
+    description: string,
+    createdAt: string,
+    status: unknown, // JSON of ProposalStatus
+    votingResults: unknown, // JSON of VotingResults
+    details: unknown[], // JSON of ParsedProposalDetails
+    type: string,
+    proposer: ParsedMember,
+    createdAtBlock: number,
+    cancellationFee: number
+  },
+  votes: {
+    councilMembersLength: number,
+    votes: {
+      vote: number;
+      member: ParsedMember & { memberId: number },
+    }[]
+  },
+  discussion: {
+    title: string,
+    threadId: number,
+    posts: HistoricalParsedPost[]
+  }
+}

pioneer/packages/page-accounts/src/Accounts/Account.tsx

@@ -14,7 +14,7 @@ import React, { useCallback, useContext, useState, useEffect } from 'react';
 import styled from 'styled-components';
 import { ApiPromise } from '@polkadot/api';
 import { getLedger } from '@polkadot/react-api';
-import { AddressInfo, AddressMini, AddressSmall, Badge, Button, ChainLock, CryptoType, Forget, Icon, IdentityIcon, LinkExternal, Menu, Popup, StatusContext, Tags } from '@polkadot/react-components';
+import { AddressInfo, AddressMini, AddressSmall, Badge, Button, CryptoType, Forget, Icon, IdentityIcon, LinkExternal, Menu, Popup, StatusContext, Tags } from '@polkadot/react-components';
 import { useAccountInfo, useApi, useCall, useToggle } from '@polkadot/react-hooks';
 import { Option } from '@polkadot/types';
 import keyring from '@polkadot/ui-keyring';
@@ -83,7 +83,7 @@ function Account ({ account: { address, meta }, className = '', delegation, filt
   });
   const multiInfos = useMultisigApprovals(address);
   const proxyInfo = useProxies(address);
-  const { flags: { isDevelopment, isExternal, isHardware, isInjected, isMultisig, isProxied }, genesisHash, identity, name: accName, onSetGenesisHash, tags } = useAccountInfo(address);
+  const { flags: { isDevelopment, isExternal, isHardware, isInjected, isMultisig, isProxied }, identity, name: accName, tags } = useAccountInfo(address);
   const [{ democracyUnlockTx }, setUnlockableIds] = useState<DemocracyUnlockable>({ democracyUnlockTx: null, ids: [] });
   const [vestingVestTx, setVestingTx] = useState<SubmittableExtrinsic<'promise'> | null>(null);
   const [isVisible, setIsVisible] = useState(true);
@@ -531,12 +531,13 @@ function Account ({ account: { address, meta }, className = '', delegation, filt
                 {t('Delegate democracy votes')}
               </Menu.Item>
             ])}
-            <ChainLock
+            {/* Joystream specific - disallow "Only this network" to avoid confusion */}
+            {/* <ChainLock
               className='accounts--network-toggle'
               genesisHash={genesisHash}
               isDisabled={api.isDevelopment}
               onChange={onSetGenesisHash}
-            />
+            /> */}
           </Menu>
         </Popup>
       </td>

pioneer/packages/page-accounts/src/Accounts/PasswordInput.tsx

@@ -3,7 +3,8 @@
 // of the Apache-2.0 license. See the LICENSE file for details.
 
 import { Modal, Password } from '@polkadot/react-components';
-import keyring from '@polkadot/ui-keyring';
+// import keyring from '@polkadot/ui-keyring';
+import { isPasswordValid } from '@polkadot/joy-utils/functions/accounts';
 
 import React, { useCallback, useState } from 'react';
 
@@ -17,12 +18,12 @@ type Props = {
 
 export default function PasswordInput ({ onChange, onEnter, password }: Props): React.ReactElement {
   const { t } = useTranslation();
-  const [isPassValid, setPassValid] = useState<boolean>(false);
-  const [{ isPass2Valid, password2 }, setPassword2] = useState({ isPass2Valid: false, password2: '' });
+  const [isPassValid, setPassValid] = useState<boolean>(true);
+  const [{ isPass2Valid, password2 }, setPassword2] = useState({ isPass2Valid: true, password2: '' });
 
   const _onPasswordChange = useCallback(
     (password: string) => {
-      const isPassValid = keyring.isPassValid(password);
+      const isPassValid = isPasswordValid(password);
 
       setPassValid(isPassValid);
 
@@ -35,7 +36,7 @@ export default function PasswordInput ({ onChange, onEnter, password }: Props):
 
   const onPassword2Change = useCallback(
     (password2: string) => {
-      const isPass2Valid = keyring.isPassValid(password2) && (password2 === password);
+      const isPass2Valid = isPasswordValid(password2) && (password2 === password);
 
       setPassword2({ isPass2Valid, password2 });
 

pioneer/packages/page-accounts/src/Accounts/modals/Backup.tsx

@@ -6,6 +6,7 @@ import FileSaver from 'file-saver';
 import React, { useCallback, useState } from 'react';
 import { AddressRow, Button, Modal, Password } from '@polkadot/react-components';
 import keyring from '@polkadot/ui-keyring';
+import { isPasswordValid } from '@polkadot/joy-utils/functions/accounts';
 
 import { useTranslation } from '../../translate';
 
@@ -17,9 +18,9 @@ interface Props {
 function Backup ({ address, onClose }: Props): React.ReactElement<Props> {
   const { t } = useTranslation();
   const [isBusy, setIsBusy] = useState(false);
-  const [{ isPassTouched, password }, setPassword] = useState({ isPassTouched: false, password: '' });
+  const [{ isPassTouched, password }, setPassword] = useState({ isPassTouched: true, password: '' });
   const [backupFailed, setBackupFailed] = useState(false);
-  const isPassValid = !backupFailed && keyring.isPassValid(password);
+  const isPassValid = !backupFailed && isPasswordValid(password);
 
   const _onChangePass = useCallback(
     (password: string): void => {

pioneer/packages/page-accounts/src/Accounts/modals/ChangePass.tsx

@@ -5,6 +5,7 @@
 import React, { useCallback, useState } from 'react';
 import { AddressRow, Button, Modal, Password } from '@polkadot/react-components';
 import keyring from '@polkadot/ui-keyring';
+import { isPasswordValid } from '@polkadot/joy-utils/functions/accounts';
 
 import { useTranslation } from '../../translate';
 
@@ -27,24 +28,24 @@ interface OldPass {
 function ChangePass ({ address, className = '', onClose }: Props): React.ReactElement<Props> {
   const { t } = useTranslation();
   const [isBusy, setIsBusy] = useState(false);
-  const [newPass1, setNewPass1] = useState<NewPass>({ isValid: false, password: '' });
-  const [newPass2, setNewPass2] = useState<NewPass>({ isValid: false, password: '' });
-  const [{ isOldValid, oldPass }, setOldPass] = useState<OldPass>({ isOldValid: false, oldPass: '' });
+  const [newPass1, setNewPass1] = useState<NewPass>({ isValid: true, password: '' });
+  const [newPass2, setNewPass2] = useState<NewPass>({ isValid: true, password: '' });
+  const [{ isOldValid, oldPass }, setOldPass] = useState<OldPass>({ isOldValid: true, oldPass: '' });
 
   const _onChangeNew1 = useCallback(
     (password: string) =>
-      setNewPass1({ isValid: keyring.isPassValid(password), password }),
+      setNewPass1({ isValid: isPasswordValid(password), password }),
     []
   );
 
   const _onChangeNew2 = useCallback(
     (password: string) =>
-      setNewPass2({ isValid: keyring.isPassValid(password) && (newPass1.password === password), password }),
+      setNewPass2({ isValid: isPasswordValid(password) && (newPass1.password === password), password }),
     [newPass1]
   );
 
   const _onChangeOld = useCallback(
-    (oldPass: string) => setOldPass({ isOldValid: keyring.isPassValid(oldPass), oldPass }),
+    (oldPass: string) => setOldPass({ isOldValid: isPasswordValid(oldPass), oldPass }),
     []
   );
 

pioneer/packages/page-accounts/src/Accounts/modals/Create.tsx

@@ -138,12 +138,13 @@ export function downloadAccount ({ json, pair }: CreateResult): void {
   FileSaver.saveAs(blob, `${pair.address}.json`);
 }
 
-function createAccount (suri: string, pairType: KeypairType, { genesisHash, name, tags = [] }: CreateOptions, password: string, success: string): ActionStatus {
+function createAccount (suri: string, pairType: KeypairType, { name, tags = [] }: CreateOptions, password: string, success: string): ActionStatus {
   // we will fill in all the details below
   const status = { action: 'create' } as ActionStatus;
 
   try {
-    const result = keyring.addUri(suri, password, { genesisHash, name, tags }, pairType);
+    // Joystream-specific - ignore genesisHash when creating new accounts
+    const result = keyring.addUri(suri, password, { name, tags }, pairType);
     const { address } = result.pair;
 
     status.account = address;
@@ -170,7 +171,7 @@ function Create ({ className = '', onClose, onStatusChange, seed: propsSeed, typ
   const [isConfirmationOpen, toggleConfirmation] = useToggle();
   const [isBusy, setIsBusy] = useState(false);
   const [{ isNameValid, name }, setName] = useState({ isNameValid: false, name: '' });
-  const [{ isPasswordValid, password }, setPassword] = useState({ isPasswordValid: false, password: '' });
+  const [{ isPasswordValid, password }, setPassword] = useState({ isPasswordValid: true, password: '' });
   const isValid = !!address && !deriveError && isNameValid && isPasswordValid && isSeedValid;
   const seedOpt = useMemo(() => (
     isDevelopment

pioneer/packages/page-accounts/src/Accounts/modals/Derive.tsx

@@ -11,6 +11,7 @@ import { AddressRow, Button, Input, InputAddress, Modal, Password, StatusContext
 import { useApi, useDebounce, useToggle } from '@polkadot/react-hooks';
 import keyring from '@polkadot/ui-keyring';
 import { keyExtractPath } from '@polkadot/util-crypto';
+import { isPasswordValid } from '@polkadot/joy-utils/functions/accounts';
 
 import { useTranslation } from '../../translate';
 import { downloadAccount } from './Create';
@@ -83,9 +84,9 @@ function Derive ({ className = '', from, onClose }: Props): React.ReactElement {
   const [isConfirmationOpen, toggleConfirmation] = useToggle();
   const [{ isLocked, lockedError }, setIsLocked] = useState<LockState>({ isLocked: source.isLocked, lockedError: null });
   const [{ isNameValid, name }, setName] = useState({ isNameValid: false, name: '' });
-  const [{ isPassValid, password }, setPassword] = useState({ isPassValid: false, password: '' });
-  const [{ isPass2Valid, password2 }, setPassword2] = useState({ isPass2Valid: false, password2: '' });
-  const [{ isRootValid, rootPass }, setRootPass] = useState({ isRootValid: false, rootPass: '' });
+  const [{ isPassValid, password }, setPassword] = useState({ isPassValid: true, password: '' });
+  const [{ isPass2Valid, password2 }, setPassword2] = useState({ isPass2Valid: true, password2: '' });
+  const [{ isRootValid, rootPass }, setRootPass] = useState({ isRootValid: true, rootPass: '' });
   const [suri, setSuri] = useState('');
   const debouncedSuri = useDebounce(suri);
   const isValid = !!address && !deriveError && isNameValid && isPassValid && isPass2Valid;
@@ -115,18 +116,18 @@ function Derive ({ className = '', from, onClose }: Props): React.ReactElement {
   );
 
   const _onChangePass = useCallback(
-    (password: string) => setPassword({ isPassValid: keyring.isPassValid(password), password }),
+    (password: string) => setPassword({ isPassValid: isPasswordValid(password), password }),
     []
   );
 
   const _onChangePass2 = useCallback(
-    (password2: string) => setPassword2({ isPass2Valid: keyring.isPassValid(password2) && (password2 === password), password2 }),
+    (password2: string) => setPassword2({ isPass2Valid: isPasswordValid(password2) && (password2 === password), password2 }),
     [password]
   );
 
   const _onChangeRootPass = useCallback(
     (rootPass: string): void => {
-      setRootPass({ isRootValid: !!rootPass, rootPass });
+      setRootPass({ isRootValid: isPasswordValid(rootPass), rootPass });
       setIsLocked(({ isLocked }) => ({ isLocked, lockedError: null }));
     },
     []

pioneer/packages/page-accounts/src/Accounts/modals/Import.tsx

@@ -10,6 +10,7 @@ import React, { useCallback, useState } from 'react';
 import { AddressRow, Button, InputAddress, InputFile, Modal, Password } from '@polkadot/react-components';
 import { isObject, u8aToString } from '@polkadot/util';
 import keyring from '@polkadot/ui-keyring';
+import { isPasswordValid } from '@polkadot/joy-utils/functions/accounts';
 
 import { useTranslation } from '../../translate';
 
@@ -55,7 +56,7 @@ function Import ({ className = '', onClose, onStatusChange }: Props): React.Reac
   const { t } = useTranslation();
   const [isBusy, setIsBusy] = useState(false);
   const [{ address, isFileValid, json }, setFile] = useState<FileState>({ address: null, isFileValid: false, json: null });
-  const [{ isPassValid, password }, setPass] = useState<PassState>({ isPassValid: false, password: '' });
+  const [{ isPassValid, password }, setPass] = useState<PassState>({ isPassValid: true, password: '' });
 
   const _onChangeFile = useCallback(
     (file: Uint8Array) => setFile(parseFile(file)),
@@ -63,7 +64,7 @@ function Import ({ className = '', onClose, onStatusChange }: Props): React.Reac
   );
 
   const _onChangePass = useCallback(
-    (password: string) => setPass({ isPassValid: keyring.isPassValid(password), password }),
+    (password: string) => setPass({ isPassValid: isPasswordValid(password), password }),
     []
   );
 

pioneer/packages/react-api/src/Api.tsx

@@ -131,7 +131,7 @@ async function loadOnReady (api: ApiPromise, store: KeyringStore | undefined, ty
 
   // finally load the keyring
   keyring.loadAll({
-    genesisHash: api.genesisHash,
+    // genesisHash: api.genesisHash, Joystream-specific - Don't care about genesis hash when loading accounts
     isDevelopment,
     ss58Format,
     store,

pioneer/packages/react-components/src/InputNumber.tsx

@@ -120,12 +120,12 @@ function inputToBn (input: string, si: SiDef | null, bitLength: BitLength, isZer
     }
 
     const div = new BN(input.replace(/\.\d*$/, ''));
-    const modString = input.replace(/^\d+\./, '');
-    const mod = new BN(modString);
+    // const modString = input.replace(/^\d+\./, '');
+    // const mod = new BN(modString);
 
     result = div
-      .mul(BN_TEN.pow(siPower))
-      .add(mod.mul(BN_TEN.pow(new BN(basePower + siUnitPower - modString.length))));
+      .mul(BN_TEN.pow(siPower));
+    // .add(mod.mul(BN_TEN.pow(new BN(basePower + siUnitPower - modString.length))));
   } else {
     result = new BN(input.replace(/[^\d]/g, ''))
       .mul(BN_TEN.pow(siPower));

pioneer/scripts/dev-build-ts.js

@@ -12,6 +12,7 @@ const cpx = require('cpx');
 const fs = require('fs');
 const mkdirp = require('mkdirp');
 const path = require('path');
+const IS_LIVE = !(process.env.IS_LIVE === false || process.env.IS_LIVE === 'false');
 
 const CPX = ['css', 'gif', 'hbs', 'jpg', 'js', 'png', 'svg', 'd.ts']
   .map((ext) => `src/**/*.${ext}`)
@@ -55,7 +56,8 @@ async function buildJs (dir) {
 
     if (fs.existsSync(path.join(process.cwd(), 'public'))) {
       buildWebpack(dir);
-    } else {
+      // Skip building anything else if we're not going for a LIVE build
+    } else if (IS_LIVE) {
       await buildBabel(dir);
     }
 
@@ -66,6 +68,8 @@ async function buildJs (dir) {
 async function main () {
   execSync('yarn polkadot-dev-clean-build');
 
+  console.log('IS_LIVE:', process.env.IS_LIVE);
+
   // By default the entry point is pioneer/, so here we move to pioneer/packages
   process.chdir('packages');
 
@@ -74,7 +78,9 @@ async function main () {
   // This caused the build folder to end up in the root directory of the monorepo (instead of "pioneer/build")
   //
   // execSync('yarn polkadot-exec-tsc --emitDeclarationOnly --outdir ../build');
-  execSync('yarn tsc --emitDeclarationOnly --outdir ./build');
+  if (IS_LIVE) {
+    execSync('yarn tsc --emitDeclarationOnly --outdir ./build');
+  }
 
   const dirs = fs
     .readdirSync('.')

runtime-modules/membership/src/lib.rs

@@ -401,6 +401,7 @@ decl_module! {
                 });
 
                 membership.root_account = new_root_account.clone();
+                <MembershipById<T>>::insert(member_id, membership);
                 Self::deposit_event(RawEvent::MemberSetRootAccount(member_id, new_root_account));
             }
         }

runtime-modules/membership/src/tests.rs

@@ -332,7 +332,7 @@ fn set_root_account() {
 
             let membership = Members::membership(member_id);
 
-            assert_eq!(ALICE_ACCOUNT_ID, membership.root_account);
+            assert_eq!(ALICE_NEW_ROOT_ACCOUNT, membership.root_account);
 
             assert!(<crate::MemberIdsByRootAccountId<Test>>::get(&ALICE_ACCOUNT_ID).is_empty());
         });

runtime/CHANGELOG.md

@@ -1,11 +1,14 @@
-### Version 6.21.0 - Constantinople runtime upgrade B (Nicaea) - July 29 2020
+### Version 7.4.0 - Alexandria - new chain - September 21 2020
+- Update to substrate v2.0.0-rc4
+
+### Version 6.21.0 - (Constantinople) runtime upgrade B (Nicaea) - July 29 2020
 
 - Introduction of general Working Group runtime module
 - Adds a new instance of the working group module - the Storage Working Group which
   replaces the old actors module for managing the Storge Provider enrollment process
 - New governance proposals to support new working groups
 
-### Version 6.15.0 - Constantinople runtime upgrade A - June 2020
+### Version 6.15.0 - (Constantinople) runtime upgrade A - June 2020
 
 - Updated runtime to sort out type name clashes between the proposal discussion module
   and forum module, in preparing to roll out proposal discussion system in pioneer.

runtime/Cargo.toml

@@ -4,7 +4,7 @@ edition = '2018'
 name = 'joystream-node-runtime'
 # Follow convention: https://github.com/Joystream/substrate-runtime-joystream/issues/1
 # {Authoring}.{Spec}.{Impl} of the RuntimeVersion
-version = '7.3.0'
+version = '7.5.1'
 
 [dependencies]
 # Third-party dependencies

runtime/src/lib.rs

@@ -70,8 +70,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
     spec_name: create_runtime_str!("joystream-node"),
     impl_name: create_runtime_str!("joystream-node"),
     authoring_version: 7,
-    spec_version: 3,
-    impl_version: 0,
+    spec_version: 5,
+    impl_version: 1,
     apis: crate::runtime_api::EXPORTED_RUNTIME_API_VERSIONS,
     transaction_version: 1,
 };

storage-node/packages/colossus/bin/cli.js

@@ -18,7 +18,7 @@ const debug = require('debug')('joystream:colossus')
 const PROJECT_ROOT = path.resolve(__dirname, '..')
 
 // Number of milliseconds to wait between synchronization runs.
-const SYNC_PERIOD_MS = 300000 // 5min
+const SYNC_PERIOD_MS = 120000 // 2min
 
 // Parse CLI
 const FLAG_DEFINITIONS = {

storage-node/packages/colossus/lib/middleware/ipfs_proxy.js

@@ -0,0 +1,85 @@
+const { createProxyMiddleware } = require('http-proxy-middleware')
+const debug = require('debug')('joystream:ipfs-proxy')
+const mime = require('mime-types')
+
+/* 
+For this proxying to work correctly, ensure IPFS HTTP Gateway is configured as a path gateway:
+This can be done manually with the following command:
+
+  $ ipfs config --json Gateway.PublicGateways '{"localhost": null }' 
+  
+The implicit default config is below which is not what we want!
+
+  $ ipfs config --json Gateway.PublicGateways '{
+    "localhost": {
+        "Paths": ["/ipfs", "/ipns"],
+        "UseSubdomains": true
+      }
+    }'
+
+https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway
+*/
+
+const pathFilter = function (path, req) {
+  // we get the full path here so it needs to match the path where
+  // it is used by the openapi initializer
+  return path.match('^/asset/v0') && (req.method === 'GET' || req.method === 'HEAD')
+}
+
+const createPathRewriter = (resolve) => {
+  return async (_path, req) => {
+    // we expect the handler was used in openapi/express path with an id in the path:
+    // "/asset/v0/:id"
+    // TODO: catch and deal with hash == undefined if content id not found
+    const contentId = req.params.id
+    const hash = await resolve(contentId)
+    return `/ipfs/${hash}`
+  }
+}
+
+const createResolver = (storage) => {
+  return async (id) => await storage.resolveContentIdWithTimeout(5000, id)
+}
+
+const createProxy = (storage) => {
+  const pathRewrite = createPathRewriter(createResolver(storage))
+
+  return createProxyMiddleware(pathFilter, {
+    // Default path to local IPFS HTTP GATEWAY
+    target: 'http://localhost:8080/',
+    pathRewrite,
+    onProxyRes: function (proxRes, req, res) {
+      /*
+        Make sure the reverse proxy used infront of colosss (nginx/caddy) Does not duplicate
+        these headers to prevent some browsers getting confused especially
+        with duplicate access-control-allow-origin headers!
+        'accept-ranges': 'bytes',
+        'access-control-allow-headers': 'Content-Type, Range, User-Agent, X-Requested-With',
+        'access-control-allow-methods': 'GET',
+        'access-control-allow-origin': '*',
+        'access-control-expose-headers': 'Content-Range, X-Chunked-Output, X-Stream-Output',
+      */
+
+      if (proxRes.statusCode === 301) {
+        // capture redirect when IPFS HTTP Gateway is configured with 'UseDomains':true
+        // and treat it as an error.
+        console.error('IPFS HTTP Gateway is configured for "UseSubdomains". Killing stream')
+        res.status(500).end()
+        proxRes.destroy()
+      } else {
+        // Handle downloading as attachment /asset/v0/:id?download
+        if (req.query.download) {
+          const contentId = req.params.id
+          const contentType = proxRes.headers['content-type']
+          const ext = mime.extension(contentType) || 'bin'
+          const fileName = `${contentId}.${ext}`
+          proxRes.headers['Content-Disposition'] = `attachment; filename=${fileName}`
+        }
+      }
+    },
+  })
+}
+
+module.exports = {
+  createProxy,
+}

storage-node/packages/colossus/lib/sync.js

@@ -19,83 +19,117 @@
 'use strict'
 
 const debug = require('debug')('joystream:sync')
+const _ = require('lodash')
+const { ContentId } = require('@joystream/types/media')
+// The number of concurrent sync sessions allowed. Must be greater than zero.
+const MAX_CONCURRENT_SYNC_ITEMS = 20
+
+async function syncContent({ api, storage, contentBeingSynced, contentCompleteSynced }) {
+  const knownEncodedContentIds = (await api.assets.getKnownContentIds()).map((id) => id.encode())
+
+  // Select ids which we have not yet fully synced
+  const needsSync = knownEncodedContentIds
+    .filter((id) => !contentCompleteSynced.has(id))
+    .filter((id) => !contentBeingSynced.has(id))
+
+  // Since we are limiting concurrent content ids being synced, to ensure
+  // better distribution of content across storage nodes during a potentially long
+  // sync process we don't want all nodes to replicate items in the same order, so
+  // we simply shuffle.
+  const candidatesForSync = _.shuffle(needsSync)
+
+  // TODO: get the data object
+  // make sure the data object was Accepted by the liaison,
+  // don't just blindly attempt to fetch them
+  while (contentBeingSynced.size < MAX_CONCURRENT_SYNC_ITEMS && candidatesForSync.length) {
+    const id = candidatesForSync.shift()
 
-async function syncCallback(api, storage) {
-  // The first step is to gather all data objects from chain.
-  // TODO: in future, limit to a configured tranche
-  // FIXME this isn't actually on chain yet, so we'll fake it.
-  const knownContentIds = (await api.assets.getKnownContentIds()) || []
+    try {
+      contentBeingSynced.set(id)
+      const contentId = ContentId.decode(api.api.registry, id)
+      await storage.synchronize(contentId, (err, status) => {
+        if (err) {
+          contentBeingSynced.delete(id)
+          debug(`Error Syncing ${err}`)
+        } else if (status.synced) {
+          contentBeingSynced.delete(id)
+          contentCompleteSynced.set(id)
+        }
+      })
+    } catch (err) {
+      // Most likely failed to resolve the content id
+      debug(`Failed calling synchronize ${err}`)
+      contentBeingSynced.delete(id)
+    }
+  }
+}
 
+async function createNewRelationships({ api, contentCompleteSynced }) {
   const roleAddress = api.identities.key.address
   const providerId = api.storageProviderId
 
-  // Iterate over all sync objects, and ensure they're synced.
-  const allChecks = knownContentIds.map(async (contentId) => {
-    // eslint-disable-next-line prefer-const
-    let { relationship, relationshipId } = await api.assets.getStorageRelationshipAndId(providerId, contentId)
-
-    // get the data object
-    // make sure the data object was Accepted by the liaison,
-    // don't just blindly attempt to fetch them
-
-    let fileLocal
-    try {
-      // check if we have content or not
-      const stats = await storage.stat(contentId)
-      fileLocal = stats.local
-    } catch (err) {
-      // on error stating or timeout
-      debug(err.message)
-      // we don't have content if we can't stat it
-      fileLocal = false
-    }
+  // Create new relationships for synced content if required and
+  // compose list of relationship ids to be set to ready.
+  return (
+    await Promise.all(
+      [...contentCompleteSynced.keys()].map(async (id) => {
+        const contentId = ContentId.decode(api.api.registry, id)
+        const { relationship, relationshipId } = await api.assets.getStorageRelationshipAndId(providerId, contentId)
+
+        if (relationship) {
+          // maybe prior transaction to set ready failed for some reason..
+          if (!relationship.ready) {
+            return relationshipId
+          }
+        } else {
+          // create relationship
+          debug(`Creating new storage relationship for ${id}`)
+          try {
+            return await api.assets.createStorageRelationship(roleAddress, providerId, contentId)
+          } catch (err) {
+            debug(`Error creating new storage relationship ${id}: ${err.stack}`)
+          }
+        }
+
+        return null
+      })
+    )
+  ).filter((id) => id !== null)
+}
 
-    if (!fileLocal) {
-      try {
-        await storage.synchronize(contentId)
-      } catch (err) {
-        // duplicate logging
-        // debug(err.message)
-        return
-      }
-      // why are we returning, if we synced the file
-      return
-    }
+async function setRelationshipsReady({ api, relationshipIds }) {
+  const roleAddress = api.identities.key.address
+  const providerId = api.storageProviderId
 
-    if (!relationship) {
-      // create relationship
-      debug(`Creating new storage relationship for ${contentId.encode()}`)
-      try {
-        relationshipId = await api.assets.createStorageRelationship(roleAddress, providerId, contentId)
-        await api.assets.toggleStorageRelationshipReady(roleAddress, providerId, relationshipId, true)
-      } catch (err) {
-        debug(`Error creating new storage relationship ${contentId.encode()}: ${err.stack}`)
-      }
-    } else if (!relationship.ready) {
-      debug(`Updating storage relationship to ready for ${contentId.encode()}`)
-      // update to ready. (Why would there be a relationship set to ready: false?)
+  return Promise.all(
+    relationshipIds.map(async (relationshipId) => {
       try {
         await api.assets.toggleStorageRelationshipReady(roleAddress, providerId, relationshipId, true)
       } catch (err) {
-        debug(`Error setting relationship ready ${contentId.encode()}: ${err.stack}`)
+        debug('Error setting relationship ready')
       }
-    } else {
-      // we already have content and a ready relationship set. No need to do anything
-      // debug(`content already stored locally ${contentId.encode()}`);
-    }
-  })
-
-  return Promise.all(allChecks)
+    })
+  )
 }
 
-async function syncPeriodic(api, flags, storage) {
+async function syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced }) {
+  const retry = () => {
+    setTimeout(syncPeriodic, flags.syncPeriod, {
+      api,
+      flags,
+      storage,
+      contentBeingSynced,
+      contentCompleteSynced,
+    })
+  }
+
   try {
-    debug('Starting sync run...')
+    debug('Sync run started.')
 
     const chainIsSyncing = await api.chainIsSyncing()
     if (chainIsSyncing) {
       debug('Chain is syncing. Postponing sync run.')
-      return setTimeout(syncPeriodic, flags.syncPeriod, api, flags, storage)
+      return retry()
     }
 
     const recommendedBalance = await api.providerHasMinimumBalance(300)
@@ -106,20 +140,29 @@ async function syncPeriodic(api, flags, storage) {
     const sufficientBalance = await api.providerHasMinimumBalance(100)
     if (!sufficientBalance) {
       debug('Provider role account does not have sufficient balance. Postponing sync run!')
-      return setTimeout(syncPeriodic, flags.syncPeriod, api, flags, storage)
+      return retry()
     }
 
-    await syncCallback(api, storage)
-    debug('sync run complete')
+    await syncContent({ api, storage, contentBeingSynced, contentCompleteSynced })
+    const relationshipIds = await createNewRelationships({ api, contentCompleteSynced })
+    await setRelationshipsReady({ api, relationshipIds })
+
+    debug(`Sync run completed, set ${relationshipIds.length} new relationships to ready`)
   } catch (err) {
-    debug(`Error in syncPeriodic ${err.stack}`)
+    debug(`Error in sync run ${err.stack}`)
   }
+
   // always try again
-  setTimeout(syncPeriodic, flags.syncPeriod, api, flags, storage)
+  retry()
 }
 
 function startSyncing(api, flags, storage) {
-  syncPeriodic(api, flags, storage)
+  // ids of content currently being synced
+  const contentBeingSynced = new Map()
+  // ids of content that completed sync and may require creating a new relationship
+  const contentCompleteSynced = new Map()
+
+  syncPeriodic({ api, flags, storage, contentBeingSynced, contentCompleteSynced })
 }
 
 module.exports = {

storage-node/packages/colossus/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/colossus",
   "private": true,
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Colossus - Joystream Storage Node",
   "author": "Joystream",
   "homepage": "https://github.com/Joystream/joystream",
@@ -50,17 +50,19 @@
     "temp": "^0.9.0"
   },
   "dependencies": {
-    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-node-backend": "^0.1.0",
+    "@joystream/storage-runtime-api": "^0.1.0",
     "@joystream/storage-utils": "^0.1.0",
     "body-parser": "^1.19.0",
     "chalk": "^2.4.2",
     "cors": "^2.8.5",
     "express-openapi": "^4.6.1",
     "figlet": "^1.2.1",
+    "http-proxy-middleware": "^1.0.5",
     "js-yaml": "^3.13.1",
     "lodash": "^4.17.11",
     "meow": "^7.0.1",
+    "mime-types": "^2.1.27",
     "multer": "^1.4.1",
     "si-prefix": "^0.2.0"
   }

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -18,12 +18,9 @@
 
 'use strict'
 
-const path = require('path')
-
 const debug = require('debug')('joystream:colossus:api:asset')
-
-const utilRanges = require('@joystream/storage-utils/ranges')
 const filter = require('@joystream/storage-node-backend/filter')
+const ipfsProxy = require('../../../lib/middleware/ipfs_proxy')
 
 function errorHandler(response, err, code) {
   debug(err)
@@ -31,6 +28,9 @@ function errorHandler(response, err, code) {
 }
 
 module.exports = function (storage, runtime) {
+  // Creat the IPFS HTTP Gateway proxy middleware
+  const proxy = ipfsProxy.createProxy(storage)
+
   const doc = {
     // parameters for all operations in this path
     parameters: [
@@ -45,34 +45,6 @@ module.exports = function (storage, runtime) {
       },
     ],
 
-    // Head: report that ranges are OK
-    async head(req, res) {
-      const id = req.params.id
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-        const type = stream.fileInfo.mimeType
-
-        // Close the stream; we don't need to fetch the file (if we haven't
-        // already). Then return result.
-        stream.destroy()
-
-        res.status(200)
-        res.contentType(type)
-        res.header('Content-Disposition', 'inline')
-        res.header('Content-Transfer-Encoding', 'binary')
-        res.header('Accept-Ranges', 'bytes')
-        if (size > 0) {
-          res.header('Content-Length', size)
-        }
-        res.send()
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
-    },
-
     // Put for uploads
     async put(req, res) {
       const id = req.params.id // content id
@@ -184,61 +156,21 @@ module.exports = function (storage, runtime) {
       }
     },
 
-    // Get content
     async get(req, res) {
-      const id = req.params.id
-      const download = req.query.download
-
-      // Parse range header
-      let ranges
-      if (!download) {
-        try {
-          const rangeHeader = req.headers.range
-          ranges = utilRanges.parse(rangeHeader)
-        } catch (err) {
-          // Do nothing; it's ok to ignore malformed ranges and respond with the
-          // full content according to https://www.rfc-editor.org/rfc/rfc7233.txt
-        }
-        if (ranges && ranges.unit !== 'bytes') {
-          // Ignore ranges that are not byte units.
-          ranges = undefined
-        }
-      }
-      debug('Requested range(s) is/are', ranges)
-
-      // Open file
-      try {
-        const size = await storage.size(id)
-        const stream = await storage.open(id, 'r')
-
-        // Add a file extension to download requests if necessary. If the file
-        // already contains an extension, don't add one.
-        let sendName = id
-        const type = stream.fileInfo.mimeType
-        if (download) {
-          let ext = path.extname(sendName)
-          if (!ext) {
-            ext = stream.fileInfo.ext
-            if (ext) {
-              sendName = `${sendName}.${ext}`
-            }
-          }
-        }
+      proxy(req, res)
+    },
 
-        const opts = {
-          name: sendName,
-          type,
-          size,
-          ranges,
-          download,
-        }
-        utilRanges.send(res, stream, opts)
-      } catch (err) {
-        errorHandler(res, err, err.code)
-      }
+    async head(req, res) {
+      proxy(req, res)
     },
   }
 
+  // doc.get = proxy
+  // doc.head = proxy
+  // Note: Adding the middleware this way is causing problems!
+  // We are loosing some information from the request, specifically req.query.download parameters for some reason.
+  // Does it have to do with how/when the apiDoc is being processed? binding issue?
+
   // OpenAPI specs
   doc.get.apiDoc = {
     description: 'Download an asset.',

storage-node/packages/colossus/paths/discover/v0/{id}.js

@@ -12,7 +12,7 @@ module.exports = function (runtime) {
         name: 'id',
         in: 'path',
         required: true,
-        description: 'Actor accouuntId',
+        description: 'Storage Provider Id',
         schema: {
           type: 'string', // integer ?
         },

storage-node/packages/runtime-api/index.js

@@ -61,9 +61,6 @@ class RuntimeApi {
 
     this.asyncLock = new AsyncLock()
 
-    // Keep track locally of account nonces.
-    this.nonces = {}
-
     // The storage provider id to use
     this.storageProviderId = parseInt(options.storageProviderId) // u64 instead ?
 
@@ -148,28 +145,6 @@ class RuntimeApi {
     })
   }
 
-  // Get cached nonce and use unless system nonce is greater, to avoid stale nonce if
-  // there was a long gap in time between calls to signAndSend during which an external app
-  // submitted a transaction.
-  async selectBestNonce(accountId) {
-    const cachedNonce = this.nonces[accountId]
-    // In future use this rpc method to take the pending tx pool into account when fetching the nonce
-    // const nonce = await this.api.rpc.system.accountNextIndex(accountId)
-    const { nonce } = await this.api.query.system.account(accountId)
-
-    const systemNonce = nonce
-
-    const bestNonce = cachedNonce && cachedNonce.gte(systemNonce) ? cachedNonce : systemNonce
-
-    this.nonces[accountId] = bestNonce
-
-    return bestNonce.toNumber()
-  }
-
-  incrementAndSaveNonce(accountId) {
-    this.nonces[accountId] = this.nonces[accountId].addn(1)
-  }
-
   /*
    * signAndSend() with nonce tracking, to enable concurrent sending of transacctions
    * so that they can be included in the same block. Allows you to use the accountId instead
@@ -213,7 +188,7 @@ class RuntimeApi {
 
     // synchronize access to nonce
     await this.executeWithAccountLock(accountId, async () => {
-      const nonce = await this.selectBestNonce(accountId)
+      const nonce = await this.api.rpc.system.accountNextIndex(accountId)
       const signed = tx.sign(fromKey, { nonce })
       const txhash = signed.hash
 
@@ -237,9 +212,6 @@ class RuntimeApi {
         } else {
           debugTx(`Submitted: ${serialized}`)
         }
-
-        // transaction submitted successfully, increment and save nonce.
-        this.incrementAndSaveNonce(accountId)
       } catch (err) {
         const errstr = err.toString()
         debugTx(`Rejected: ${errstr} txhash: ${txhash} nonce: ${nonce}`)

storage-node/packages/storage/storage.js

@@ -217,13 +217,18 @@ class Storage {
 
     this.ipfs = ipfsClient(this.options.ipfs.connect_options)
 
-    this.pins = {}
+    this.pinned = {}
+    this.pinning = {}
 
     this.ipfs.id((err, identity) => {
       if (err) {
         debug(`Warning IPFS daemon not running: ${err.message}`)
       } else {
         debug(`IPFS node is up with identity: ${identity.id}`)
+        // TODO: wait for IPFS daemon to be online for this to be effective..?
+        // set the IPFS HTTP Gateway config we desire.. operator might need
+        // to restart their daemon if the config was changed.
+        this.ipfs.config.set('Gateway.PublicGateways', { 'localhost': null })
       }
     })
   }
@@ -363,27 +368,38 @@ class Storage {
   /*
    * Synchronize the given content ID
    */
-  async synchronize(contentId) {
+  async synchronize(contentId, callback) {
     const resolved = await this.resolveContentIdWithTimeout(this._timeout, contentId)
 
-    // validate resolved id is proper ipfs_cid, not null or empty string
+    // TODO: validate resolved id is proper ipfs_cid, not null or empty string
 
-    if (this.pins[resolved]) {
-      return
-    }
+    if (!this.pinning[resolved] && !this.pinned[resolved]) {
+      debug(`Pinning hash: ${resolved} content-id: ${contentId}`)
+      this.pinning[resolved] = true
 
-    debug(`Pinning ${resolved}`)
+      // Callback passed to add() will be called on error or when the entire file
+      // is retrieved. So on success we consider the content synced.
+      this.ipfs.pin.add(resolved, { quiet: true, pin: true }, (err) => {
+        delete this.pinning[resolved]
+        if (err) {
+          debug(`Error Pinning: ${resolved}`)
+          callback && callback(err)
+        } else {
+          debug(`Pinned ${resolved}`)
+          this.pinned[resolved] = true
+          callback && callback(null, this.syncStatus(resolved))
+        }
+      })
+    } else {
+      callback && callback(null, this.syncStatus(resolved))
+    }
+  }
 
-    // This call blocks until file is retrieved..
-    this.ipfs.pin.add(resolved, { quiet: true, pin: true }, (err) => {
-      if (err) {
-        debug(`Error Pinning: ${resolved}`)
-        delete this.pins[resolved]
-      } else {
-        debug(`Pinned ${resolved}`)
-        // why aren't we doing this.pins[resolved] = true
-      }
-    })
+  syncStatus(ipfsHash) {
+    return {
+      syncing: this.pinning[ipfsHash] === true,
+      synced: this.pinned[ipfsHash] === true,
+    }
   }
 }
 

storage-node/packages/util/ranges.js

@@ -21,18 +21,20 @@
 const uuid = require('uuid')
 const streamBuf = require('stream-buffers')
 
-const debug = require('debug')('joystream:util:ranges')
-
 /*
  * Range parsing
  */
 
+// Increase performance by "pre-computing" these regex expressions
+const PARSE_RANGE_REGEX = /^(\d+-\d+|\d+-|-\d+|\*)$/u
+const PARSE_RANGE_HEADERS_REGEX = /^(([^\s]+)=)?((?:(?:\d+-\d+|-\d+|\d+-),?)+)$/u
+
 /*
  * Parse a range string, e.g. '0-100' or '-100' or '0-'. Return the values
  * in an array of int or undefined (if not provided).
  */
 function parseRange(range) {
-  const matches = range.match(/^(\d+-\d+|\d+-|-\d+|\*)$/u)
+  const matches = range.match(PARSE_RANGE_REGEX)
   if (!matches) {
     throw new Error(`Not a valid range: ${range}`)
   }
@@ -56,8 +58,7 @@ function parseRange(range) {
  */
 function parse(rangeStr) {
   const res = {}
-  debug('Parse range header value:', rangeStr)
-  const matches = rangeStr.match(/^(([^\s]+)=)?((?:(?:\d+-\d+|-\d+|\d+-),?)+)$/u)
+  const matches = rangeStr.match(PARSE_RANGE_HEADERS_REGEX)
   if (!matches) {
     throw new Error(`Not a valid range header: ${rangeStr}`)
   }
@@ -74,15 +75,12 @@ function parse(rangeStr) {
 
   // Merge ranges into result.
   ranges.forEach((newRange) => {
-    debug('Found range:', newRange)
-
     let isMerged = false
     for (const i in res.ranges) {
       const oldRange = res.ranges[i]
 
       // Skip if the new range is fully separate from the old range.
       if (oldRange[1] + 1 < newRange[0] || newRange[1] + 1 < oldRange[0]) {
-        debug('Range does not overlap with', oldRange)
         continue
       }
 
@@ -92,11 +90,9 @@ function parse(rangeStr) {
       const merged = [Math.min(oldRange[0], newRange[0]), Math.max(oldRange[1], newRange[1])]
       res.ranges[i] = merged
       isMerged = true
-      debug('Merged', newRange, 'into', oldRange, 'as', merged)
     }
 
     if (!isMerged) {
-      debug('Non-overlapping range!')
       res.ranges.push(newRange)
     }
   })
@@ -110,7 +106,6 @@ function parse(rangeStr) {
     return first[0] < second[0] ? -1 : 1
   })
 
-  debug('Result of parse is', res)
   return res
 }
 
@@ -159,11 +154,6 @@ class RangeSender {
     this.handlers = {}
     this.opened = false
 
-    debug('RangeSender:', this)
-    if (opts.ranges) {
-      debug('Parsed ranges:', opts.ranges.ranges)
-    }
-
     // Parameters
     this.response = response
     this.stream = stream
@@ -173,7 +163,6 @@ class RangeSender {
 
   onError(err) {
     // Assume hiding the actual error is best, and default to 404.
-    debug('Error:', err)
     if (!this.response.headersSent) {
       this.response.status(err.code || 404).send({
         message: err.message || `File not found: ${this.name}`,
@@ -185,7 +174,6 @@ class RangeSender {
   }
 
   onEnd() {
-    debug('End of stream.')
     this.response.end()
     if (this.endCallback) {
       this.endCallback()
@@ -195,7 +183,6 @@ class RangeSender {
   // **** No ranges
   onOpenNoRange() {
     // File got opened, so we can set headers/status
-    debug('Open succeeded:', this.name, this.type)
     this.opened = true
 
     this.response.status(200)
@@ -228,7 +215,6 @@ class RangeSender {
     // Next range
     this.rangeIndex += 1
     if (this.rangeIndex >= this.ranges.ranges.length) {
-      debug('Cannot advance range index; we are done.')
       return undefined
     }
 
@@ -276,7 +262,6 @@ class RangeSender {
 
   nextRange() {
     if (this.ranges.ranges.length === 1) {
-      debug('Cannot start new range; only one requested.')
       this.stream.off('data', this.handlers.data)
       return false
     }
@@ -294,20 +279,17 @@ class RangeSender {
       }
       onDataRanges.write('\r\n')
       this.response.write(onDataRanges.getContents())
-      debug('New range started.')
       return true
     }
 
     // No headers means we're finishing the last range.
     this.response.write(`\r\n--${this.rangeBoundary}--\r\n`)
-    debug('End of ranges sent.')
     this.stream.off('data', this.handlers.data)
     return false
   }
 
   onOpenRanges() {
     // File got opened, so we can set headers/status
-    debug('Open succeeded:', this.name, this.type)
     this.opened = true
 
     this.response.header('Accept-Ranges', 'bytes')
@@ -347,34 +329,29 @@ class RangeSender {
     // The simplest optimization would be at ever range start to seek() to the
     // start.
     const chunkRange = [this.readOffset, this.readOffset + chunk.length - 1]
-    debug('= Got chunk with byte range', chunkRange)
     while (true) {
       let reqRange = this.ranges.ranges[this.rangeIndex]
       if (!reqRange) {
         break
       }
-      debug('Current requested range is', reqRange)
+
       if (!reqRange[1]) {
         reqRange = [reqRange[0], Number.MAX_SAFE_INTEGER]
-        debug('Treating as', reqRange)
       }
 
       // No overlap in the chunk and requested range; don't write.
       if (chunkRange[1] < reqRange[0] || chunkRange[0] > reqRange[1]) {
-        debug('Ignoring chunk; it is out of range.')
         break
       }
 
       // Since there is overlap, find the segment that's entirely within the
       // chunk.
       const segment = [Math.max(chunkRange[0], reqRange[0]), Math.min(chunkRange[1], reqRange[1])]
-      debug('Segment to send within chunk is', segment)
 
       // Normalize the segment to a chunk offset
       const start = segment[0] - this.readOffset
       const end = segment[1] - this.readOffset
       const len = end - start + 1
-      debug('Offsets into buffer are', [start, end], 'with length', len)
 
       // Write the slice that we want to write. We first create a buffer from the
       // chunk. Then we slice a new buffer from the same underlying ArrayBuffer,
@@ -385,12 +362,10 @@ class RangeSender {
 
       // If the requested range is finished, we should start the next one.
       if (reqRange[1] > chunkRange[1]) {
-        debug('Chunk is finished, but the requested range is missing bytes.')
         break
       }
 
       if (reqRange[1] <= chunkRange[1]) {
-        debug('Range is finished.')
         if (!this.nextRange(segment)) {
           break
         }
@@ -424,11 +399,9 @@ class RangeSender {
     this.handlers.end = this.onEnd.bind(this)
 
     if (this.ranges) {
-      debug('Preparing to handle ranges.')
       this.handlers.open = this.onOpenRanges.bind(this)
       this.handlers.data = this.onDataRanges.bind(this)
     } else {
-      debug('No ranges, just send the whole file.')
       this.handlers.open = this.onOpenNoRange.bind(this)
       this.handlers.data = this.onDataNoRange.bind(this)
     }

storage-node/scripts/compose/devchain-and-ipfs-node/docker-compose.yaml

@@ -4,8 +4,16 @@ services:
     image: ipfs/go-ipfs:latest
     ports:
       - '127.0.0.1:5001:5001'
+      - '127.0.0.1:8080:8080'
     volumes:
       - ipfs-data:/data/ipfs
+    entrypoint: ''
+    command: |
+      /bin/sh -c "
+        set -e
+        /usr/local/bin/start_ipfs config --json Gateway.PublicGateways '{\"localhost\": null }'
+        /sbin/tini -- /usr/local/bin/start_ipfs daemon --migrate=true
+      "
   chain:
     image: joystream/node:latest
     ports:

types/README.md

@@ -15,37 +15,30 @@ yarn add @joystream/types
 npm install --save @joystream/types
 ```
 
-## Registering the types
-
-Call `registerJoystreamTypes()` before creating a Polkadot API client.
+## Example usage
 
 ```javascript
-import { registerJoystreamTypes } from '@joystream/types';
-import { ApiPromise, WsProvider } from '@polkadot/api';
+import { types } from '@joystream/types'
+import { ApiPromise, WsProvider } from '@polkadot/api'
 
-async function main () {
+async function main() {
   // Initialise the provider to connect to the local node
-  const provider = new WsProvider('ws://127.0.0.1:9944');
-
-  // Register types before creating the API
-  registerJoystreamTypes();
+  const provider = new WsProvider('ws://127.0.0.1:9944')
 
   // Create the API and wait until ready
-  const api = await ApiPromise.create({ provider });
+  const api = await ApiPromise.create({ provider, types })
+
+  await api.isReady
 
-  // Retrieve the chain & node information information via RPC calls
+  // Retrieve the chain & node information information via rpc calls
   const [chain, nodeName, nodeVersion] = await Promise.all([
     api.rpc.system.chain(),
     api.rpc.system.name(),
-    api.rpc.system.version()
-  ]);
+    api.rpc.system.version(),
+  ])
 
-  console.log(`Chain ${chain} using ${nodeName} v${nodeVersion}`);
+  console.log(`Chain ${chain} using ${nodeName} v${nodeVersion}`)
 }
 
-main();
-```
-
-## Examples
-
-See [joystream-api-examples](https://github.com/Joystream/joystream-api-examples) for some additional examples on usage.
+main()
+```

yarn.lock

@@ -3361,7 +3361,7 @@
     memoizee "^0.4.14"
     rxjs "^6.6.0"
 
-"@polkadot/api@1.26.1", "@polkadot/api@^0.96.1", "@polkadot/api@^1.26.1":
+"@polkadot/api@1.26.1", "@polkadot/api@^1.26.1":
   version "1.26.1"
   resolved "https://registry.yarnpkg.com/@polkadot/api/-/api-1.26.1.tgz#215268489c10b1a65429c6ce451c8d65bd3ad843"
   integrity sha512-al8nmLgIU1EKo0oROEgw1mqUvrHJu4gKYBwnFONaEOxHSxBgBSSgNy1MWKNntAQYDKA4ETCj4pz7ZpMXTx2SDA==
@@ -13595,7 +13595,7 @@ http-proxy-middleware@0.19.1:
     lodash "^4.17.11"
     micromatch "^3.1.10"
 
-http-proxy-middleware@^1.0.3:
+http-proxy-middleware@^1.0.3, http-proxy-middleware@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-1.0.5.tgz#4c6e25d95a411e3d750bc79ccf66290675176dc2"
   integrity sha512-CKzML7u4RdGob8wuKI//H8Ein6wNTEQR7yjVEzPbhBLGdOfkfvgTnp2HLnniKBDP9QW4eG10/724iTWLBeER3g==
@@ -17753,7 +17753,7 @@ mime-types@^2.1.12, mime-types@~2.1.19, mime-types@~2.1.24:
   dependencies:
     mime-db "1.42.0"
 
-mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@~2.1.17:
+mime-types@^2.1.18, mime-types@^2.1.22, mime-types@^2.1.26, mime-types@^2.1.27, mime-types@~2.1.17:
   version "2.1.27"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
   integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==