update @joystream/types to 0.16.1 ; optimize data retrieval

package.json

@@ -1,13 +1,12 @@
 {
   "name": "jsstats",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "private": true,
   "dependencies": {
     "axios": "^0.21.1",
     "bootstrap": "^4.3.1",
     "chalk": "^4.1.0",
     "concurrently": "^4.1.1",
-    "connect-session-sequelize": "^6.0.0",
     "cors": "^2.8.5",
     "express": "^4.17.1",
     "express-session": "^1.16.2",
@@ -15,8 +14,6 @@
     "morgan": "^1.9.1",
     "multer": "^1.4.2",
     "nodemon": "^1.19.1",
-    "passport": "^0.4.0",
-    "passport-local": "^1.0.0",
     "pg": "^8.5.1",
     "pm2": "^4.5.5",
     "react": "^17.0.1",
@@ -59,7 +56,8 @@
     ]
   },
   "devDependencies": {
-    "@joystream/types": "^0.14.0",
+    "@joystream/types": "^0.16.1",
+    "@types/cors": "^2.8.12",
     "@types/express": "^4.17.11",
     "@types/morgan": "^1.9.2",
     "@types/node": "^14.14.31",

server/api/councils.ts

@@ -4,8 +4,7 @@ import donate from '../donate'
 
 router.get('/', async (req: any, res: any, next: any) => {
   try {
-    Council.findAll().then((m: any) => res.json(m))
-    //Council.findAllWithIncludes().then((m: any) => res.json(m))
+    Council.findAllWithIncludes().then((m: any) => res.json(m))
   } catch (err) {
     next(err)
   }

server/api/posts.ts

@@ -4,8 +4,7 @@ import donate from '../donate'
 
 router.get('/', async (req: any, res: any, next: any) => {
   try {
-    Post.findAll().then((p: any) => res.json(p))
-    //Post.findAllWithIncludes().then((p: any) => res.json(p))
+    Post.findAllWithIncludes().then((p: any[]) => res.json(p))
   } catch (err) {
     next(err)
   }

server/api/proposals.ts

@@ -4,8 +4,7 @@ import donate from '../donate'
 
 router.get('/', async (req: any, res: any, next: any) => {
   try {
-    Proposal.findAll().then((p: any) => res.json(p))
-    //Proposal.findAllWithIncludes().then((p: any) => res.json(p))
+    Proposal.findAllWithIncludes().then((p: any) => res.json(p))
   } catch (err) {
     next(err)
   }

server/config.json

@@ -1,4 +1,7 @@
 {
+    "port": "3500",
     "domain": "https://testnet.joystream.org",
-    "wsLocation": "wss://joystreamstats.live:9945"
+    "wsLocation": "wss://joystreamstats.live:9945",
+    "jsEndpoint": "wss://rome-rpc-endpoint.joystream.org:9944",
+    "allowedOrigins": "https://joystreamstats.live,https://dev.joystreamstats.live"
 }

server/index.ts

@@ -1,4 +1,5 @@
 import express from 'express'
+import cors from 'cors'
 import path from 'path'
 import morgan from 'morgan'
 import socketio from 'socket.io'
@@ -8,93 +9,38 @@ import db from './db'
 const pg = require('pg')
 delete pg.native
 
-const { connectUpstream } = require('./joystream/ws')
-const { addBlockRange } = require('./joystream')
+import { connectUpstream } from './joystream/ws'
+import { addBlockRange } from './joystream'
+import { setupSocket } from './socket'
 
-const { setupSocket } = require('./socket')
-const PORT: number = process.env.PORT ? +process.env.PORT : 3500
+import { allowedOrigins, port } from './config.json'
+const PORT: number = process.env.PORT ? +process.env.PORT : +port
 
 const app = express()
-const server = app.listen(PORT, () =>
+const server = app.listen(PORT, async () => {
   console.log(`[Express] Listening on port ${PORT}`, ascii)
-)
 
-;(async () => {
   const api = await connectUpstream()
 
   const io: any = socketio(server)
   setupSocket(io, api)
   const args = process.argv.slice(2)
   if (args.length === 2) {
-    const startBlock = args[0]
-    const endBlock = args[1]
+    const startBlock = +args[0]
+    const endBlock = +args[1]
     console.log(`[Joystream] syncing blocks from ${startBlock} to ${endBlock}`)
     addBlockRange(api, startBlock, endBlock)
   }
-})()
-
-//const cors = require("cors");
-//const passport = require('passport')
-//const LocalStrategy = require('passport-local')
-//const session = require('express-session')
-//const SequelizeStore = require('connect-session-sequelize')(session.Store)
-//const sessionStore = new SequelizeStore({ db })
+})
 
 app.use(morgan('dev'))
-//app.use(cors({ credentials: true, origin: URL }))
-// passport.use(
-//   new LocalStrategy(async (username, password, done) => {
-//     const user = await db.models.user.findOne({ where: { username } })
-//     if (!user) {
-//       return done(null, false, { message: 'Incorrect username.' })
-//     }
-//     if (!user.correctPassword(password)) {
-//       return done(null, false, { message: 'Incorrect password.' })
-//     }
-//     return done(null, user)
-//   })
-// )
-
-//passport.serializeUser((user, cb) => cb(null, user.id))
-
-// passport.deserializeUser(async (id, cb) => {
-//   try {
-//     const user = await db.models.user.findByPk(id)
-//     if (!user) return cb(null, { id: 0 })
-//     const idType = user.isAdmin ? 'trainerId' : 'userId'
-//     const { sessions, reviews, notes, referrers } = await user.fetchMyData(
-//       idType
-//     )
-//     user.setDataValue('sessions', sessions)
-//     user.setDataValue('reviews', reviews)
-//     user.setDataValue('notes', notes)
-//     user.setDataValue('referrers', referrers)
-//     cb(null, user)
-//   } catch (err) {
-//     console.log('error', err)
-//     cb(err)
-//   }
-// })
-
-// app.use(
-//   session({
-//     secret: process.env.SESSION_SECRET || 'LAKaLIHWIUH9*&h3ISVAEOIUFHAW83w',
-//     store: sessionStore,
-//     resave: false,
-//     saveUninitialized: false,
-//     cookie: { maxAge: 3600000 },
-//   })
-// )
-//
-// app.use(passport.initialize())
-// app.use(passport.session())
+app.use(cors({ credentials: true, origin: allowedOrigins.split(',') }))
 
 // body parsing middleware
 app.use(express.json())
 app.use(express.urlencoded({ extended: true }))
 app.use(require('body-parser').text())
 app.use('/api', require('./api'))
-//app.use('/auth', require('./auth'))
 app.use(
   '/static',
   express.static(path.resolve(__dirname, '..', 'build', 'static'))
@@ -114,6 +60,3 @@ app.use((err: any, req: any, res: any, next: any) => {
   res.status(err.status || 500).send(err.message || 'Internal server error.')
   next()
 })
-
-//module.exports = {}
-//export {}

server/joystream/index.ts

@@ -25,6 +25,7 @@ import axios from 'axios'
 import moment from 'moment'
 import chalk from 'chalk'
 
+import { MemberId } from '@joystream/types/members'
 import { VoteKind } from '@joystream/types/proposals'
 import { Seats } from '@joystream/types/council'
 import { AccountInfo } from '@polkadot/types/interfaces/system'
@@ -49,7 +50,6 @@ import {
   ActiveEraInfo,
   EventRecord,
 } from '@polkadot/types/interfaces'
-import Option from '@polkadot/types/codec/Option'
 import { Vec } from '@polkadot/types'
 
 // TODO fetch consts from db/chain
@@ -57,29 +57,36 @@ const TERMDURATION = 144000
 const VOTINGDURATION = 57601
 const CYCLE = VOTINGDURATION + TERMDURATION
 
-const DELAY = 0 // ms
+const WORKERS = 2
+const DELAY = 100 // ms
 let lastUpdate = 0
 let queuedAll = false
 let queue: any[] = []
 let processing = ''
-let busy = false
+let busy = 0
 
 const processNext = async () => {
-  if (busy) return
+  if (busy === WORKERS) return
   const task = queue.shift()
   if (!task) return
+  busy++
+  if (busy < WORKERS) setTimeout(processNext, DELAY)
   const result = await task()
-  busy = false
-  setTimeout(() => processNext(), DELAY)
+  busy--
+  setTimeout(processNext, DELAY)
 }
 
-const getBlockHash = (api: Api, blockId: number) =>
-  api.rpc.chain.getBlockHash(blockId).then((array: any) => array.toHuman())
+const getBlockHash = (api: Api, blockId: number) => {
+  //console.log(`getBlockHash`, blockId)
+  return api.rpc.chain
+    .getBlockHash(blockId)
+    .then((array: any) => array.toHuman())
+}
 
 const getEraAtHash = (api: Api, hash: string) =>
   api.query.staking.activeEra
     .at(hash)
-    .then((era: Option<ActiveEraInfo>) => era.unwrap().index.toNumber())
+    .then((era: any) => era.unwrap().index.toNumber())
 
 const getEraAtBlock = async (api: Api, block: number) =>
   getEraAtHash(api, await getBlockHash(api, block))
@@ -99,7 +106,7 @@ const findCouncilAtBlock = (api: Api, block: number) =>
     },
   })
 
-const addBlock = async (
+export const addBlock = async (
   api: Api,
   io: any,
   header: { number: number; author: string },
@@ -127,15 +134,18 @@ const addBlock = async (
   const key = header.author?.toString()
   const [account] = await Account.findOrCreate({ where: { key } })
   await block.setValidator(account.key)
-  //account.addBlock(block.id) // TODO needed?
   io.emit('block', await Block.findByIdWithIncludes(id))
 
-  // logging
+  // log process
   const handle = await getHandleOrKey(api, key)
-  const q = queue.length ? chalk.green(` [${queue.length}:${processing}]`) : ''
+  const q = queue.length
+    ? chalk.green(` [${busy}/${queue.length}:${processing}]`)
+    : '[no tasks]'
   console.log(`[Joystream] block ${id} ${handle} ${q}`)
 
-  return updateStatus(api, status, id)
+  return id / 10 === Math.floor(id / 10)
+    ? updateStatus(api, status, id)
+    : status
 }
 
 const processBlock = async (api: Api, id: number) => {
@@ -143,28 +153,31 @@ const processBlock = async (api: Api, id: number) => {
   if (exists) return exists
 
   processing = `block ${id}`
+  console.log(processing)
   const last = await Block.findByPk(id - 1)
-  let lastBlockTimestamp;
+  let lastBlockTimestamp
   if (last) {
-    lastBlockTimestamp = last.timestamp.getTime();
+    if (last.timestamp) lastBlockTimestamp = last.timestamp.getTime()
   } else {
-    let lastBlockHash = await getBlockHash(api, id - 1);
-    lastBlockTimestamp = await getTimestamp(api, lastBlockHash);
+    let lastBlockHash = await getBlockHash(api, id - 1)
+    lastBlockTimestamp = await getTimestamp(api, lastBlockHash)
   }
 
   const [block] = await Block.findOrCreate({ where: { id } })
   block.hash = await getBlockHash(api, id)
-  let currentBlockTimestamp = await getTimestamp(api, block.hash);
+  let currentBlockTimestamp = await getTimestamp(api, block.hash)
   block.timestamp = new Date(currentBlockTimestamp)
-  block.blocktime = (currentBlockTimestamp - lastBlockTimestamp)
+  if (lastBlockTimestamp)
+    block.blocktime = currentBlockTimestamp - lastBlockTimestamp
   block.save()
 
   processEvents(api, id, block.hash)
-  await importEraAtBlock(api, id, block.hash)
+  importEraAtBlock(api, id, block.hash)
+  processNext()
   return block
 }
 
-const addBlockRange = async (
+export const addBlockRange = async (
   api: Api,
   startBlock: number,
   endBlock: number
@@ -179,7 +192,7 @@ const updateStatus = async (api: Api, old: Status, block: number) => {
     era: await getEraAtBlock(api, block),
     round: Number(await api.query.councilElection.round()),
     members: (await api.query.members.nextMemberId()) - 1,
-    channels: await get.currentChannelId(api),
+    channels: 0, //await get.currentChannelId(api),
 
     categories: await get.currentCategoryId(api),
     threads: await get.currentThreadId(api),
@@ -188,55 +201,39 @@ const updateStatus = async (api: Api, old: Status, block: number) => {
     proposals: await get.proposalCount(api),
     proposalPosts: (await api.query.proposalsDiscussion.postCount()).toHuman(),
   }
+  console.log(`updated status`)
   if (!queuedAll) fetchAll(api, status)
   else {
-    // TODO catch if more than one are added
-    status.members > old.members && fetchMember(api, status.members)
-    status.posts > old.posts && fetchPost(api, status.posts)
-    status.proposals > old.proposals && fetchProposal(api, status.proposals)
-    status.channels > old.channels && fetchChannel(api, status.channels)
-    status.categories > old.categories && fetchCategory(api, status.categories)
-    status.proposalPosts > old.proposalPosts &&
-      fetchProposalPosts(api, status.proposalPosts)
+    fetchMember(api, status.members)
+    fetchCategory(api, status.categories)
+    fetchThread(api, status.threads)
+    fetchPost(api, status.posts)
+    const active = Proposal.findAll({
+      where: { result: `Pending` },
+    }).then((pp: { id: number }[]) =>
+      pp.forEach((p: { id: number }) => {
+        fetchProposal(api, p.id)
+        ProposalVote.destroy({ where: { vote: `Reject` } })
+      })
+    )
   }
   return status
 }
 
 const fetchAll = async (api: Api, status: Status) => {
+  queue.push(() => fetchProposal(api, status.proposals))
   queue.push(() => fetchAccounts(api, status.block))
-
-  for (let id = status.members; id > 0; id--) {
-    queue.push(() => fetchMember(api, id))
-  }
-  for (let id = status.round; id > 0; id--) {
-    queue.push(() => fetchCouncil(api, id))
-  }
-
-  for (let id = status.proposals; id > 0; id--) {
-    queue.push(() => fetchProposal(api, id))
-  }
-
-  for (let id = status.channels; id > 0; id--) {
-    queue.push(() => fetchChannel(api, id))
-  }
-  for (let id = status.categories; id > 0; id--) {
-    queue.push(() => fetchCategory(api, id))
-  }
-  for (let id = status.threads; id > 0; id--) {
-    queue.push(() => fetchThread(api, id))
-  }
-  for (let id = status.posts; id > 0; id--) {
-    queue.push(() => fetchPost(api, id))
-  }
-
+  queue.push(() => fetchMember(api, status.members))
   queue.push(() => fetchProposalPosts(api, status.proposalPosts))
-  queue.push(() => addBlockRange(api, 1, status.block))
+
+  queue.push(() => fetchCouncil(api, status.round, status.block))
+  //queue.push(() => addBlockRange(api, 1, status.block))
   queuedAll = true
-  processNext()
 }
 
 const processEvents = async (api: Api, blockId: number, hash: string) => {
   processing = `events block ${blockId}`
+  console.log(processing)
   try {
     const blockEvents = await api.query.system.events.at(hash)
     blockEvents.forEach(({ event }: EventRecord) => {
@@ -250,7 +247,7 @@ const processEvents = async (api: Api, blockId: number, hash: string) => {
 }
 
 const fetchValidators = async (api: Api, hash: string) =>
-  api.query.staking.snapshotValidators.at(hash) as Option<Vec<AccountId>>
+  api.query.staking.snapshotValidators.at(hash)
 
 const importEraAtBlock = async (api: Api, blockId: number, hash: string) => {
   const id = await getEraAtHash(api, hash)
@@ -260,26 +257,21 @@ const importEraAtBlock = async (api: Api, blockId: number, hash: string) => {
 
   processing = `era ${id}`
   try {
-    fetchValidators(api, hash).then(
-      async (snapshot: Option<Vec<AccountId>>) => {
-        if (snapshot.isEmpty) return
-        console.log(`[Joystream] Found validator info for era ${id}`)
-        const validatorCount = snapshot.unwrap().length
-        era.slots = (await api.query.staking.validatorCount.at(hash)).toNumber()
-        era.active = Math.min(era.slots, validatorCount)
-        era.waiting =
-          validatorCount > era.slots ? validatorCount - era.slots : 0
-        era.stake = await api.query.staking.erasTotalStake.at(hash, id)
-        const chainTimestamp = (await api.query.timestamp.now.at(
-          hash
-        )) as Moment
-        era.timestamp = moment(chainTimestamp.toNumber())
-        // era.update({ slots, active, waiting, stake, timestamp })
-        era.blockId = id
-        era.save()
-        updateBalances(api, hash)
-      }
-    )
+    fetchValidators(api, hash).then(async (snapshot: any) => {
+      if (snapshot.isEmpty) return
+      console.log(`[Joystream] Found validator info for era ${id}`)
+      const validatorCount = snapshot.unwrap().length
+      era.slots = (await api.query.staking.validatorCount.at(hash)).toNumber()
+      era.active = Math.min(era.slots, validatorCount)
+      era.waiting = validatorCount > era.slots ? validatorCount - era.slots : 0
+      era.stake = await api.query.staking.erasTotalStake.at(hash, id)
+      const chainTimestamp = (await api.query.timestamp.now.at(hash)) as Moment
+      era.timestamp = moment(chainTimestamp.toNumber())
+      // era.update({ slots, active, waiting, stake, timestamp })
+      era.blockId = id
+      era.save()
+      updateBalances(api, hash)
+    })
   } catch (e) {
     console.error(`import era ${blockId} ${hash}`, e)
   }
@@ -332,40 +324,14 @@ const fetchTokenomics = async () => {
   // TODO save 'tokenomics', data
 }
 
-const fetchChannel = async (api: Api, id: number) => {
-  if (id <= 0) return
-  const exists = await Channel.findByPk(id)
-  if (exists) return exists
-
-  processing = `channel ${id}`
-  const data = await api.query.contentWorkingGroup.channelById(id)
-  const { handle, title, description, avatar, banner, content, created } = data
-  // TODO const accountId = String(data.role_account)
-  const channel = {
-    id,
-    handle: String(handle),
-    title: String(title),
-    description: String(description),
-    avatar: String(avatar),
-    banner: String(banner),
-    content: String(content),
-    publicationStatus: data.publication_status === 'Public' ? true : false,
-    curation: String(data.curation_status),
-    createdAt: +created,
-    principal: Number(data.principal_id),
-  }
-  const chan = await Channel.create(channel)
-  const owner = await fetchMember(api, data.owner)
-  chan.setOwner(owner)
-  return chan
-}
-
 const fetchCategory = async (api: Api, id: number) => {
   if (id <= 0) return
+  queue.push(() => fetchCategory(api, id - 1))
   const exists = await Category.findByPk(+id)
   if (exists) return exists
 
   processing = `category ${id}`
+  console.log(processing)
   const data = await api.query.forum.categoryById(id)
 
   const { title, description, deleted, archived } = data
@@ -388,10 +354,12 @@ const fetchCategory = async (api: Api, id: number) => {
 
 const fetchPost = async (api: Api, id: number) => {
   if (id <= 0) return
+  queue.push(() => fetchPost(api, id - 1))
   const exists = await Post.findByPk(id)
   if (exists) return exists
 
   processing = `post ${id}`
+  console.log(processing)
   const data = await api.query.forum.postById(id)
 
   const author: string = String(data.author_id)
@@ -417,6 +385,7 @@ const createModeration = async (
   object: { setModeration: (id: number) => {} }
 ) => {
   if (key === '') return
+  return // TODO
   await Account.findOrCreate({ where: { key } })
   const moderation = await Moderation.create({ moderatorKey: key })
   object.setModeration(moderation.id)
@@ -429,6 +398,7 @@ const fetchThread = async (api: Api, id: number) => {
   if (exists) return exists
 
   processing = `thread ${id}`
+  console.log(processing)
   const data = await api.query.forum.threadById(id)
   const { title, moderation, nr_in_category } = data
   const account = String(data.author_id)
@@ -457,19 +427,22 @@ const fetchThread = async (api: Api, id: number) => {
   return thread
 }
 
-const fetchCouncil = async (api: Api, round: number) => {
+const fetchCouncil = async (api: Api, round: number, head: number) => {
   if (round <= 0) return console.log(chalk.red(`[fetchCouncil] round:${round}`))
+  queue.push(() => fetchCouncil(api, round - 1, head))
 
   const exists = await Council.findByPk(round)
   if (exists) return exists
-
   processing = `council ${round}`
+  console.log(processing)
   const start = 57601 + (round - 1) * CYCLE
   const end = start + TERMDURATION
   let council = { round, start, end, startDate: 0, endDate: 0 }
   let seats: Seats
+  if (start > head || end > head) return
   try {
     const startHash = await getBlockHash(api, start)
+    console.log(`start`, start, `hash`, startHash)
     council.startDate = await getTimestamp(api, startHash)
     seats = await api.query.council.activeCouncil.at(startHash)
   } catch (e) {
@@ -512,17 +485,21 @@ const fetchCouncil = async (api: Api, round: number) => {
 
 const fetchProposal = async (api: Api, id: number) => {
   if (id <= 0) return
-  const exists = await Proposal.findByPk(+id)
-  if (exists) {
-    fetchProposalVotes(api, exists)
+  queue.push(() => fetchProposal(api, id - 1))
+
+  const exists = await Proposal.findByIdWithIncludes(id)
+  if (exists && exists.result !== `Pending`) {
+    //if (!exists.votes.length) queue.push(() => fetchProposalVotes(api, id))
     return exists
   }
-
   processing = `proposal ${id}`
   const proposal = await get.proposalDetail(api, id)
+  console.log(`proposal ${id}: ${proposal.result}`)
   await fetchMember(api, proposal.authorId)
-  fetchProposalVotes(api, proposal)
-  return Proposal.create(proposal)
+  queue.push(() => fetchProposalVotes(api, id))
+  if (exists) Proposal.update(proposal, { where: { id } })
+  else Proposal.create(proposal)
+  return proposal
 }
 
 const fetchProposalPost = (api: Api, threadId: number, postId: number) =>
@@ -541,6 +518,7 @@ const fetchProposalPosts = async (api: Api, posts: number) => {
     }
 
     processing = `proposal post ${id}/${posts} ${proposalId}/${threads}`
+    console.log(processing)
     const post = await fetchProposalPost(api, proposalId, id)
 
     if (!post.text.length) {
@@ -548,7 +526,8 @@ const fetchProposalPosts = async (api: Api, posts: number) => {
       continue
     }
 
-    const proposal = await Proposal.findByPk(proposalId)
+    let proposal = await Proposal.findByPk(proposalId)
+    if (!proposal) proposal = await fetchProposal(api, proposalId)
     if (!proposal) {
       console.warn(`[fetchProposalPosts] proposal ${proposalId} not found.`)
       id++
@@ -568,52 +547,45 @@ const fetchProposalPosts = async (api: Api, posts: number) => {
   }
 }
 
-const fetchProposalVotes = async (api: Api, proposal: ProposalDetail) => {
-  if (!proposal) return console.error(`[fetchProposalVotes] empty proposal`)
+const fetchProposalVotes = async (api: Api, id: number) => {
+  const proposal = await Proposal.findByPk(id)
+  if (!proposal) return
   processing = `votes proposal ${proposal.id}`
+  console.log(processing)
+
+  // find council for creation and finalization time
   const { createdAt, finalizedAt } = proposal
-  try {
-    const start = createdAt ? await findCouncilAtBlock(api, createdAt) : null
-    if (start) start.addProposal(proposal.id)
-    else
-      return console.error(
-        `[fetchProposalVotes] no council found for proposal ${proposal.id}`
-      )
-    // some proposals make it into a second term
-    const end = finalizedAt ? await findCouncilAtBlock(api, finalizedAt) : null
-    const councils = [start && start.round, end && end.round]
-    const consuls = await Consul.findAll({
-      where: { councilRound: { [Op.or]: councils } },
-    })
-    consuls.map(({ id, memberId }: any) =>
-      fetchProposalVoteByConsul(api, proposal.id, id, memberId)
-    )
-  } catch (e) {
-    console.log(`failed to fetch votes of proposal ${proposal.id}`, e)
-  }
-}
+  const start = createdAt ? await findCouncilAtBlock(api, createdAt) : null
+  if (start) start.addProposal(proposal.id)
+  else return
 
-const fetchProposalVoteByConsul = async (
-  api: Api,
-  proposalId: number,
-  consulId: number,
-  memberId: number
-): Promise<any> => {
-  processing = `vote by ${consulId} for proposal ${proposalId}`
-  const exists = await ProposalVote.findOne({
-    where: { proposalId, memberId, consulId },
-  })
-  if (exists) return exists
+  // some proposals make it into a second term
+  const end = finalizedAt ? await findCouncilAtBlock(api, finalizedAt) : null
+  const councils = [start && start.round, end && end.round]
 
-  const query = api.query.proposalsEngine
-  const args = [proposalId, memberId]
+  // fetch votes
+  const entries = await api.query.proposalsEngine.voteExistsByProposalByVoter.entries(
+    id
+  )
+  // find memberId and consulId for each vote
 
-  const hasVoted = await query.voteExistsByProposalByVoter.size(...args)
-  if (!hasVoted.toNumber()) return
+  entries.forEach(async (e: any) => {
+    const memberId = Number(e[0].args[1])
+    const vote = String(e[1])
 
-  const vote = (await query.voteExistsByProposalByVoter(...args)).toHuman()
-  await fetchMember(api, memberId) // TODO needed?
-  return ProposalVote.create({ vote: vote, proposalId, consulId, memberId })
+    const consul = await Consul.findOne({
+      where: { memberId, councilRound: { [Op.or]: councils } },
+    })
+    if (consul) {
+      const v = { vote, proposalId: id, consulId: consul.id, memberId }
+      const exists = await ProposalVote.findOne({
+        where: { proposalId: id, consulId: consul.id, memberId },
+      })
+      if (exists) return
+      console.log(v)
+      ProposalVote.create(v)
+    }
+  })
 }
 
 // accounts
@@ -653,6 +625,7 @@ const fetchMember = async (
   id: number
 ): Promise<MemberType | undefined> => {
   if (id <= 0) return
+  queue.push(() => fetchMember(api, id - 1))
   const exists = await Member.findByPk(+id)
   if (exists) return exists
 
@@ -672,5 +645,3 @@ const fetchMember = async (
     }
   )
 }
-
-module.exports = { addBlock, addBlockRange }

server/joystream/lib/announcements.ts

@@ -41,7 +41,7 @@ export const channels = async (
     member.handle = await memberHandle(api, member.id.toJSON());
     member.url = `${domain}/#/members/${member.handle}`;
     messages.push(
-      `<b>Channel <a href="${domain}/#//media/channels/${id}">${channel.title}</a> by <a href="${member.url}">${member.handle} (${member.id})</a></b>`
+      `<b>Channel <a href="${domain}/#//media/channels/${id}">${id}</a> by <a href="${member.url}">${member.handle} (${member.id})</a></b>`
     );
   }
   sendMessage(messages.join("\r\n\r\n"));

server/joystream/lib/getters.ts

@@ -15,13 +15,6 @@ import { MemberId, Membership } from '@joystream/types/members'
 import { Proposal } from '@joystream/types/proposals'
 import { AccountId } from '@polkadot/types/interfaces'
 
-// channel
-
-export const currentChannelId = async (api: Api): Promise<number> => {
-  const id: ChannelId = await api.query.contentWorkingGroup.nextChannelId()
-  return Number(id) - 1
-}
-
 // members
 
 export const membership = async (

server/joystream/ws.ts

@@ -1,20 +1,17 @@
- import {wsLocation} from "../config.json"
 import { ApiPromise, WsProvider } from '@polkadot/api'
 import { types } from '@joystream/types'
+import { wsLocation } from '../config.json'
 
-
-const connectUpstream = async (): Promise<ApiPromise> => {
-    try {
-        //console.debug(`[Joystream] Connecting to ${wsLocation}`)
-        const provider = new WsProvider(wsLocation)
-        const api = await ApiPromise.create({ provider, types })
-        await api.isReady
-        console.debug(`[Joystream] Connected to ${wsLocation}`)
-        return api
-    } catch (e) {
-        console.error(`[Joystream] upstream connection failed`, e)
-        throw new Error()
-    }
+export const connectUpstream = async (): Promise<ApiPromise> => {
+  console.debug(`[Joystream] Connecting to ${wsLocation}`)
+  try {
+    const provider = new WsProvider(wsLocation)
+    const api = await ApiPromise.create({ provider, types })
+    await api.isReady
+    console.debug(`[Joystream] Connected to ${wsLocation}`)
+    return api
+  } catch (e) {
+    console.error(`[Joystream] upstream connection failed`, e)
+    throw new Error()
+  }
 }
-
-module.exports = { connectUpstream }

server/socket.ts

@@ -1,12 +1,12 @@
+import  { ApiPromise, WsProvider } from '@polkadot/api'
 import { Api, Header, Status } from './types'
-const { types } = require('@joystream/types')
+import { types } from '@joystream/types'
+import chalk from 'chalk'
 
-const { Block } = require('./db/models')
-const { ApiPromise, WsProvider } = require('@polkadot/api')
-const { addBlock } = require('./joystream')
-const chalk = require('chalk')
+import { Block } from './db/models'
+import  { addBlock } from './joystream'
 
-const setupSocket = async (io: any, api: Api) => {
+export const setupSocket = async (io: any, api: Api) => {
   let status: Status
   let lastHeader: Header = { number: 0, timestamp: 0, author: '' }
 
@@ -36,5 +36,3 @@ const setupSocket = async (io: any, api: Api) => {
     })
   })
 }
-
-module.exports = { setupSocket }

tsconfig.json

@@ -20,7 +20,5 @@
     "noEmit": true,
     "jsx": "react-jsx"
   },
-  "include": [
-      "src", "server"
-  ]
+  "include": [ "server" ]
 }