3
1

index.ts 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. import {
  2. Block,
  3. Category,
  4. Channel,
  5. Council,
  6. Era,
  7. Event,
  8. Member,
  9. Post,
  10. Proposal,
  11. Thread,
  12. } from '../db/models'
  13. const models: { [key: string]: any } = {
  14. channel: Channel,
  15. proposal: Proposal,
  16. category: Category,
  17. thread: Thread,
  18. post: Post,
  19. block: Block,
  20. council: Council,
  21. member: Member,
  22. era: Era,
  23. }
  24. import * as get from './lib/getters'
  25. import axios from 'axios'
  26. import moment from 'moment'
  27. import { VoteKind } from '@joystream/types/proposals'
  28. import { EventRecord } from '@polkadot/types/interfaces'
  29. import {
  30. Api,
  31. Handles,
  32. IState,
  33. MemberType,
  34. CategoryType,
  35. ChannelType,
  36. PostType,
  37. Seat,
  38. ThreadType,
  39. CouncilType,
  40. ProposalDetail,
  41. Status,
  42. } from '../types'
  43. // queuing
  44. let lastUpdate = 0
  45. const queue: any[] = []
  46. let inProgress = false
  47. const enqueue = (fn: any) => {
  48. queue.push(fn)
  49. processNext()
  50. }
  51. const processNext = async () => {
  52. if (inProgress) return
  53. inProgress = true
  54. const task = queue.pop()
  55. if (task) await task()
  56. inProgress = false
  57. //processNext()
  58. //return queue.length
  59. }
  60. const save = async (model: any, data: any) => {
  61. const Model = models[model]
  62. try {
  63. const exists = await Model.findByPk(data.id)
  64. if (exists) return exists.update(data)
  65. } catch (e) {}
  66. //console.debug(`saving ${data.id}`, `queued tasks: ${queue.length}`)
  67. try {
  68. return Model.create(data)
  69. } catch (e) {
  70. console.warn(`Failed to save ${Model}`, e.message)
  71. }
  72. }
  73. const addBlock = async (
  74. api: Api,
  75. io: any,
  76. header: { number: number; author: string },
  77. status: Status
  78. ): Promise<Status> => {
  79. const id = +header.number
  80. const last = await Block.findByPk(id - 1)
  81. const timestamp = moment.utc(await api.query.timestamp.now()).valueOf()
  82. const blocktime = last ? timestamp - last.timestamp : 6000
  83. const block = await save('block', { id, timestamp, blocktime })
  84. io.emit('block', block)
  85. const author = header.author?.toString()
  86. const member = await fetchMemberByAccount(api, author)
  87. if (member && member.id) block.setAuthor(member)
  88. const currentEra = Number(await api.query.staking.currentEra())
  89. const era = await save('era', { id: currentEra })
  90. era.addBlock(block)
  91. const handle = member ? member.handle : author
  92. const queued = `(queued: ${queue.length})`
  93. console.log(`[Joystream] block ${block.id} ${handle} ${queued}`)
  94. processEvents(api, id)
  95. return updateEra(api, io, status, currentEra)
  96. }
  97. const getBlockHash = (api: Api, blockId: number) =>
  98. api.rpc.chain.getBlockHash(blockId)
  99. const processEvents = async (api: Api, blockId: number) => {
  100. const blockHash = await getBlockHash(api, blockId)
  101. const blockEvents = await api.query.system.events.at(blockHash)
  102. blockEvents.forEach(({ event }: EventRecord) => {
  103. let { section, method, data } = event
  104. Event.create({ blockId, section, method, data: JSON.stringify(data) })
  105. })
  106. }
  107. const updateAccount = async (api: Api, account: string) => {}
  108. const updateEra = async (api: Api, io: any, status: any, era: number) => {
  109. const now: number = moment().valueOf()
  110. if (lastUpdate + 60000 > now) return status
  111. //console.log(`updating status`, lastUpdate)
  112. lastUpdate = now
  113. // session.disabledValidators: Vec<u32>
  114. // check online: imOnline.keys
  115. // imOnline.authoredBlocks: 2
  116. // session.currentIndex: 17,081
  117. const lastReward = Number(await api.query.staking.erasValidatorReward(era))
  118. console.debug(`last reward`, era, lastReward)
  119. if (lastReward > 0) {
  120. } // TODO save lastReward
  121. const nominatorEntries = await api.query.staking.nominators.entries()
  122. const nominators = nominatorEntries.map((n: any) => String(n[0].toHuman()))
  123. const rewardPoints = await api.query.staking.erasRewardPoints(era)
  124. const validatorEntries = await api.query.session.validators()
  125. const validators = validatorEntries.map((v: any) => String(v))
  126. // TODO staking.bondedEras: Vec<(EraIndex,SessionIndex)>
  127. const stashes = await api.derive.staking.stashes()
  128. console.debug(`fetching stakes`)
  129. if (!stashes) return
  130. stashes.forEach(async (validator: string) => {
  131. try {
  132. const prefs = await api.query.staking.erasValidatorPrefs(era, validator)
  133. const commission = Number(prefs.commission) / 10000000
  134. const data = await api.query.staking.erasStakers(era, validator)
  135. let { total, own, others } = data.toJSON()
  136. } catch (e) {
  137. console.warn(`Failed to fetch stakes for ${validator} in era ${era}`, e)
  138. }
  139. })
  140. return {
  141. members: (await api.query.members.nextMemberId()) - 1,
  142. categories: await get.currentCategoryId(api),
  143. threads: await get.currentThreadId(api),
  144. proposals: await get.proposalCount(api),
  145. channels: await get.currentChannelId(api),
  146. posts: await get.currentPostId(api),
  147. proposalPosts: await api.query.proposalsDiscussion.postCount(),
  148. queued: queue.length,
  149. era,
  150. }
  151. }
  152. const validatorStatus = async (api: Api, blockId: number) => {
  153. const hash = await getBlockHash(api, blockId)
  154. let totalValidators = await api.query.staking.snapshotValidators.at(hash)
  155. if (totalValidators.isEmpty) return
  156. let totalNrValidators = totalValidators.unwrap().length
  157. const maxSlots = Number(await api.query.staking.validatorCount.at(hash))
  158. const actives = Math.min(maxSlots, totalNrValidators)
  159. const waiting =
  160. totalNrValidators > maxSlots ? totalNrValidators - maxSlots : 0
  161. let timestamp = await api.query.timestamp.now.at(hash)
  162. const date = moment(timestamp.toNumber()).valueOf()
  163. return { blockId, actives, waiting, maxSlots, date }
  164. }
  165. const fetchTokenomics = async () => {
  166. console.debug(`Updating tokenomics`)
  167. const { data } = await axios.get('https://status.joystream.org/status')
  168. if (!data) return
  169. // TODO save 'tokenomics', data
  170. }
  171. const fetchChannel = async (api: Api, id: number) => {
  172. const exists = await Channel.findByPk(id)
  173. if (exists) return exists
  174. console.debug(`Fetching channel ${id}`)
  175. const data = await api.query.contentWorkingGroup.channelById(id)
  176. const handle = String(data.handle)
  177. const title = String(data.title)
  178. const description = String(data.description)
  179. const avatar = String(data.avatar)
  180. const banner = String(data.banner)
  181. const content = String(data.content)
  182. const ownerId = Number(data.owner)
  183. const accountId = String(data.role_account)
  184. const publicationStatus = data.publication_status === 'Public' ? true : false
  185. const curation = String(data.curation_status)
  186. const createdAt = +data.created
  187. const principal = Number(data.principal_id)
  188. const channel = {
  189. id,
  190. handle,
  191. title,
  192. description,
  193. avatar,
  194. banner,
  195. content,
  196. publicationStatus,
  197. curation,
  198. createdAt,
  199. principal,
  200. }
  201. const chan = await save('channel', channel)
  202. const owner = await fetchMember(api, ownerId)
  203. chan.setOwner(owner)
  204. if (id > 1) fetchChannel(api, id - 1)
  205. return chan
  206. }
  207. const fetchCategory = async (api: Api, id: number) => {
  208. const exists = await Category.findByPk(id)
  209. if (exists) return exists
  210. console.debug(`fetching category ${id}`)
  211. const data = await api.query.forum.categoryById(id)
  212. const threadId = +data.thread_id
  213. const title = String(data.title)
  214. const description = String(data.description)
  215. const createdAt = +data.created_at.block
  216. const deleted = data.deleted
  217. const archived = data.archived
  218. const subcategories = Number(data.num_direct_subcategories)
  219. const moderatedThreads = Number(data.num_direct_moderated_threads)
  220. const unmoderatedThreads = Number(data.num_direct_unmoderated_threads)
  221. const position = +data.position_in_parent_category // TODO sometimes NaN
  222. const moderator: string = String(data.moderator_id) // account
  223. const cat = {
  224. id,
  225. title,
  226. description,
  227. createdAt,
  228. deleted,
  229. archived,
  230. subcategories,
  231. moderatedThreads,
  232. unmoderatedThreads,
  233. //position,
  234. }
  235. const category = await save('category', cat)
  236. const mod = await fetchMemberByAccount(api, moderator)
  237. if (mod) category.setModerator(mod)
  238. if (id > 1) fetchCategory(api, id - 1)
  239. return category
  240. }
  241. const fetchPost = async (api: Api, id: number) => {
  242. const exists = await Post.findByPk(id)
  243. if (exists) return exists
  244. console.debug(`fetching post ${id}`)
  245. const data = await api.query.forum.postById(id)
  246. const threadId = Number(data.thread_id)
  247. const text = data.current_text
  248. const moderation = data.moderation
  249. //const history = data.text_change_history;
  250. const createdAt = data.created_at.block
  251. const author: string = String(data.author_id)
  252. const post = await save('post', { id, text, createdAt })
  253. const thread = await fetchThread(api, threadId)
  254. if (thread) post.setThread(thread)
  255. const member = await fetchMemberByAccount(api, author)
  256. if (member) post.setAuthor(member)
  257. const mod = await fetchMemberByAccount(api, moderation)
  258. if (id > 1) fetchPost(api, id - 1)
  259. return post
  260. }
  261. const fetchThread = async (api: Api, id: number) => {
  262. const exists = await Thread.findByPk(id)
  263. if (exists) return exists
  264. console.debug(`fetching thread ${id}`)
  265. const data = await api.query.forum.threadById(id)
  266. const title = String(data.title)
  267. const categoryId = Number(data.category_id)
  268. const nrInCategory = Number(data.nr_in_category)
  269. const moderation = data.moderation
  270. const createdAt = +data.created_at.block
  271. const account = String(data.author_id)
  272. const thread = await save('thread', { id, title, nrInCategory, createdAt })
  273. const category = await fetchCategory(api, categoryId)
  274. if (category) thread.setCategory(category)
  275. const author = await fetchMemberByAccount(api, account)
  276. if (author) thread.setAuthor(author)
  277. if (moderation) {
  278. /* TODO
  279. Error: Invalid value ModerationAction(3) [Map] {
  280. [1] 'moderated_at' => BlockAndTime(2) [Map] {
  281. [1] 'block' => <BN: 4f4ff>,
  282. [1] 'time' => <BN: 17526e65a40>,
  283. [1] registry: TypeRegistry {},
  284. [1] block: [Getter],
  285. [1] time: [Getter],
  286. [1] typeDefs: { block: [Function: U32], time: [Function: U64] }
  287. [1] },
  288. [1] 'moderator_id'
  289. [1] 'rationale' => [String (Text): 'Irrelevant as posted in another thread.'] {
  290. */
  291. //const mod = await fetchMemberByAccount(api, moderation)
  292. //if (mod) thread.setModeration(mod)
  293. }
  294. if (id > 1) fetchThread(api, id - 1)
  295. return thread
  296. }
  297. const fetchCouncils = async (api: Api, lastBlock: number) => {
  298. const round = await api.query.councilElection.round()
  299. let councils: CouncilType[] = await Council.findAll()
  300. const cycle = 201600
  301. for (let round = 0; round < round; round++) {
  302. const block = 57601 + round * cycle
  303. if (councils.find((c) => c.round === round) || block > lastBlock) continue
  304. //enqueue(() => fetchCouncil(api, block))
  305. }
  306. }
  307. const fetchCouncil = async (api: Api, block: number) => {
  308. console.debug(`Fetching council at block ${block}`)
  309. const blockHash = await api.rpc.chain.getBlockHash(block)
  310. if (!blockHash)
  311. return console.error(`Error: empty blockHash fetchCouncil ${block}`)
  312. const council = await api.query.council.activeCouncil.at(blockHash)
  313. return save('council', council)
  314. }
  315. const fetchProposal = async (api: Api, id: number) => {
  316. const exists = await Proposal.findByPk(id)
  317. if (exists) return exists
  318. //if (exists && exists.stage === 'Finalized')
  319. //if (exists.votesByAccount && exists.votesByAccount.length) return
  320. //else return //TODO fetchVotesPerProposal(api, exists)
  321. console.debug(`Fetching proposal ${id}`)
  322. const proposal = await get.proposalDetail(api, id)
  323. if (id > 1) fetchProposal(api, id - 1)
  324. return save('proposal', proposal)
  325. //TODO fetchVotesPerProposal(api, proposal)
  326. }
  327. const fetchVotesPerProposal = async (api: Api, proposal: ProposalDetail) => {
  328. if (proposal.votesByAccount && proposal.votesByAccount.length) return
  329. const proposals = await Proposal.findAll()
  330. const councils = await Council.findAll()
  331. console.debug(`Fetching proposal votes (${proposal.id})`)
  332. let members: MemberType[] = []
  333. councils.map((seats: Seat[]) =>
  334. seats.forEach(async (seat: Seat) => {
  335. if (members.find((member) => member.account === seat.member)) return
  336. const member = await Member.findOne({ where: { account: seat.member } })
  337. member && members.push(member)
  338. })
  339. )
  340. const { id } = proposal
  341. const votesByAccount = await Promise.all(
  342. members.map(async (member) => {
  343. const vote = await fetchVoteByProposalByVoter(api, id, member.id)
  344. return { vote, handle: member.handle }
  345. })
  346. )
  347. Proposal.findByPk(id).then((p: any) => p.update({ votesByAccount }))
  348. }
  349. const fetchVoteByProposalByVoter = async (
  350. api: Api,
  351. proposalId: number,
  352. voterId: number
  353. ): Promise<string> => {
  354. console.debug(`Fetching vote by ${voterId} for proposal ${proposalId}`)
  355. const vote: VoteKind = await api.query.proposalsEngine.voteExistsByProposalByVoter(
  356. proposalId,
  357. voterId
  358. )
  359. const hasVoted: number = (
  360. await api.query.proposalsEngine.voteExistsByProposalByVoter.size(
  361. proposalId,
  362. voterId
  363. )
  364. ).toNumber()
  365. return hasVoted ? String(vote) : ''
  366. }
  367. // accounts
  368. const fetchMemberByAccount = async (
  369. api: Api,
  370. account: string
  371. ): Promise<MemberType | undefined> => {
  372. const exists = await Member.findOne({ where: { account } })
  373. if (exists) return exists
  374. const id: number = Number(await get.memberIdByAccount(api, account))
  375. return id ? fetchMember(api, id) : undefined
  376. }
  377. const fetchMember = async (api: Api, id: number): Promise<MemberType> => {
  378. try {
  379. const exists = await Member.findByPk(id)
  380. if (exists) return exists
  381. } catch (e) {
  382. console.debug(`Fetching member ${id}`)
  383. }
  384. const membership = await get.membership(api, id)
  385. const handle = String(membership.handle)
  386. const account = String(membership.root_account)
  387. const about = String(membership.about)
  388. const createdAt = +membership.registered_at_block
  389. if (id > 1) fetchMember(api, id - 1)
  390. return save('member', { id, handle, createdAt, about })
  391. }
  392. const fetchReports = () => {
  393. const domain = `https://raw.githubusercontent.com/Joystream/community-repo/master/council-reports`
  394. const apiBase = `https://api.github.com/repos/joystream/community-repo/contents/council-reports`
  395. const urls: { [key: string]: string } = {
  396. alexandria: `${apiBase}/alexandria-testnet`,
  397. archive: `${apiBase}/archived-reports`,
  398. template: `${domain}/templates/council_report_template_v1.md`,
  399. }
  400. ;['alexandria', 'archive'].map((folder) => fetchGithubDir(urls[folder]))
  401. fetchGithubFile(urls.template)
  402. }
  403. const fetchGithubFile = async (url: string): Promise<string> => {
  404. const { data } = await axios.get(url)
  405. return data
  406. }
  407. const fetchGithubDir = async (url: string) => {
  408. const { data } = await axios.get(url)
  409. data.forEach(
  410. async (o: {
  411. name: string
  412. type: string
  413. url: string
  414. download_url: string
  415. }) => {
  416. const match = o.name.match(/^(.+)\.md$/)
  417. const name = match ? match[1] : o.name
  418. if (o.type === 'file') {
  419. const file = await fetchGithubFile(o.download_url)
  420. // TODO save file
  421. } else fetchGithubDir(o.url)
  422. }
  423. )
  424. }
  425. module.exports = { addBlock }