validator-logging/server/joystream/index.js

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __generator = (this && this.__generator) || function (thisArg, body) {
    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
    function verb(n) { return function (v) { return step([n, v]); }; }
    function step(op) {
        if (f) throw new TypeError("Generator is already executing.");
        while (_) try {
            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
            if (y = 0, t) op = [op[0] & 2, t.value];
            switch (op[0]) {
                case 0: case 1: t = op; break;
                case 4: _.label++; return { value: op[1], done: false };
                case 5: _.label++; y = op[1]; op = [0]; continue;
                case 7: op = _.ops.pop(); _.trys.pop(); continue;
                default:
                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
                    if (t[2]) _.ops.pop();
                    _.trys.pop(); continue;
            }
            op = body.call(thisArg, _);
        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
    }
};
exports.__esModule = true;
var sequelize_1 = require("sequelize");
var models_1 = require("../db/models");
var get = require("./lib/getters");
//import {fetchReports} from './lib/github'
var axios_1 = require("axios");
var moment_1 = require("moment");
var chalk_1 = require("chalk");
// TODO fetch consts from db/chain
var TERMDURATION = 144000;
var VOTINGDURATION = 57601;
var CYCLE = VOTINGDURATION + TERMDURATION;
var DELAY = 0; // ms
var lastUpdate = 0;
var queuedAll = false;
var queue = [];
var processing = '';
var busy = false;
var processNext = function () { return __awaiter(void 0, void 0, void 0, function () {
    var task, result;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (busy)
                    return [2 /*return*/];
                task = queue.shift();
                if (!task)
                    return [2 /*return*/];
                return [4 /*yield*/, task()];
            case 1:
                result = _a.sent();
                busy = false;
                setTimeout(function () { return processNext(); }, DELAY);
                return [2 /*return*/];
        }
    });
}); };
var getBlockHash = function (api, blockId) {
    return api.rpc.chain.getBlockHash(blockId).then(function (array) { return array.toHuman(); });
};
var getEraAtHash = function (api, hash) {
    return api.query.staking.activeEra
        .at(hash)
        .then(function (era) { return era.unwrap().index.toNumber(); });
};
var getEraAtBlock = function (api, block) { return __awaiter(void 0, void 0, void 0, function () { var _a, _b; return __generator(this, function (_c) {
    switch (_c.label) {
        case 0:
            _a = getEraAtHash;
            _b = [api];
            return [4 /*yield*/, getBlockHash(api, block)];
        case 1: return [2 /*return*/, _a.apply(void 0, _b.concat([_c.sent()]))];
    }
}); }); };
var getTimestamp = function (api, hash) { return __awaiter(void 0, void 0, void 0, function () {
    var timestamp, _a;
    return __generator(this, function (_b) {
        switch (_b.label) {
            case 0:
                if (!hash) return [3 /*break*/, 2];
                return [4 /*yield*/, api.query.timestamp.now.at(hash)];
            case 1:
                _a = _b.sent();
                return [3 /*break*/, 4];
            case 2: return [4 /*yield*/, api.query.timestamp.now()];
            case 3:
                _a = _b.sent();
                _b.label = 4;
            case 4:
                timestamp = _a;
                return [2 /*return*/, moment_1["default"].utc(timestamp.toNumber()).valueOf()];
        }
    });
}); };
var findCouncilAtBlock = function (api, block) {
    var _a, _b;
    return models_1.Council.findOne({
        where: {
            start: (_a = {}, _a[sequelize_1.Op.lte] = block, _a),
            end: (_b = {}, _b[sequelize_1.Op.gte] = block - VOTINGDURATION, _b)
        }
    });
};
var addBlock = function (api, io, header, status) {
    if (status === void 0) { status = {
        block: 0,
        era: 0,
        round: 0,
        members: 0,
        channels: 0,
        categories: 0,
        threads: 0,
        posts: 0,
        proposals: 0,
        proposalPosts: 0
    }; }
    return __awaiter(void 0, void 0, void 0, function () {
        var id, exists, block, key, account, _a, _b, _c, handle, q;
        var _d;
        return __generator(this, function (_e) {
            switch (_e.label) {
                case 0:
                    id = +header.number;
                    return [4 /*yield*/, models_1.Block.findByPk(id)];
                case 1:
                    exists = _e.sent();
                    if (exists) {
                        console.error("TODO handle fork", String(header.author));
                        return [2 /*return*/, status];
                    }
                    return [4 /*yield*/, processBlock(api, id)];
                case 2:
                    block = _e.sent();
                    key = (_d = header.author) === null || _d === void 0 ? void 0 : _d.toString();
                    return [4 /*yield*/, models_1.Account.findOrCreate({ where: { key: key } })];
                case 3:
                    account = (_e.sent())[0];
                    return [4 /*yield*/, block.setValidator(account.key)
                        //account.addBlock(block.id) // TODO needed?
                    ];
                case 4:
                    _e.sent();
                    //account.addBlock(block.id) // TODO needed?
                    _b = (_a = io).emit;
                    _c = ['block'];
                    return [4 /*yield*/, models_1.Block.findByIdWithIncludes(id)];
                case 5:
                    //account.addBlock(block.id) // TODO needed?
                    _b.apply(_a, _c.concat([_e.sent()]));
                    return [4 /*yield*/, getHandleOrKey(api, key)];
                case 6:
                    handle = _e.sent();
                    q = queue.length ? chalk_1["default"].green(" [" + queue.length + ":" + processing + "]") : '';
                    console.log("[Joystream] block " + id + " " + handle + " " + q);
                    return [2 /*return*/, updateStatus(api, status, id)];
            }
        });
    });
};
var processBlock = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, last, lastBlockTimestamp, lastBlockHash, block, _a, currentBlockTimestamp;
    return __generator(this, function (_b) {
        switch (_b.label) {
            case 0: return [4 /*yield*/, models_1.Block.findByPk(id)];
            case 1:
                exists = _b.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "block " + id;
                return [4 /*yield*/, models_1.Block.findByPk(id - 1)];
            case 2:
                last = _b.sent();
                if (!last) return [3 /*break*/, 3];
                lastBlockTimestamp = last.timestamp.getTime();
                return [3 /*break*/, 6];
            case 3: return [4 /*yield*/, getBlockHash(api, id - 1)];
            case 4:
                lastBlockHash = _b.sent();
                return [4 /*yield*/, getTimestamp(api, lastBlockHash)];
            case 5:
                lastBlockTimestamp = _b.sent();
                _b.label = 6;
            case 6: return [4 /*yield*/, models_1.Block.findOrCreate({ where: { id: id } })];
            case 7:
                block = (_b.sent())[0];
                _a = block;
                return [4 /*yield*/, getBlockHash(api, id)];
            case 8:
                _a.hash = _b.sent();
                return [4 /*yield*/, getTimestamp(api, block.hash)];
            case 9:
                currentBlockTimestamp = _b.sent();
                block.timestamp = new Date(currentBlockTimestamp);
                block.blocktime = (currentBlockTimestamp - lastBlockTimestamp);
                block.save();
                processEvents(api, id, block.hash);
                return [4 /*yield*/, importEraAtBlock(api, id, block.hash)];
            case 10:
                _b.sent();
                return [2 /*return*/, block];
        }
    });
}); };
var addBlockRange = function (api, startBlock, endBlock) { return __awaiter(void 0, void 0, void 0, function () {
    var _loop_1, block;
    return __generator(this, function (_a) {
        _loop_1 = function (block) {
            queue.push(function () { return processBlock(api, block); });
        };
        for (block = startBlock; block <= endBlock; block++) {
            _loop_1(block);
        }
        return [2 /*return*/];
    });
}); };
var updateStatus = function (api, old, block) { return __awaiter(void 0, void 0, void 0, function () {
    var status, _a;
    var _b;
    return __generator(this, function (_c) {
        switch (_c.label) {
            case 0: return [4 /*yield*/, getEraAtBlock(api, block)];
            case 1:
                _a = Number;
                return [4 /*yield*/, api.query.councilElection.round()];
            case 2: return [4 /*yield*/, api.query.members.nextMemberId()];
            case 3: return [4 /*yield*/, get.currentChannelId(api)];
            case 4: return [4 /*yield*/, get.currentCategoryId(api)];
            case 5: return [4 /*yield*/, get.currentThreadId(api)];
            case 6: return [4 /*yield*/, get.currentPostId(api)];
            case 7: return [4 /*yield*/, get.proposalCount(api)];
            case 8: return [4 /*yield*/, api.query.proposalsDiscussion.postCount()];
            case 9:
                status = (_b = {
                    block: block
                },
                    _b.era = _c.sent(),
                    _b.round = _a.apply(void 0, [_c.sent()]),
                    _b.members = (_c.sent()) - 1,
                    _b.channels = _c.sent(),
                    _b.categories = _c.sent(),
                    _b.threads = _c.sent(),
                    _b.posts = _c.sent(),
                    _b.proposals = _c.sent(),
                    _b.proposalPosts = (_c.sent()).toHuman(),
                    _b);
                if (!queuedAll)
                    fetchAll(api, status);
                else {
                    // TODO catch if more than one are added
                    status.members > old.members && fetchMember(api, status.members);
                    status.posts > old.posts && fetchPost(api, status.posts);
                    status.proposals > old.proposals && fetchProposal(api, status.proposals);
                    status.channels > old.channels && fetchChannel(api, status.channels);
                    status.categories > old.categories && fetchCategory(api, status.categories);
                    status.proposalPosts > old.proposalPosts &&
                        fetchProposalPosts(api, status.proposalPosts);
                }
                return [2 /*return*/, status];
        }
    });
}); };
var fetchAll = function (api, status) { return __awaiter(void 0, void 0, void 0, function () {
    var _loop_2, id, _loop_3, id, _loop_4, id, _loop_5, id, _loop_6, id, _loop_7, id, _loop_8, id;
    return __generator(this, function (_a) {
        queue.push(function () { return fetchAccounts(api, status.block); });
        _loop_2 = function (id) {
            queue.push(function () { return fetchMember(api, id); });
        };
        for (id = status.members; id > 0; id--) {
            _loop_2(id);
        }
        _loop_3 = function (id) {
            queue.push(function () { return fetchCouncil(api, id); });
        };
        for (id = status.round; id > 0; id--) {
            _loop_3(id);
        }
        _loop_4 = function (id) {
            queue.push(function () { return fetchProposal(api, id); });
        };
        for (id = status.proposals; id > 0; id--) {
            _loop_4(id);
        }
        _loop_5 = function (id) {
            queue.push(function () { return fetchChannel(api, id); });
        };
        for (id = status.channels; id > 0; id--) {
            _loop_5(id);
        }
        _loop_6 = function (id) {
            queue.push(function () { return fetchCategory(api, id); });
        };
        for (id = status.categories; id > 0; id--) {
            _loop_6(id);
        }
        _loop_7 = function (id) {
            queue.push(function () { return fetchThread(api, id); });
        };
        for (id = status.threads; id > 0; id--) {
            _loop_7(id);
        }
        _loop_8 = function (id) {
            queue.push(function () { return fetchPost(api, id); });
        };
        for (id = status.posts; id > 0; id--) {
            _loop_8(id);
        }
        queue.push(function () { return fetchProposalPosts(api, status.proposalPosts); });
        queue.push(function () { return addBlockRange(api, 1, status.block); });
        queuedAll = true;
        processNext();
        return [2 /*return*/];
    });
}); };
var processEvents = function (api, blockId, hash) { return __awaiter(void 0, void 0, void 0, function () {
    var blockEvents, e_1;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                processing = "events block " + blockId;
                _a.label = 1;
            case 1:
                _a.trys.push([1, 3, , 4]);
                return [4 /*yield*/, api.query.system.events.at(hash)];
            case 2:
                blockEvents = _a.sent();
                blockEvents.forEach(function (_a) {
                    var event = _a.event;
                    var section = event.section, method = event.method, data = event.data;
                    models_1.Event.create({ blockId: blockId, section: section, method: method, data: JSON.stringify(data) });
                });
                return [3 /*break*/, 4];
            case 3:
                e_1 = _a.sent();
                console.log("failed to fetch events for block  " + blockId + " " + hash);
                return [3 /*break*/, 4];
            case 4: return [2 /*return*/];
        }
    });
}); };
var fetchValidators = function (api, hash) { return __awaiter(void 0, void 0, void 0, function () { return __generator(this, function (_a) {
    return [2 /*return*/, api.query.staking.snapshotValidators.at(hash)];
}); }); };
var importEraAtBlock = function (api, blockId, hash) { return __awaiter(void 0, void 0, void 0, function () {
    var id, era;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0: return [4 /*yield*/, getEraAtHash(api, hash)];
            case 1:
                id = _a.sent();
                return [4 /*yield*/, models_1.Era.findOrCreate({ where: { id: id } })];
            case 2:
                era = (_a.sent())[0];
                era.addBlock(blockId);
                if (era.active)
                    return [2 /*return*/];
                processing = "era " + id;
                try {
                    fetchValidators(api, hash).then(function (snapshot) { return __awaiter(void 0, void 0, void 0, function () {
                        var validatorCount, _a, _b, chainTimestamp;
                        return __generator(this, function (_c) {
                            switch (_c.label) {
                                case 0:
                                    if (snapshot.isEmpty)
                                        return [2 /*return*/];
                                    console.log("[Joystream] Found validator info for era " + id);
                                    validatorCount = snapshot.unwrap().length;
                                    _a = era;
                                    return [4 /*yield*/, api.query.staking.validatorCount.at(hash)];
                                case 1:
                                    _a.slots = (_c.sent()).toNumber();
                                    era.active = Math.min(era.slots, validatorCount);
                                    era.waiting =
                                        validatorCount > era.slots ? validatorCount - era.slots : 0;
                                    _b = era;
                                    return [4 /*yield*/, api.query.staking.erasTotalStake.at(hash, id)];
                                case 2:
                                    _b.stake = _c.sent();
                                    return [4 /*yield*/, api.query.timestamp.now.at(hash)];
                                case 3:
                                    chainTimestamp = (_c.sent());
                                    era.timestamp = moment_1["default"](chainTimestamp.toNumber());
                                    // era.update({ slots, active, waiting, stake, timestamp })
                                    era.blockId = id;
                                    era.save();
                                    updateBalances(api, hash);
                                    return [2 /*return*/];
                            }
                        });
                    }); });
                }
                catch (e) {
                    console.error("import era " + blockId + " " + hash, e);
                }
                return [2 /*return*/];
        }
    });
}); };
var validatorStatus = function (api, blockId) { return __awaiter(void 0, void 0, void 0, function () {
    var hash, totalValidators, totalNrValidators, maxSlots, _a, actives, waiting, timestamp, date;
    return __generator(this, function (_b) {
        switch (_b.label) {
            case 0: return [4 /*yield*/, getBlockHash(api, blockId)];
            case 1:
                hash = _b.sent();
                return [4 /*yield*/, api.query.staking.snapshotValidators.at(hash)];
            case 2:
                totalValidators = _b.sent();
                if (totalValidators.isEmpty)
                    return [2 /*return*/];
                totalNrValidators = totalValidators.unwrap().length;
                _a = Number;
                return [4 /*yield*/, api.query.staking.validatorCount.at(hash)];
            case 3:
                maxSlots = _a.apply(void 0, [_b.sent()]);
                actives = Math.min(maxSlots, totalNrValidators);
                waiting = totalNrValidators > maxSlots ? totalNrValidators - maxSlots : 0;
                return [4 /*yield*/, api.query.timestamp.now.at(hash)];
            case 4:
                timestamp = _b.sent();
                date = moment_1["default"](timestamp.toNumber()).valueOf();
                return [2 /*return*/, { blockId: blockId, actives: actives, waiting: waiting, maxSlots: maxSlots, date: date }];
        }
    });
}); };
var updateBalances = function (api, blockHash) { return __awaiter(void 0, void 0, void 0, function () {
    var currentEra, era;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0: return [4 /*yield*/, api.query.staking.currentEra.at(blockHash)];
            case 1:
                currentEra = _a.sent();
                return [4 /*yield*/, models_1.Era.findOrCreate({ where: { id: currentEra } })];
            case 2:
                era = _a.sent();
                try {
                    processing = "balances " + era;
                    models_1.Account.findAll().then(function (account) { return __awaiter(void 0, void 0, void 0, function () {
                        var key, data, free, reserved, miscFrozen, feeFrozen, balance;
                        return __generator(this, function (_a) {
                            switch (_a.label) {
                                case 0:
                                    key = account.key;
                                    if (!key)
                                        return [2 /*return*/];
                                    console.log("updating balance of", key, key);
                                    return [4 /*yield*/, getAccountAtBlock(api, blockHash, key)];
                                case 1:
                                    data = (_a.sent()).data;
                                    free = data.free, reserved = data.reserved, miscFrozen = data.miscFrozen, feeFrozen = data.feeFrozen;
                                    balance = { available: free, reserved: reserved, frozen: miscFrozen };
                                    console.log("balance " + era, balance);
                                    models_1.Balance.create(balance).then(function (balance) {
                                        balance.setAccount(key);
                                        balance.setEra(era.id);
                                        console.log("balance", era.id, key, balance.available);
                                    });
                                    return [2 /*return*/];
                            }
                        });
                    }); });
                }
                catch (e) {
                    console.error("balances era " + era);
                }
                return [2 /*return*/];
        }
    });
}); };
var fetchTokenomics = function () { return __awaiter(void 0, void 0, void 0, function () {
    var data;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                console.debug("Updating tokenomics");
                return [4 /*yield*/, axios_1["default"].get('https://status.joystream.org/status')];
            case 1:
                data = (_a.sent()).data;
                if (!data)
                    return [2 /*return*/];
                return [2 /*return*/];
        }
    });
}); };
var fetchChannel = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, data, handle, title, description, avatar, banner, content, created, channel, chan, owner;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Channel.findByPk(id)];
            case 1:
                exists = _a.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "channel " + id;
                return [4 /*yield*/, api.query.contentWorkingGroup.channelById(id)];
            case 2:
                data = _a.sent();
                handle = data.handle, title = data.title, description = data.description, avatar = data.avatar, banner = data.banner, content = data.content, created = data.created;
                channel = {
                    id: id,
                    handle: String(handle),
                    title: String(title),
                    description: String(description),
                    avatar: String(avatar),
                    banner: String(banner),
                    content: String(content),
                    publicationStatus: data.publication_status === 'Public' ? true : false,
                    curation: String(data.curation_status),
                    createdAt: +created,
                    principal: Number(data.principal_id)
                };
                return [4 /*yield*/, models_1.Channel.create(channel)];
            case 3:
                chan = _a.sent();
                return [4 /*yield*/, fetchMember(api, data.owner)];
            case 4:
                owner = _a.sent();
                chan.setOwner(owner);
                return [2 /*return*/, chan];
        }
    });
}); };
var fetchCategory = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, data, title, description, deleted, archived, category;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Category.findByPk(+id)];
            case 1:
                exists = _a.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "category " + id;
                return [4 /*yield*/, api.query.forum.categoryById(id)];
            case 2:
                data = _a.sent();
                title = data.title, description = data.description, deleted = data.deleted, archived = data.archived;
                return [4 /*yield*/, models_1.Category.create({
                        id: id,
                        title: title,
                        threadId: +data.thread_id,
                        description: description,
                        createdAt: +data.created_at.block,
                        deleted: deleted,
                        archived: archived,
                        subcategories: Number(data.num_direct_subcategories),
                        moderatedThreads: Number(data.num_direct_moderated_threads),
                        unmoderatedThreads: Number(data.num_direct_unmoderated_threads)
                    })];
            case 3:
                category = _a.sent();
                createModeration(api, { categoryId: id }, String(data.moderator_id), category);
                return [2 /*return*/, category];
        }
    });
}); };
var fetchPost = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, data, author, member, authorId, threadId, thread, text, history, createdAt, post;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Post.findByPk(id)];
            case 1:
                exists = _a.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "post " + id;
                return [4 /*yield*/, api.query.forum.postById(id)];
            case 2:
                data = _a.sent();
                author = String(data.author_id);
                return [4 /*yield*/, fetchMemberByAccount(api, author)];
            case 3:
                member = _a.sent();
                authorId = member ? member.id : null;
                threadId = Number(data.thread_id);
                return [4 /*yield*/, fetchThread(api, threadId)];
            case 4:
                thread = _a.sent();
                text = data.current_text;
                history = data.text_change_history // TODO needed?
                ;
                createdAt = data.created_at.block;
                return [4 /*yield*/, models_1.Post.create({ id: id, authorId: authorId, text: text, createdAt: createdAt, threadId: threadId })];
            case 5:
                post = _a.sent();
                if (data.moderation)
                    createModeration(api, { postId: id }, data.moderation, post);
                return [2 /*return*/, post];
        }
    });
}); };
var createModeration = function (api, where, key, object) { return __awaiter(void 0, void 0, void 0, function () {
    var moderation;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (key === '')
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Account.findOrCreate({ where: { key: key } })];
            case 1:
                _a.sent();
                return [4 /*yield*/, models_1.Moderation.create({ moderatorKey: key })];
            case 2:
                moderation = _a.sent();
                object.setModeration(moderation.id);
                return [2 /*return*/, moderation];
        }
    });
}); };
var fetchThread = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, data, title, moderation, nr_in_category, account, t, thread, category, author, moderated_at, moderator_id, rationale, created, createdAt;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Thread.findByPk(id)];
            case 1:
                exists = _a.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "thread " + id;
                return [4 /*yield*/, api.query.forum.threadById(id)];
            case 2:
                data = _a.sent();
                title = data.title, moderation = data.moderation, nr_in_category = data.nr_in_category;
                account = String(data.author_id);
                t = {
                    id: id,
                    title: title,
                    nrInCategory: +nr_in_category,
                    createdAt: +data.created_at.block
                };
                return [4 /*yield*/, models_1.Thread.create(t)];
            case 3:
                thread = _a.sent();
                return [4 /*yield*/, fetchCategory(api, +data.category_id)];
            case 4:
                category = _a.sent();
                if (category)
                    thread.setCategory(category.id);
                return [4 /*yield*/, fetchMemberByAccount(api, account)];
            case 5:
                author = _a.sent();
                if (author)
                    thread.setCreator(author.id);
                if (moderation) {
                    moderated_at = moderation.moderated_at, moderator_id = moderation.moderator_id, rationale = moderation.rationale;
                    created = moderated_at.block;
                    createdAt = moment_1["default"].utc(moderated_at.time);
                    createModeration(api, { created: created, createdAt: createdAt, rationale: rationale }, moderator_id.toHuman(), thread);
                }
                return [2 /*return*/, thread];
        }
    });
}); };
var fetchCouncil = function (api, round) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, start, end, council, seats, startHash, _a, e_2, endHash, _b, e_3;
    return __generator(this, function (_c) {
        switch (_c.label) {
            case 0:
                if (round <= 0)
                    return [2 /*return*/, console.log(chalk_1["default"].red("[fetchCouncil] round:" + round))];
                return [4 /*yield*/, models_1.Council.findByPk(round)];
            case 1:
                exists = _c.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "council " + round;
                start = 57601 + (round - 1) * CYCLE;
                end = start + TERMDURATION;
                council = { round: round, start: start, end: end, startDate: 0, endDate: 0 };
                _c.label = 2;
            case 2:
                _c.trys.push([2, 6, , 7]);
                return [4 /*yield*/, getBlockHash(api, start)];
            case 3:
                startHash = _c.sent();
                _a = council;
                return [4 /*yield*/, getTimestamp(api, startHash)];
            case 4:
                _a.startDate = _c.sent();
                return [4 /*yield*/, api.query.council.activeCouncil.at(startHash)];
            case 5:
                seats = _c.sent();
                return [3 /*break*/, 7];
            case 6:
                e_2 = _c.sent();
                return [2 /*return*/, console.log("council term " + round + " lies in the future " + start)];
            case 7:
                _c.trys.push([7, 10, , 11]);
                return [4 /*yield*/, getBlockHash(api, end)];
            case 8:
                endHash = _c.sent();
                _b = council;
                return [4 /*yield*/, getTimestamp(api, endHash)];
            case 9:
                _b.endDate = _c.sent();
                return [3 /*break*/, 11];
            case 10:
                e_3 = _c.sent();
                console.warn("end of council term " + round + " lies in the future " + end);
                return [3 /*break*/, 11];
            case 11:
                try {
                    models_1.Council.create(council).then(function (_a) {
                        var round = _a.round;
                        return seats.map(function (_a) {
                            var member = _a.member, stake = _a.stake, backers = _a.backers;
                            return fetchMemberByAccount(api, member.toHuman()).then(function (_a) {
                                var id = _a.id;
                                return models_1.Consul.create({
                                    stake: Number(stake),
                                    councilRound: round,
                                    memberId: id
                                }).then(function (consul) {
                                    return backers.map(function (_a) {
                                        var member = _a.member, stake = _a.stake;
                                        return __awaiter(void 0, void 0, void 0, function () {
                                            return __generator(this, function (_b) {
                                                return [2 /*return*/, fetchMemberByAccount(api, member.toHuman()).then(function (_a) {
                                                        var id = _a.id;
                                                        return models_1.ConsulStake.create({
                                                            stake: Number(stake),
                                                            consulId: consul.id,
                                                            memberId: id
                                                        });
                                                    })];
                                            });
                                        });
                                    });
                                });
                            });
                        });
                    });
                }
                catch (e) {
                    console.error("Failed to save council " + round, e);
                }
                return [2 /*return*/];
        }
    });
}); };
var fetchProposal = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, proposal;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Proposal.findByPk(+id)];
            case 1:
                exists = _a.sent();
                if (exists) {
                    fetchProposalVotes(api, exists);
                    return [2 /*return*/, exists];
                }
                processing = "proposal " + id;
                return [4 /*yield*/, get.proposalDetail(api, id)];
            case 2:
                proposal = _a.sent();
                return [4 /*yield*/, fetchMember(api, proposal.authorId)];
            case 3:
                _a.sent();
                fetchProposalVotes(api, proposal);
                return [2 /*return*/, models_1.Proposal.create(proposal)];
        }
    });
}); };
var fetchProposalPost = function (api, threadId, postId) {
    return api.query.proposalsDiscussion.postThreadIdByPostId(threadId, postId);
};
var fetchProposalPosts = function (api, posts) { return __awaiter(void 0, void 0, void 0, function () {
    var threads, proposalId, _loop_9, out_id_1, id;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0: return [4 /*yield*/, api.query.proposalsDiscussion.threadCount()];
            case 1:
                threads = (_a.sent()).toNumber();
                proposalId = 1;
                _loop_9 = function (id) {
                    var exists, post, proposal;
                    return __generator(this, function (_a) {
                        switch (_a.label) {
                            case 0: return [4 /*yield*/, models_1.ProposalPost.findByPk(id)];
                            case 1:
                                exists = _a.sent();
                                if (exists) {
                                    id++;
                                    proposalId = 1;
                                    return [2 /*return*/, (out_id_1 = id, "continue")];
                                }
                                processing = "proposal post " + id + "/" + posts + " " + proposalId + "/" + threads;
                                return [4 /*yield*/, fetchProposalPost(api, proposalId, id)];
                            case 2:
                                post = _a.sent();
                                if (!post.text.length) {
                                    proposalId++;
                                    return [2 /*return*/, (out_id_1 = id, "continue")];
                                }
                                return [4 /*yield*/, models_1.Proposal.findByPk(proposalId)];
                            case 3:
                                proposal = _a.sent();
                                if (!proposal) {
                                    console.warn("[fetchProposalPosts] proposal " + proposalId + " not found.");
                                    id++;
                                    return [2 /*return*/, (out_id_1 = id, "continue")];
                                }
                                models_1.ProposalPost.create({
                                    id: id,
                                    text: post.text.toHuman(),
                                    created: Number(post.created_at),
                                    updated: Number(post.updated_at),
                                    edition: Number(post.edition_number),
                                    authorId: Number(post.author_id)
                                }).then(function (p) { return proposal.addPost(p); });
                                id++;
                                proposalId = 1;
                                out_id_1 = id;
                                return [2 /*return*/];
                        }
                    });
                };
                id = 1;
                _a.label = 2;
            case 2:
                if (!(id <= posts && proposalId <= threads)) return [3 /*break*/, 5];
                return [5 /*yield**/, _loop_9(id)];
            case 3:
                _a.sent();
                id = out_id_1;
                _a.label = 4;
            case 4: return [3 /*break*/, 2];
            case 5: return [2 /*return*/];
        }
    });
}); };
var fetchProposalVotes = function (api, proposal) { return __awaiter(void 0, void 0, void 0, function () {
    var createdAt, finalizedAt, start, _a, end, _b, councils, consuls, e_4;
    var _c;
    return __generator(this, function (_d) {
        switch (_d.label) {
            case 0:
                if (!proposal)
                    return [2 /*return*/, console.error("[fetchProposalVotes] empty proposal")];
                processing = "votes proposal " + proposal.id;
                createdAt = proposal.createdAt, finalizedAt = proposal.finalizedAt;
                _d.label = 1;
            case 1:
                _d.trys.push([1, 9, , 10]);
                if (!createdAt) return [3 /*break*/, 3];
                return [4 /*yield*/, findCouncilAtBlock(api, createdAt)];
            case 2:
                _a = _d.sent();
                return [3 /*break*/, 4];
            case 3:
                _a = null;
                _d.label = 4;
            case 4:
                start = _a;
                if (start)
                    start.addProposal(proposal.id);
                else
                    return [2 /*return*/, console.error("[fetchProposalVotes] no council found for proposal " + proposal.id)
                        // some proposals make it into a second term
                    ];
                if (!finalizedAt) return [3 /*break*/, 6];
                return [4 /*yield*/, findCouncilAtBlock(api, finalizedAt)];
            case 5:
                _b = _d.sent();
                return [3 /*break*/, 7];
            case 6:
                _b = null;
                _d.label = 7;
            case 7:
                end = _b;
                councils = [start && start.round, end && end.round];
                return [4 /*yield*/, models_1.Consul.findAll({
                        where: { councilRound: (_c = {}, _c[sequelize_1.Op.or] = councils, _c) }
                    })];
            case 8:
                consuls = _d.sent();
                consuls.map(function (_a) {
                    var id = _a.id, memberId = _a.memberId;
                    return fetchProposalVoteByConsul(api, proposal.id, id, memberId);
                });
                return [3 /*break*/, 10];
            case 9:
                e_4 = _d.sent();
                console.log("failed to fetch votes of proposal " + proposal.id, e_4);
                return [3 /*break*/, 10];
            case 10: return [2 /*return*/];
        }
    });
}); };
var fetchProposalVoteByConsul = function (api, proposalId, consulId, memberId) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, query, args, hasVoted, vote;
    var _a;
    return __generator(this, function (_b) {
        switch (_b.label) {
            case 0:
                processing = "vote by " + consulId + " for proposal " + proposalId;
                return [4 /*yield*/, models_1.ProposalVote.findOne({
                        where: { proposalId: proposalId, memberId: memberId, consulId: consulId }
                    })];
            case 1:
                exists = _b.sent();
                if (exists)
                    return [2 /*return*/, exists];
                query = api.query.proposalsEngine;
                args = [proposalId, memberId];
                return [4 /*yield*/, (_a = query.voteExistsByProposalByVoter).size.apply(_a, args)];
            case 2:
                hasVoted = _b.sent();
                if (!hasVoted.toNumber())
                    return [2 /*return*/];
                return [4 /*yield*/, query.voteExistsByProposalByVoter.apply(query, args)];
            case 3:
                vote = (_b.sent()).toHuman();
                return [4 /*yield*/, fetchMember(api, memberId)]; // TODO needed?
            case 4:
                _b.sent(); // TODO needed?
                return [2 /*return*/, models_1.ProposalVote.create({ vote: vote, proposalId: proposalId, consulId: consulId, memberId: memberId })];
        }
    });
}); };
// accounts
var getHandleOrKey = function (api, key) { return __awaiter(void 0, void 0, void 0, function () {
    var member;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0: return [4 /*yield*/, fetchMemberByAccount(api, key)];
            case 1:
                member = _a.sent();
                return [2 /*return*/, member ? member.handle : key]; //abbrKey(key)
        }
    });
}); };
var abbrKey = function (key) {
    return key.slice(0, 5) + ".." + key.slice(key.length - 5);
};
var getAccountAtBlock = function (api, hash, account) { return api.query.system.account.at(hash, account); };
var fetchAccounts = function (api, blockId) { return __awaiter(void 0, void 0, void 0, function () {
    return __generator(this, function (_a) {
        processing = "accounts";
        api.query.system.account
            .entries()
            .then(function (account) {
            return models_1.Account.findOrCreate({ where: { key: account[0][0].toHuman()[0] } });
        });
        return [2 /*return*/];
    });
}); };
var fetchMemberByAccount = function (api, rootKey) { return __awaiter(void 0, void 0, void 0, function () {
    var member, id, _a;
    return __generator(this, function (_b) {
        switch (_b.label) {
            case 0: return [4 /*yield*/, models_1.Member.findOne({ where: { rootKey: rootKey } })];
            case 1:
                member = _b.sent();
                if (member)
                    return [2 /*return*/, member];
                _a = Number;
                return [4 /*yield*/, get.memberIdByAccount(api, rootKey)];
            case 2:
                id = _a.apply(void 0, [_b.sent()]);
                if (id)
                    return [2 /*return*/, fetchMember(api, id)];
                else
                    models_1.Account.findOrCreate({ where: { key: rootKey } });
                return [2 /*return*/];
        }
    });
}); };
var fetchMember = function (api, id) { return __awaiter(void 0, void 0, void 0, function () {
    var exists, membership, about, handle, createdAt, rootKey;
    return __generator(this, function (_a) {
        switch (_a.label) {
            case 0:
                if (id <= 0)
                    return [2 /*return*/];
                return [4 /*yield*/, models_1.Member.findByPk(+id)];
            case 1:
                exists = _a.sent();
                if (exists)
                    return [2 /*return*/, exists];
                processing = "member " + id;
                return [4 /*yield*/, get.membership(api, id)];
            case 2:
                membership = _a.sent();
                about = String(membership.about);
                handle = String(membership.handle);
                createdAt = +membership.registered_at_block;
                rootKey = String(membership.root_account);
                return [2 /*return*/, models_1.Member.create({ id: id, about: about, createdAt: createdAt, handle: handle, rootKey: rootKey }).then(function (member) {
                        models_1.Account.findOrCreate({ where: { key: rootKey } }).then(function (_a) {
                            var account = _a[0];
                            return account.setMember(id);
                        });
                        return member;
                    })];
        }
    });
}); };
module.exports = { addBlock: addBlock, addBlockRange: addBlockRange };