joystream/storage-node/packages/helios/bin/cli.js

#!/usr/bin/env node

const { RuntimeApi } = require('@joystream/storage-runtime-api')
const { encodeAddress } = require('@polkadot/keyring')
const { discover } = require('@joystream/service-discovery')
const axios = require('axios')
const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')

async function main () {
  const runtime = await RuntimeApi.create()
  const { api } = runtime

  // get current blockheight
  const currentHeader = await api.rpc.chain.getHeader()
  const currentHeight = currentHeader.number.toBn()

  // get all providers
  const { ids: storageProviders } = await runtime.workers.getAllProviders()
  console.log(`Found ${storageProviders.length} staked providers`)

  const storageProviderAccountInfos = await Promise.all(storageProviders.map(async (providerId) => {
    return ({
      providerId,
      info: await runtime.discovery.getAccountInfo(providerId)
    })
  }))

  // providers that have updated their account info and published ipfs id
  // considered live if the record hasn't expired yet
  const liveProviders = storageProviderAccountInfos.filter(({info}) => {
    return info && info.expires_at.gte(currentHeight)
  })

  const downProviders = storageProviderAccountInfos.filter(({info}) => {
    return info == null
  })

  const expiredTtlProviders = storageProviderAccountInfos.filter(({info}) => {
    return info && currentHeight.gte(info.expires_at)
  })

  let providersStatuses = mapInfoToStatus(liveProviders, currentHeight)
  console.log('\n== Live Providers\n', providersStatuses)

  let expiredProviderStatuses = mapInfoToStatus(expiredTtlProviders, currentHeight)
  console.log('\n== Expired Providers\n', expiredProviderStatuses)

  console.log('\n== Down Providers!\n', downProviders.map(provider => {
    return ({
      providerId: provider.providerId
    })
  }))

  // Resolve IPNS identities of providers
  console.log('\nResolving live provider API Endpoints...')
  let endpoints = await Promise.all(providersStatuses.map(async ({providerId}) => {
    try {
      let serviceInfo = await discover.discover_over_joystream_discovery_service(providerId, runtime)

      if (serviceInfo == null) {
        console.log(`provider ${providerId} has not published service information`)
        return { providerId, endpoint: null }
      }

      let info = JSON.parse(serviceInfo.serialized)
      console.log(`${providerId} -> ${info.asset.endpoint}`)
      return { providerId, endpoint: info.asset.endpoint }
    } catch (err) {
      console.log('resolve failed for id', providerId, err.message)
      return { providerId, endpoint: null }
    }
  }))

  console.log('\nChecking API Endpoints are online')
  await Promise.all(endpoints.map(async (provider) => {
    if (!provider.endpoint) {
      console.log('skipping', provider.address)
      return
    }
    const swaggerUrl = `${stripEndingSlash(provider.endpoint)}/swagger.json`
    let error
    try {
      await axios.get(swaggerUrl)
      // maybe print out api version information to detect which version of colossus is running?
      // or add anothe api endpoint for diagnostics information
    } catch (err) { error = err }
    console.log(`${provider.endpoint} - ${error ? error.message : 'OK'}`)
  }))

  let knownContentIds = await runtime.assets.getKnownContentIds()
  console.log(`\nData Directory has ${knownContentIds.length} assets`)

  // Check which providers are reporting a ready relationship for each asset
  await Promise.all(knownContentIds.map(async (contentId) => {
    let [relationshipsCount, judgement] = await assetRelationshipState(api, contentId, storageProviders)
    console.log(`${encodeAddress(contentId)} replication ${relationshipsCount}/${storageProviders.length} - ${judgement}`)
  }))

  // interesting disconnect doesn't work unless an explicit provider was created
  // for underlying api instance
  // We no longer need a connection to the chain
  api.disconnect()

  console.log(`\nChecking available assets on providers (this can take some time)...`)
  endpoints.forEach(async ({ providerId, endpoint }) => {
    if (!endpoint) { return }
    const total = knownContentIds.length
    let { found } = await countContentAvailability(knownContentIds, endpoint)
    console.log(`provider ${providerId}: has ${found} out of ${total}`)
  })
}

function mapInfoToStatus (providers, currentHeight) {
  return providers.map(({providerId, info}) => {
    if (info) {
      return {
        providerId,
        identity: info.identity.toString(),
        expiresIn: info.expires_at.sub(currentHeight).toNumber(),
        expired: currentHeight.gte(info.expires_at)
      }
    } else {
      return {
        providerId,
        identity: null,
        status: 'down'
      }
    }
  })
}

// HTTP HEAD with axios all known content ids on each provider
async function countContentAvailability (contentIds, source) {
  let content = {}
  let found = 0
  let missing = 0
  for (let i = 0; i < contentIds.length; i++) {
    const assetUrl = makeAssetUrl(contentIds[i], source)
    try {
      let info = await axios.head(assetUrl)
      content[encodeAddress(contentIds[i])] = {
        type: info.headers['content-type'],
        bytes: info.headers['content-length']
      }
      // TODO: cross check against dataobject size
      found++
    } catch (err) {
      missing++
    }
  }

  return { found, missing, content }
}

function makeAssetUrl (contentId, source) {
  source = stripEndingSlash(source)
  return `${source}/asset/v0/${encodeAddress(contentId)}`
}

async function assetRelationshipState (api, contentId, providers) {
  let dataObject = await api.query.dataDirectory.dataObjectByContentId(contentId)

  let relationshipIds = await api.query.dataObjectStorageRegistry.relationshipsByContentId(contentId)

  // how many relationships associated with active providers and in ready state
  let activeRelationships = await Promise.all(relationshipIds.map(async (id) => {
    let relationship = await api.query.dataObjectStorageRegistry.relationships(id)
    relationship = relationship.unwrap()
    // only interested in ready relationships
    if (!relationship.ready) {
      return undefined
    }
    // Does the relationship belong to an active provider ?
    return providers.find((provider) => relationship.storage_provider.eq(provider))
  }))

  return ([
    activeRelationships.filter(active => active).length,
    dataObject.unwrap().liaison_judgement
  ])
}

main()