joystream/storage-node/packages/discovery/discover.js

const axios = require('axios')
const debug = require('debug')('joystream:discovery:discover')
const stripEndingSlash = require('@joystream/storage-utils/stripEndingSlash')

const ipfs = require('ipfs-http-client')('localhost', '5001', { protocol: 'http' })
const BN = require('bn.js')
const { newExternallyControlledPromise } = require('@joystream/storage-utils/externalPromise')

/**
 * Determines if code is running in a browser by testing for the global window object
 */
function inBrowser() {
	return typeof window !== 'undefined'
}

/**
 * Map storage-provider id to a Promise of a discovery result. The purpose
 * is to avoid concurrent active discoveries for the same provider.
 */
const activeDiscoveries = {}

/**
 * Map of storage provider id to string
 * Cache of past discovery lookup results
 */
const accountInfoCache = {}

/**
 * After what period of time a cached record is considered stale, and would
 * trigger a re-discovery, but only if a query is made for the same provider.
 */
const CACHE_TTL = 60 * 60 * 1000

/**
 * Queries the ipns id (service key) of the storage provider from the blockchain.
 * If the storage provider is not registered it will return null.
 * @param {number | BN | u64} storageProviderId - the provider id to lookup
 * @param { RuntimeApi } runtimeApi - api instance to query the chain
 * @returns { Promise<string | null> } - ipns multiformat address
 */
async function getIpnsIdentity(storageProviderId, runtimeApi) {
	storageProviderId = new BN(storageProviderId)
	// lookup ipns identity from chain corresponding to storageProviderId
	const info = await runtimeApi.discovery.getAccountInfo(storageProviderId)

	if (info == null) {
		// no identity found on chain for account
		return null
	}
	return info.identity.toString()
}

/**
 * Resolves provider id to its service information.
 * Will use an IPFS HTTP gateway. If caller doesn't provide a url the default gateway on
 * the local ipfs node will be used.
 * If the storage provider is not registered it will throw an error
 * @param {number | BN | u64} storageProviderId - the provider id to lookup
 * @param {RuntimeApi} runtimeApi - api instance to query the chain
 * @param {string} gateway - optional ipfs http gateway url to perform ipfs queries
 * @returns { Promise<object> } - the published service information
 */
async function discover_over_ipfs_http_gateway(storageProviderId, runtimeApi, gateway = 'http://localhost:8080') {
	storageProviderId = new BN(storageProviderId)
	const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)

	if (!isProvider) {
		throw new Error('Cannot discover non storage providers')
	}

	const identity = await getIpnsIdentity(storageProviderId, runtimeApi)

	if (identity == null) {
		// dont waste time trying to resolve if no identity was found
		throw new Error('no identity to resolve')
	}

	gateway = stripEndingSlash(gateway)

	const url = `${gateway}/ipns/${identity}`

	const response = await axios.get(url)

	return response.data
}

/**
 * Resolves id of provider to its service information.
 * Will use the provided colossus discovery api endpoint. If no api endpoint
 * is provided it attempts to use the configured endpoints from the chain.
 * If the storage provider is not registered it will throw an error
 * @param {number | BN | u64 } storageProviderId - provider id to lookup
 * @param {RuntimeApi} runtimeApi - api instance to query the chain
 * @param {string} discoverApiEndpoint - url for a colossus discovery api endpoint
 * @returns { Promise<object> } - the published service information
 */
async function discover_over_joystream_discovery_service(storageProviderId, runtimeApi, discoverApiEndpoint) {
	storageProviderId = new BN(storageProviderId)
	const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)

	if (!isProvider) {
		throw new Error('Cannot discover non storage providers')
	}

	const identity = await getIpnsIdentity(storageProviderId, runtimeApi)

	// dont waste time trying to resolve if no identity was found
	if (identity == null) {
		throw new Error('no identity to resolve')
	}

	if (!discoverApiEndpoint) {
		// Use bootstrap nodes
		const discoveryBootstrapNodes = await runtimeApi.discovery.getBootstrapEndpoints()

		if (discoveryBootstrapNodes.length) {
			discoverApiEndpoint = stripEndingSlash(discoveryBootstrapNodes[0].toString())
		} else {
			throw new Error('No known discovery bootstrap nodes found on network')
		}
	}

	const url = `${discoverApiEndpoint}/discover/v0/${storageProviderId.toNumber()}`

	// should have parsed if data was json?
	const response = await axios.get(url)

	return response.data
}

/**
 * Resolves id of provider to its service information.
 * Will use the local IPFS node over RPC interface.
 * If the storage provider is not registered it will throw an error.
 * @param {number | BN | u64 } storageProviderId - provider id to lookup
 * @param {RuntimeApi} runtimeApi - api instance to query the chain
 * @returns { Promise<object> } - the published service information
 */
async function discover_over_local_ipfs_node(storageProviderId, runtimeApi) {
	storageProviderId = new BN(storageProviderId)
	const isProvider = await runtimeApi.workers.isStorageProvider(storageProviderId)

	if (!isProvider) {
		throw new Error('Cannot discover non storage providers')
	}

	const identity = await getIpnsIdentity(storageProviderId, runtimeApi)

	if (identity == null) {
		// dont waste time trying to resolve if no identity was found
		throw new Error('no identity to resolve')
	}

	const ipns_address = `/ipns/${identity}/`

	debug('resolved ipns to ipfs object')
	// Can this call hang forever!? can/should we set a timeout?
	const ipfs_name = await ipfs.name.resolve(ipns_address, {
		// don't recurse, there should only be one indirection to the service info file
		recursive: false,
		nocache: false,
	})

	debug('getting ipfs object', ipfs_name)
	const data = await ipfs.get(ipfs_name) // this can sometimes hang forever!?! can we set a timeout?

	// there should only be one file published under the resolved path
	const content = data[0].content

	return JSON.parse(content)
}

/**
 * Cached discovery of storage provider service information. If useCachedValue is
 * set to true, will always return the cached result if found. New discovery will be triggered
 * if record is found to be stale. If a stale record is not desired (CACHE_TTL old) pass a non zero
 * value for maxCacheAge, which will force a new discovery and return the new resolved value.
 * This method in turn calls _discovery which handles concurrent discoveries and selects the appropriate
 * protocol to perform the query.
 * If the storage provider is not registered it will resolve to null
 * @param {number | BN | u64} storageProviderId - provider to discover
 * @param {RuntimeApi} runtimeApi - api instance to query the chain
 * @param {bool} useCachedValue - optionaly use chached queries
 * @param {number} maxCacheAge - maximum age of a cached query that triggers automatic re-discovery
 * @returns { Promise<object | null> } - the published service information
 */
async function discover(storageProviderId, runtimeApi, useCachedValue = false, maxCacheAge = 0) {
	storageProviderId = new BN(storageProviderId)
	const id = storageProviderId.toNumber()
	const cached = accountInfoCache[id]

	if (cached && useCachedValue) {
		if (maxCacheAge > 0) {
			// get latest value
			if (Date.now() > cached.updated + maxCacheAge) {
				return _discover(storageProviderId, runtimeApi)
			}
		}
		// refresh if cache if stale, new value returned on next cached query
		if (Date.now() > cached.updated + CACHE_TTL) {
			_discover(storageProviderId, runtimeApi)
		}
		// return best known value
		return cached.value
	}
	return _discover(storageProviderId, runtimeApi)
}

/**
 * Internal method that handles concurrent discoveries and caching of results. Will
 * select the appropriate discovery protocol based on wether we are in a browser environemtn or not.
 * If not in a browser it expects a local ipfs node to be running.
 * @param {number | BN | u64} storageProviderId
 * @param {RuntimeApi} runtimeApi - api instance for querying the chain
 * @returns { Promise<object | null> } - the published service information
 */
async function _discover(storageProviderId, runtimeApi) {
	storageProviderId = new BN(storageProviderId)
	const id = storageProviderId.toNumber()

	const discoveryResult = activeDiscoveries[id]
	if (discoveryResult) {
		debug('discovery in progress waiting for result for', id)
		return discoveryResult
	}

	debug('starting new discovery for', id)
	const deferredDiscovery = newExternallyControlledPromise()
	activeDiscoveries[id] = deferredDiscovery.promise

	let result
	try {
		if (inBrowser()) {
			result = await discover_over_joystream_discovery_service(storageProviderId, runtimeApi)
		} else {
			result = await discover_over_local_ipfs_node(storageProviderId, runtimeApi)
		}

		debug(result)
		result = JSON.stringify(result)
		accountInfoCache[id] = {
			value: result,
			updated: Date.now(),
		}

		deferredDiscovery.resolve(result)
		delete activeDiscoveries[id]
		return result
	} catch (err) {
		// we catch the error so we can update all callers
		// and throw again to inform the first caller.
		debug(err.message)
		delete activeDiscoveries[id]
		// deferredDiscovery.reject(err)
		deferredDiscovery.resolve(null) // resolve to null until we figure out the issue below
		// throw err // <-- throwing but this isn't being
		// caught correctly in express server! Is it because there is an uncaught promise somewhere
		// in the prior .reject() call ?
		// I've only seen this behaviour when error is from ipfs-client
		// ... is this unique to errors thrown from ipfs-client?
		// Problem is its crashing the node so just return null for now
		return null
	}
}

module.exports = {
	discover,
	discover_over_joystream_discovery_service,
	discover_over_ipfs_http_gateway,
	discover_over_local_ipfs_node,
}