add storage provider fallback, add asset response timeout (#1017)

src/config/assets.ts

@@ -0,0 +1 @@
+export const ASSET_RESPONSE_TIMEOUT = 5000

src/providers/assets/assetsManager.tsx

@@ -1,3 +1,4 @@
+import { shuffle } from 'lodash'
 import React, { useEffect } from 'react'
 
 import { Logger } from '@/utils/logger'
@@ -8,7 +9,7 @@ import { useAssetStore } from './store'
 import { useStorageProviders } from '../storageProviders'
 
 export const AssetsManager: React.FC = () => {
-  const { getStorageProvider } = useStorageProviders()
+  const { getStorageProviders, getRandomStorageProvider } = useStorageProviders()
   const pendingAssets = useAssetStore((state) => state.pendingAssets)
   const assetIdsBeingResolved = useAssetStore((state) => state.assetIdsBeingResolved)
   const { addAsset, addAssetBeingResolved, removeAssetBeingResolved, removePendingAsset } = useAssetStore(
@@ -24,26 +25,40 @@ export const AssetsManager: React.FC = () => {
       addAssetBeingResolved(contentId)
 
       const resolutionData = pendingAssets[contentId]
-      const assetUrl = getAssetUrl(resolutionData, getStorageProvider()?.url)
-      if (!assetUrl) {
-        Logger.warn('Unable to create asset url', resolutionData)
-        addAsset(contentId, {})
-        return
-      }
-      try {
-        await testAssetDownload(assetUrl, resolutionData.assetType)
-        addAsset(contentId, { url: assetUrl })
-        removePendingAsset(contentId)
-        removeAssetBeingResolved(contentId)
-      } catch (e) {
-        Logger.error(`Failed to load ${resolutionData.assetType}`, { contentId, assetUrl })
+      const allStorageProviders = shuffle(getStorageProviders() || [])
+      const storageProvidersWithoutLiaison = allStorageProviders.filter(
+        (provider) => provider.id !== resolutionData.dataObject?.liaison?.id
+      )
+      const storageProvidersToTry = [
+        ...(resolutionData.dataObject?.liaison ? [resolutionData.dataObject.liaison] : []),
+        ...storageProvidersWithoutLiaison,
+      ]
+      for (const storageProvider of storageProvidersToTry) {
+        const assetUrl = getAssetUrl(resolutionData, storageProvider.metadata ?? '')
+        if (!assetUrl) {
+          Logger.warn('Unable to create asset url', resolutionData)
+          addAsset(contentId, {})
+          return
+        }
+
+        try {
+          await testAssetDownload(assetUrl, resolutionData.assetType)
+          addAsset(contentId, { url: assetUrl })
+          removePendingAsset(contentId)
+          removeAssetBeingResolved(contentId)
+          return
+        } catch (e) {
+          Logger.error(`Failed to load ${resolutionData.assetType}`, { contentId, assetUrl })
+        }
       }
+      Logger.error(`No storage provider was able to provide asset`, { contentId })
     })
   }, [
     addAsset,
     addAssetBeingResolved,
     assetIdsBeingResolved,
-    getStorageProvider,
+    getStorageProviders,
+    getRandomStorageProvider,
     pendingAssets,
     removeAssetBeingResolved,
     removePendingAsset,

src/providers/assets/helpers.ts

@@ -4,13 +4,14 @@ import {
   BasicChannelFieldsFragment,
   VideoFieldsFragment,
 } from '@/api/queries'
+import { ASSET_RESPONSE_TIMEOUT } from '@/config/assets'
 import { createStorageNodeUrl } from '@/utils/asset'
-import { Logger } from '@/utils/logger'
+import { withTimeout } from '@/utils/misc'
 
 import { AssetResolutionData, AssetType } from './types'
 
 export const testAssetDownload = (url: string, type: AssetType) => {
-  return new Promise((resolve, reject) => {
+  const testPromise = new Promise((resolve, reject) => {
     if ([AssetType.COVER, AssetType.THUMBNAIL, AssetType.AVATAR].includes(type)) {
       const img = new Image()
       img.addEventListener('error', reject)
@@ -23,6 +24,7 @@ export const testAssetDownload = (url: string, type: AssetType) => {
       video.src = url
     }
   })
+  return withTimeout(testPromise, ASSET_RESPONSE_TIMEOUT)
 }
 export const readAssetData = (
   entity: VideoFieldsFragment | AllChannelFieldsFragment | BasicChannelFieldsFragment | null | undefined,
@@ -52,10 +54,7 @@ export const readAssetData = (
   }
   return null
 }
-export const getAssetUrl = (
-  assetData: AssetResolutionData,
-  randomStorageUrl: string | null | undefined
-): string | null | void => {
+export const getAssetUrl = (assetData: AssetResolutionData, storageProviderUrl: string): string | null | void => {
   if (assetData.availability !== AssetAvailability.Accepted) {
     return
   }
@@ -66,14 +65,5 @@ export const getAssetUrl = (
     return
   }
 
-  if (assetData.dataObject.liaison?.isActive && assetData.dataObject.liaison.metadata) {
-    return createStorageNodeUrl(assetData.dataObject.joystreamContentId, assetData.dataObject.liaison.metadata)
-  } else {
-    if (randomStorageUrl) {
-      return createStorageNodeUrl(assetData.dataObject.joystreamContentId, randomStorageUrl)
-    } else {
-      Logger.warn('Unable to create asset url', assetData)
-      return null
-    }
-  }
+  return createStorageNodeUrl(assetData.dataObject.joystreamContentId, storageProviderUrl)
 }

src/providers/connectionStatus.tsx

@@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useRef } from 'react'
 
 import { useSnackbar } from '@/providers/snackbars'
 import { createStore } from '@/store'
+import { withTimeout } from '@/utils/misc'
 
 export type ConnectionStatus = 'connected' | 'disconnected' | 'connecting'
 
@@ -82,8 +83,3 @@ export const ConnectionStatusManager: React.FC = () => {
 
   return null
 }
-
-const withTimeout = async <T,>(promise: Promise<T>, timeout: number) => {
-  const timeoutPromise = new Promise<T>((resolve, reject) => setTimeout(() => reject(new Error('Timed out!')), timeout))
-  return await Promise.race([timeoutPromise, promise])
-}

src/providers/storageProviders.tsx

@@ -66,7 +66,7 @@ export const useStorageProviders = () => {
     setNotWorkingStorageProvidersIds,
   } = ctx
 
-  const getStorageProvider = useCallback(() => {
+  const getStorageProviders = useCallback(() => {
     // make sure we finished fetching providers list
     if (storageProvidersLoading) {
       // TODO: we need to handle that somehow, possibly make it async and block until ready
@@ -86,6 +86,14 @@ export const useStorageProviders = () => {
       )
     }
 
+    return workingStorageProviders
+  }, [notWorkingStorageProvidersIds, storageProviders, storageProvidersLoading])
+
+  const getRandomStorageProvider = useCallback(() => {
+    const workingStorageProviders = getStorageProviders()
+    if (!workingStorageProviders) {
+      return null
+    }
     const randomStorageProviderIdx = getRandomIntInclusive(0, workingStorageProviders.length - 1)
     const randomStorageProvider = workingStorageProviders[randomStorageProviderIdx]
 
@@ -93,7 +101,7 @@ export const useStorageProviders = () => {
       id: randomStorageProvider.workerId,
       url: randomStorageProvider.metadata as string,
     }
-  }, [notWorkingStorageProvidersIds, storageProvidersLoading, storageProviders])
+  }, [getStorageProviders])
 
   const markStorageProviderNotWorking = useCallback(
     (workerId: string) => {
@@ -102,5 +110,5 @@ export const useStorageProviders = () => {
     [setNotWorkingStorageProvidersIds]
   )
 
-  return { getStorageProvider, markStorageProviderNotWorking }
+  return { getStorageProviders, getRandomStorageProvider, markStorageProviderNotWorking }
 }

src/providers/uploadsManager/useStartFileUpload.tsx

@@ -22,7 +22,7 @@ const UPLOADED_SNACKBAR_TIMEOUT = 13000
 export const useStartFileUpload = () => {
   const navigate = useNavigate()
   const { displaySnackbar } = useSnackbar()
-  const { getStorageProvider, markStorageProviderNotWorking } = useStorageProviders()
+  const { getRandomStorageProvider, markStorageProviderNotWorking } = useStorageProviders()
 
   const setAssetsFiles = useUploadsStore((state) => state.setAssetsFiles)
   const addAsset = useUploadsStore((state) => state.addAsset)
@@ -82,7 +82,7 @@ export const useStartFileUpload = () => {
     async (file: File | Blob | null, asset: InputAssetUpload, opts?: StartFileUploadOptions) => {
       let storageUrl: string, storageProviderId: string
       try {
-        const storageProvider = getStorageProvider()
+        const storageProvider = getRandomStorageProvider()
         if (!storageProvider) {
           return
         }
@@ -190,7 +190,7 @@ export const useStartFileUpload = () => {
       addAsset,
       assetsFiles,
       displaySnackbar,
-      getStorageProvider,
+      getRandomStorageProvider,
       markStorageProviderNotWorking,
       navigate,
       setAssetsFiles,

src/utils/misc.ts

@@ -0,0 +1,4 @@
+export const withTimeout = async <T>(promise: Promise<T>, timeout: number) => {
+  const timeoutPromise = new Promise<T>((resolve, reject) => setTimeout(() => reject(new Error('Timed out!')), timeout))
+  return await Promise.race([timeoutPromise, promise])
+}