storage-node-v2: Resolve storage-nodes urls

storage-node-v2/src/services/sync/dataObjectsModel.ts

@@ -7,7 +7,7 @@ import {
   DataObjectDetailsFragment,
 } from '../queryNode/generated/queries'
 
-type Model = {
+export type Model = {
   storageBuckets: StorageBucket[]
   bags: Bag[]
   dataObjects: DataObject[]
@@ -118,13 +118,12 @@ async function getAllObjectsWithPaging<T>(
   return result
 }
 
-function extractOperatorUrl(encodedString: string): string{
+function extractOperatorUrl(encodedString: string): string {
   try {
     return u8aToString(hexToU8a(encodedString))
-  }
-  catch (err) {
+  } catch (err) {
     logger.error(`Sync - ${err}`)
   }
 
   return ''
-}
+}

storage-node-v2/src/services/sync/synchronizer.ts

@@ -1,4 +1,4 @@
-import { getRuntimeModel } from '../../services/sync/dataObjectsModel'
+import { getRuntimeModel, Model } from '../../services/sync/dataObjectsModel'
 import logger from '../../services/logger'
 import _ from 'lodash'
 import fs from 'fs'
@@ -11,6 +11,15 @@ import AwaitLock from 'await-lock'
 import sleep from 'sleep-promise'
 const fsPromises = fs.promises
 
+//TODO: use caching
+export async function getLocalDataObjects(
+  uploadDirectory: string
+): Promise<string[]> {
+  const localCids = await getLocalFileNames(uploadDirectory)
+
+  return localCids
+}
+
 export async function performSync(
   workerId: number,
   processNumber: number,
@@ -23,6 +32,7 @@ export async function performSync(
     getRuntimeModel(queryNodeUrl, workerId),
     getLocalFileNames(uploadDirectory),
   ])
+  console.log(model)
 
   const requiredCids = model.dataObjects.map((obj) => obj.cid)
 
@@ -35,16 +45,14 @@ export async function performSync(
   const deletedTasks = deleted.map(
     (fileName) => new DeleteLocalFileTask(uploadDirectory, fileName)
   )
-  const addedTasks = added.map(
-    (fileName) => new DownloadFileTask(operatorUrl, fileName, uploadDirectory)
-  )
+  const addedTasks = await getDownloadTasks(model, operatorUrl, added, uploadDirectory)
 
   logger.debug(`Sync - started processing...`)
   const workingStack = new WorkingStack()
   const processSpawner = new TaskProcessorSpawner(workingStack, processNumber)
 
-  workingStack.add(deletedTasks)
   workingStack.add(addedTasks)
+  workingStack.add(deletedTasks)
 
   await processSpawner.process()
   logger.info('Sync ended.')
@@ -195,3 +203,21 @@ class TaskProcessor {
     }
   }
 }
+
+
+async function getDownloadTasks(
+  model: Model,
+  operatorUrl: string,
+  addedCids: string[],
+  uploadDirectory: string
+): Promise<DownloadFileTask[]> {
+  //model.dataObjects.
+
+  model.bags
+
+  const addedTasks = addedCids.map(
+    (fileName) => new DownloadFileTask(operatorUrl, fileName, uploadDirectory)
+  )
+
+  return addedTasks
+}

storage-node-v2/src/services/webApi/app.ts

@@ -9,6 +9,7 @@ import { ApiPromise } from '@polkadot/api'
 import { RequestData, verifyTokenSignature, parseUploadToken, UploadToken } from '../helpers/auth'
 import { checkRemoveNonce } from '../../services/helpers/tokenNonceKeeper'
 import { httpLogger, errorLogger } from '../../services/logger'
+import { getLocalDataObjects } from '../../services/sync/synchronizer'
 
 /**
  * Creates Express web application. Uses the OAS spec file for the API.
@@ -35,6 +36,12 @@ export async function createApp(
   app.use(express.json())
   app.use(httpLogger())
 
+  // TODO: put it in the API spec
+  app.get('/sync', async (request, response) => {
+    const cids = await getLocalDataObjects(uploadsDir)
+    response.json(cids)
+  })
+
   app.use(
     // Set parameters for each request.
     (req: express.Request, res: express.Response, next: NextFunction) => {