Merge pull request #2278 from mnaamani/storage-upload-to-any-provider

Uploads can be done to any storage provider

.github/workflows/run-network-tests.yml

@@ -221,10 +221,10 @@ jobs:
         run: |
           sleep 6
           export DEBUG=joystream:*
-          yarn storage-cli upload ./pioneer/packages/apps/public/images/default-thumbnail.png 1 0
+          yarn storage-cli upload ./tests/network-tests/assets/joystream.MOV 1 0
           # Wait for storage-node to set status Accepted on uploaded content
           sleep 6
           cd utils/api-scripts/
           # Assume only one accepted data object was created
           CONTENT_ID=`yarn --silent script get-first-content-id | tail -n2 | head -n1`
-          yarn storage-cli download http://localhost:3001 ${CONTENT_ID} ./download.png
+          yarn storage-cli download ${CONTENT_ID} ./joystream.mov

Cargo.lock

@@ -2053,7 +2053,7 @@ dependencies = [
 
 [[package]]
 name = "joystream-node-runtime"
-version = "7.15.0"
+version = "7.16.0"
 dependencies = [
  "frame-benchmarking",
  "frame-executive",

runtime-modules/storage/src/data_directory.rs

@@ -63,9 +63,6 @@ pub trait Trait:
     /// _Data directory_ event type.
     type Event: From<Event<Self>> + Into<<Self as system::Trait>::Event>;
 
-    /// Provides random storage provider id.
-    type StorageProviderHelper: StorageProviderHelper<Self>;
-
     /// Active data object type validator.
     type IsActiveDataObjectType: data_object_type_registry::IsActiveDataObjectType<Self>;
 
@@ -79,9 +76,6 @@ decl_error! {
         /// Content with this ID not found.
         CidNotFound,
 
-        /// Only the liaison for the content may modify its status.
-        LiaisonRequired,
-
         /// Cannot create content for inactive or missing data object type.
         DataObjectTypeMustBeActive,
 
@@ -112,9 +106,6 @@ decl_error! {
         /// Provided owner should be equal o the data object owner under given content id
         OwnersAreNotEqual,
 
-        /// No storage provider available to service the request
-        NoProviderAvailable,
-
         /// New voucher limit being set is less than used.
         VoucherLimitLessThanUsed,
 
@@ -123,7 +114,7 @@ decl_error! {
     }
 }
 
-/// The decision of the storage provider when it acts as liaison.
+/// The status of the content which can be updated by a storage provider.
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
 #[derive(Clone, Encode, Decode, PartialEq, Debug)]
 pub enum LiaisonJudgement {
@@ -132,9 +123,6 @@ pub enum LiaisonJudgement {
 
     /// Content accepted.
     Accepted,
-
-    /// Content rejected.
-    Rejected,
 }
 
 impl Default for LiaisonJudgement {
@@ -178,10 +166,10 @@ pub struct DataObjectInternal<
     /// Content size in bytes.
     pub size: u64,
 
-    /// Storage provider id of the liaison.
-    pub liaison: StorageProviderId,
+    /// Storage provider which first accepted the content.
+    pub liaison: Option<StorageProviderId>,
 
-    /// Storage provider as liaison judgment.
+    /// The liaison judgment.
     pub liaison_judgement: LiaisonJudgement,
 
     /// IPFS content id.
@@ -425,7 +413,7 @@ decl_module! {
         type Error = Error<T>;
 
         /// Adds the content to the system. The created DataObject
-        /// awaits liaison to accept or reject it.
+        /// awaits liaison to accept it.
         #[weight = 10_000_000] // TODO: adjust weight
         pub fn add_content(
             origin,
@@ -443,8 +431,6 @@ decl_module! {
             // Ensure owner and global voucher constraints satisfied.
             let (new_owner_voucher, new_global_voucher) = Self::ensure_voucher_constraints_satisfied(&owner, &content)?;
 
-            let liaison = T::StorageProviderHelper::get_random_storage_provider()?;
-
             //
             // == MUTATION SAFE ==
             //
@@ -455,7 +441,7 @@ decl_module! {
             // Update global voucher
             <GlobalVoucher>::put(new_global_voucher);
 
-            Self::upload_content(liaison, content.clone(), owner.clone());
+            Self::upload_content(content.clone(), owner.clone());
 
             Self::deposit_event(RawEvent::ContentAdded(content, owner));
         }
@@ -652,7 +638,8 @@ decl_module! {
         }
 
         /// Storage provider accepts a content. Requires signed storage provider account and its id.
-        /// The LiaisonJudgement can be updated, but only by the liaison.
+        /// The LiaisonJudgement can only be updated once from Pending to Accepted.
+        /// Subsequent calls are a no-op.
         #[weight = 10_000_000] // TODO: adjust weight
         pub(crate) fn accept_content(
             origin,
@@ -661,27 +648,19 @@ decl_module! {
         ) {
             <StorageWorkingGroup<T>>::ensure_worker_signed(origin, &storage_provider_id)?;
 
-            // == MUTATION SAFE ==
-
-            Self::update_content_judgement(&storage_provider_id, content_id, LiaisonJudgement::Accepted)?;
-
-            Self::deposit_event(RawEvent::ContentAccepted(content_id, storage_provider_id));
-        }
-
-        /// Storage provider rejects a content. Requires signed storage provider account and its id.
-        /// The LiaisonJudgement can be updated, but only by the liaison.
-        #[weight = 10_000_000] // TODO: adjust weight
-        pub(crate) fn reject_content(
-            origin,
-            storage_provider_id: StorageProviderId<T>,
-            content_id: T::ContentId
-        ) {
-            <StorageWorkingGroup<T>>::ensure_worker_signed(origin, &storage_provider_id)?;
+            let mut data = Self::get_data_object(&content_id)?;
 
             // == MUTATION SAFE ==
 
-            Self::update_content_judgement(&storage_provider_id, content_id, LiaisonJudgement::Rejected)?;
-            Self::deposit_event(RawEvent::ContentRejected(content_id, storage_provider_id));
+            if data.liaison_judgement == LiaisonJudgement::Pending {
+                // Set the liaison which is updating the judgement
+                data.liaison = Some(storage_provider_id);
+
+                // Set the judgement
+                data.liaison_judgement = LiaisonJudgement::Accepted;
+                <DataByContentId<T>>::insert(content_id, data);
+                Self::deposit_event(RawEvent::ContentAccepted(content_id, storage_provider_id));
+            }
         }
 
         /// Locks / unlocks content uploading
@@ -806,9 +785,8 @@ impl<T: Trait> Module<T> {
         Ok((new_owner_voucher, new_global_voucher))
     }
 
-    // Complete content upload, update vouchers
+    // Complete content upload
     fn upload_content(
-        liaison: StorageProviderId<T>,
         multi_content: Vec<ContentParameters<T::ContentId, DataObjectTypeId<T>>>,
         owner: ObjectOwner<T>,
     ) {
@@ -818,7 +796,7 @@ impl<T: Trait> Module<T> {
                 size: content.size,
                 added_at: common::current_block_time::<T>(),
                 owner: owner.clone(),
-                liaison,
+                liaison: None,
                 liaison_judgement: LiaisonJudgement::Pending,
                 ipfs_content_id: content.ipfs_content_id,
             };
@@ -843,31 +821,6 @@ impl<T: Trait> Module<T> {
         }
         Ok(())
     }
-
-    fn update_content_judgement(
-        storage_provider_id: &StorageProviderId<T>,
-        content_id: T::ContentId,
-        judgement: LiaisonJudgement,
-    ) -> DispatchResult {
-        let mut data = Self::get_data_object(&content_id)?;
-
-        // Make sure the liaison matches
-        ensure!(
-            data.liaison == *storage_provider_id,
-            Error::<T>::LiaisonRequired
-        );
-
-        data.liaison_judgement = judgement;
-        <DataByContentId<T>>::insert(content_id, data);
-
-        Ok(())
-    }
-}
-
-/// Provides random storage provider id. We use it when assign the content to the storage provider.
-pub trait StorageProviderHelper<T: Trait> {
-    /// Provides random storage provider id.
-    fn get_random_storage_provider() -> Result<StorageProviderId<T>, Error<T>>;
 }
 
 /// Content access helper.
@@ -906,8 +859,6 @@ impl<T: Trait> common::storage::StorageSystem<T> for Module<T> {
         let (new_owner_voucher, new_global_voucher) =
             Self::ensure_voucher_constraints_satisfied(&owner, &content)?;
 
-        let liaison = T::StorageProviderHelper::get_random_storage_provider()?;
-
         //
         // == MUTATION SAFE ==
         //
@@ -918,7 +869,7 @@ impl<T: Trait> common::storage::StorageSystem<T> for Module<T> {
         // Update global voucher
         <GlobalVoucher>::put(new_global_voucher);
 
-        Self::upload_content(liaison, content, owner);
+        Self::upload_content(content, owner);
         Ok(())
     }
 
@@ -958,8 +909,6 @@ impl<T: Trait> common::storage::StorageSystem<T> for Module<T> {
     ) -> DispatchResult {
         Self::ensure_uploading_is_not_blocked()?;
 
-        T::StorageProviderHelper::get_random_storage_provider()?;
-
         let _ = Self::ensure_voucher_constraints_satisfied(&owner, &content)?;
 
         Self::ensure_content_is_valid(&content)

runtime-modules/storage/src/tests/data_directory.rs

@@ -431,13 +431,6 @@ fn accept_and_reject_content_fail_with_invalid_storage_provider() {
             content_id,
         );
         assert_eq!(res, Err(working_group::Error::<Test, crate::StorageWorkingGroupInstance>::WorkerDoesNotExist.into()));
-
-        let res = TestDataDirectory::reject_content(
-            Origin::signed(storage_provider_account_id),
-            storage_provider_id,
-            content_id,
-        );
-        assert_eq!(res, Err(working_group::Error::<Test, crate::StorageWorkingGroupInstance>::WorkerDoesNotExist.into()));
     });
 }
 
@@ -492,57 +485,6 @@ fn accept_content_as_liaison() {
     });
 }
 
-#[test]
-fn reject_content_as_liaison() {
-    with_default_mock_builder(|| {
-        /*
-           Events are not emitted on block 0.
-           So any dispatchable calls made during genesis block formation will have no events emitted.
-           https://substrate.dev/recipes/2-appetizers/4-events.html
-        */
-        run_to_block(1);
-
-        let sender = 1u64;
-        let owner = StorageObjectOwner::Member(1u64);
-
-        let content_parameters = ContentParameters {
-            content_id: 1,
-            type_id: 1234,
-            size: 0,
-            ipfs_content_id: vec![1, 2, 3, 4],
-        };
-
-        let res =
-            TestDataDirectory::add_content(Origin::signed(sender), owner, vec![content_parameters]);
-        assert!(res.is_ok());
-
-        // An appropriate event should have been fired.
-        let (content_id, creator) = match &System::events().last().unwrap().event {
-            MetaEvent::data_directory(data_directory::RawEvent::ContentAdded(content, creator)) => {
-                (content[0].content_id, creator.clone())
-            }
-            _ => (0u64, StorageObjectOwner::Member(0xdeadbeefu64)), // invalid value, unlikely to match
-        };
-        assert_ne!(creator, StorageObjectOwner::Member(0xdeadbeefu64));
-        assert_eq!(creator, StorageObjectOwner::Member(sender));
-
-        let (storage_provider_account_id, storage_provider_id) = hire_storage_provider();
-
-        // Rejecting content should not work with some random origin
-        let res =
-            TestDataDirectory::reject_content(Origin::signed(55), storage_provider_id, content_id);
-        assert!(res.is_err());
-
-        // However, with the liaison as origin it should.
-        let res = TestDataDirectory::reject_content(
-            Origin::signed(storage_provider_account_id),
-            storage_provider_id,
-            content_id,
-        );
-        assert_eq!(res, Ok(()));
-    });
-}
-
 #[test]
 fn set_global_voucher_limits() {
     with_default_mock_builder(|| {

runtime-modules/storage/src/tests/mock.rs

@@ -111,7 +111,7 @@ impl ContentIdExists<Test> for MockContent {
                     time: 1024,
                 },
                 owner: StorageObjectOwner::Member(1),
-                liaison: TEST_MOCK_LIAISON_STORAGE_PROVIDER_ID,
+                liaison: Some(TEST_MOCK_LIAISON_STORAGE_PROVIDER_ID),
                 liaison_judgement: data_directory::LiaisonJudgement::Pending,
                 ipfs_content_id: vec![],
             }),
@@ -208,17 +208,10 @@ impl data_object_type_registry::Trait for Test {
 
 impl data_directory::Trait for Test {
     type Event = MetaEvent;
-    type StorageProviderHelper = ();
     type IsActiveDataObjectType = AnyDataObjectTypeIsActive;
     type MemberOriginValidator = ();
 }
 
-impl crate::data_directory::StorageProviderHelper<Test> for () {
-    fn get_random_storage_provider() -> Result<u32, data_directory::Error<Test>> {
-        Ok(1)
-    }
-}
-
 impl common::origin::ActorOriginValidator<Origin, u64, u64> for () {
     fn ensure_actor_origin(origin: Origin, _account_id: u64) -> Result<u64, &'static str> {
         let signed_account_id = system::ensure_signed(origin)?;

runtime/Cargo.toml

@@ -4,7 +4,7 @@ edition = '2018'
 name = 'joystream-node-runtime'
 # Follow convention: https://github.com/Joystream/substrate-runtime-joystream/issues/1
 # {Authoring}.{Spec}.{Impl} of the RuntimeVersion
-version = '7.15.0'
+version = '7.16.0'
 
 [dependencies]
 # Third-party dependencies

runtime/src/integration/mod.rs

@@ -1,6 +1,5 @@
 pub mod content_directory;
 pub mod forum;
 pub mod proposals;
-pub mod storage;
 pub mod transactions;
 pub mod working_group;

runtime/src/integration/storage.rs

@@ -1,37 +0,0 @@
-use frame_support::traits::Randomness;
-use sp_std::vec::Vec;
-
-use crate::{ActorId, Runtime};
-
-/// Provides random storage provider id. We use it when assign the content to the storage provider.
-pub struct StorageProviderHelper;
-
-impl storage::data_directory::StorageProviderHelper<Runtime> for StorageProviderHelper {
-    fn get_random_storage_provider() -> Result<ActorId, storage::data_directory::Error<Runtime>> {
-        let ids = crate::StorageWorkingGroup::get_all_worker_ids();
-
-        // Filter workers that have set value for their storage value
-        let ids: Vec<ActorId> = ids
-            .into_iter()
-            .filter(|id| !crate::StorageWorkingGroup::worker_storage(id).is_empty())
-            .collect();
-
-        if ids.is_empty() {
-            Err(storage::data_directory::Error::<Runtime>::NoProviderAvailable)
-        } else {
-            let index = Self::random_index(ids.len());
-            Ok(ids[index])
-        }
-    }
-}
-
-impl StorageProviderHelper {
-    fn random_index(upper_bound: usize) -> usize {
-        let seed = crate::RandomnessCollectiveFlip::random_seed();
-        let mut rand: u64 = 0;
-        for offset in 0..8 {
-            rand += (seed.as_ref()[offset] as u64) << offset;
-        }
-        (rand as usize) % upper_bound
-    }
-}

runtime/src/lib.rs

@@ -71,7 +71,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
     spec_name: create_runtime_str!("joystream-node"),
     impl_name: create_runtime_str!("joystream-node"),
     authoring_version: 7,
-    spec_version: 15,
+    spec_version: 16,
     impl_version: 0,
     apis: crate::runtime_api::EXPORTED_RUNTIME_API_VERSIONS,
     transaction_version: 1,
@@ -462,7 +462,6 @@ impl storage::data_object_type_registry::Trait for Runtime {
 
 impl storage::data_directory::Trait for Runtime {
     type Event = Event;
-    type StorageProviderHelper = integration::storage::StorageProviderHelper;
     type IsActiveDataObjectType = DataObjectTypeRegistry;
     type MemberOriginValidator = MembershipOriginValidator<Self>;
 }

runtime/src/tests/mod.rs

@@ -4,7 +4,6 @@
 #[macro_use]
 
 mod proposals_integration;
-mod storage_integration;
 use sp_runtime::BuildStorage;
 
 pub(crate) fn initial_test_ext() -> sp_io::TestExternalities {

runtime/src/tests/storage_integration.rs

@@ -1,40 +0,0 @@
-use super::initial_test_ext;
-use crate::integration::storage::StorageProviderHelper;
-use crate::Runtime;
-
-use frame_support::StorageMap;
-use working_group::{Instance2, Worker};
-
-#[test]
-fn storage_provider_helper_succeeds() {
-    initial_test_ext().execute_with(|| {
-		// Bug in random module requires move the initial block number.
-		<system::Module<Runtime>>::set_block_number(1);
-
-		// Error - no workers.
-		let random_provider_result = <StorageProviderHelper
-			as storage::data_directory::StorageProviderHelper<Runtime>>::get_random_storage_provider();
-		assert!(random_provider_result.is_err());
-
-		let worker_id1 = 1;
-		let worker_id2 = 7;
-		let worker_id3 = 19;
-
-		<working_group::WorkerById<Runtime, Instance2>>::insert(worker_id1, Worker::default());
-		<working_group::WorkerById<Runtime, Instance2>>::insert(worker_id2, Worker::default());
-		<working_group::WorkerById<Runtime, Instance2>>::insert(worker_id3, Worker::default());
-
-		// Still error - endpoints not set in worker storage value.
-		let random_provider_result = <StorageProviderHelper as storage::data_directory::StorageProviderHelper<Runtime>>::get_random_storage_provider();
-		assert!(random_provider_result.is_err());
-
-		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id1, b"http://storage1.net/".to_vec());
-		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id2, b"http://storage2.net/".to_vec());
-		<working_group::WorkerStorage<Runtime, Instance2>>::insert(worker_id3, b"http://storage3.net/".to_vec());
-
-		// Should work now.
-		let worker_ids = vec![worker_id1, worker_id2, worker_id3];
-		let random_provider_id = <StorageProviderHelper as storage::data_directory::StorageProviderHelper<Runtime>>::get_random_storage_provider().unwrap();
-		assert!(worker_ids.contains(&random_provider_id));
-	});
-}

storage-node/packages/cli/src/cli.ts

@@ -41,8 +41,7 @@ const usage = `
     upload            Upload a file to the Joystream Network. Requires a
                       source file path to upload, data object ID, member ID and account key file with
                       pass phrase to unlock it.
-    download          Retrieve a file. Requires a storage node URL and a content
-                      ID, as well as an output filename.
+    download          Retrieve a file. Requires a content and an output filename.
     head              Send a HEAD request for a file, and print headers.
                       Requires a storage node URL and a content ID.
 
@@ -88,11 +87,8 @@ const commands = {
 
     await uploadCmd.run()
   },
-  // needs to be updated to take a content id and resolve it a potential set
-  // of providers that has it, and select one (possibly try more than one provider)
-  // to fetch it from the get api url of a provider..
-  download: async (api: any, url: string, contentId: string, filePath: string) => {
-    const downloadCmd = new DownloadCommand(api, url, contentId, filePath)
+  download: async (api: any, contentId: string, filePath: string) => {
+    const downloadCmd = new DownloadCommand(api, contentId, filePath)
 
     await downloadCmd.run()
   },

storage-node/packages/cli/src/commands/base.ts

@@ -1,9 +1,17 @@
 import chalk from 'chalk'
 import removeEndingForwardSlash from '@joystream/storage-utils/stripEndingSlash'
 import { ContentId } from '@joystream/types/storage'
+import Debug from 'debug'
+const debug = Debug('joystream:storage-cli:base')
 
 // Commands base abstract class. Contains reusable methods.
 export abstract class BaseCommand {
+  protected readonly api: any
+
+  constructor(api: any) {
+    this.api = api
+  }
+
   // Creates the Colossus asset URL and logs it.
   protected createAndLogAssetUrl(url: string, contentId: string | ContentId): string {
     let normalizedContentId: string
@@ -50,4 +58,36 @@ export abstract class BaseCommand {
     // Maximum content length for the assets (files)
     return 2000 * 1024 * 1024
   }
+
+  // Requests the runtime and obtains the storage node endpoint URL.
+  protected async getStorageProviderEndpoint(storageProviderId: string): Promise<string> {
+    try {
+      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
+
+      debug(`Resolved endpoint: ${endpoint}`)
+
+      return endpoint
+    } catch (err) {
+      this.fail(`Could not get provider endpoint: ${err}`)
+    }
+  }
+
+  protected async getAnyProviderEndpoint(): Promise<string> {
+    try {
+      const providers = await this.api.workers.getAllProviders()
+
+      debug(`Available Providers: ${providers}`)
+      // select first provider
+      do {
+        const id = providers.ids.pop()
+        const endpoint = await this.getStorageProviderEndpoint(id)
+        if (endpoint) {
+          return endpoint
+        }
+      } while (providers.ids.length)
+      throw new Error('No Providers registered endpoint')
+    } catch (err) {
+      this.fail(`Could not get provider endpoint: ${err}`)
+    }
+  }
 }

storage-node/packages/cli/src/commands/download.ts

@@ -5,38 +5,27 @@ import { BaseCommand } from './base'
 
 // Download command class. Validates input parameters and execute the logic for asset downloading.
 export class DownloadCommand extends BaseCommand {
-  private readonly api: any
-  private readonly storageNodeUrl: string
   private readonly contentId: string
   private readonly outputFilePath: string
 
-  constructor(api: any, storageNodeUrl: string, contentId: string, outputFilePath: string) {
-    super()
+  constructor(api: any, contentId: string, outputFilePath: string) {
+    super(api)
 
-    this.api = api
-    this.storageNodeUrl = storageNodeUrl
     this.contentId = contentId
     this.outputFilePath = outputFilePath
   }
 
   // Provides parameter validation. Overrides the abstract method from the base class.
   protected validateParameters(): boolean {
-    return (
-      this.storageNodeUrl &&
-      this.storageNodeUrl !== '' &&
-      this.contentId &&
-      this.contentId !== '' &&
-      this.outputFilePath &&
-      this.outputFilePath !== ''
-    )
+    return this.contentId && this.contentId !== '' && this.outputFilePath && this.outputFilePath !== ''
   }
 
   // Shows command usage. Overrides the abstract method from the base class.
   protected showUsage() {
     console.log(
       chalk.yellow(`
-        Usage:   storage-cli download colossusURL contentID filePath
-        Example: storage-cli download http://localhost:3001 0x7a6ba7e9157e5fba190dc146fe1baa8180e29728a5c76779ed99655500cff795 ./movie.mp4
+        Usage:   storage-cli download contentID filePath
+        Example: storage-cli download 5Ec3PL3wbutqvDykhNxXJFEWSdw9rS4LBsGUXH9gSusFzc5X ./movie.mp4
       `)
     )
   }
@@ -46,7 +35,9 @@ export class DownloadCommand extends BaseCommand {
     // Checks for input parameters, shows usage if they are invalid.
     if (!this.assertParameters()) return
 
-    const assetUrl = this.createAndLogAssetUrl(this.storageNodeUrl, this.contentId)
+    const storageNodeUrl = await this.getAnyProviderEndpoint()
+
+    const assetUrl = this.createAndLogAssetUrl(storageNodeUrl, this.contentId)
     console.log(chalk.yellow('File path:', this.outputFilePath))
 
     // Create file write stream and set error handler.

storage-node/packages/cli/src/commands/head.ts

@@ -4,14 +4,12 @@ import { BaseCommand } from './base'
 
 // Head command class. Validates input parameters and obtains the asset headers.
 export class HeadCommand extends BaseCommand {
-  private readonly api: any
   private readonly storageNodeUrl: string
   private readonly contentId: string
 
   constructor(api: any, storageNodeUrl: string, contentId: string) {
-    super()
+    super(api)
 
-    this.api = api
     this.storageNodeUrl = storageNodeUrl
     this.contentId = contentId
   }

storage-node/packages/cli/src/commands/upload.ts

@@ -21,7 +21,6 @@ interface AddContentParams {
 
 // Upload command class. Validates input parameters and uploads the asset to the storage node and runtime.
 export class UploadCommand extends BaseCommand {
-  private readonly api: any
   private readonly mediaSourceFilePath: string
   private readonly dataObjectTypeId: string
   private readonly keyFile: string
@@ -36,9 +35,8 @@ export class UploadCommand extends BaseCommand {
     keyFile: string,
     passPhrase: string
   ) {
-    super()
+    super(api)
 
-    this.api = api
     this.mediaSourceFilePath = mediaSourceFilePath
     this.dataObjectTypeId = dataObjectTypeId
     this.memberId = memberId
@@ -153,19 +151,6 @@ export class UploadCommand extends BaseCommand {
     }
   }
 
-  // Requests the runtime and obtains the storage node endpoint URL.
-  private async discoverStorageProviderEndpoint(storageProviderId: string): Promise<string> {
-    try {
-      const endpoint = await this.api.workers.getWorkerStorageValue(storageProviderId)
-
-      debug(`Resolved endpoint: ${endpoint}`)
-
-      return endpoint
-    } catch (err) {
-      this.fail(`Could not get provider endpoint: ${err}`)
-    }
-  }
-
   // Loads and unlocks the runtime identity using the key file and pass phrase.
   private async loadIdentity(): Promise<any> {
     const noKeyFileProvided = !this.keyFile || this.keyFile === ''
@@ -208,7 +193,7 @@ export class UploadCommand extends BaseCommand {
     const dataObject = await this.createContent(addContentParams)
     debug(`Received data object: ${dataObject.toString()}`)
 
-    const colossusEndpoint = await this.discoverStorageProviderEndpoint(dataObject.liaison.toString())
+    const colossusEndpoint = await this.getAnyProviderEndpoint()
     debug(`Discovered storage node endpoint: ${colossusEndpoint}`)
 
     const assetUrl = this.createAndLogAssetUrl(colossusEndpoint, addContentParams.contentId)

storage-node/packages/colossus/paths/asset/v0/{id}.js

@@ -73,17 +73,23 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
 
       const id = req.params.id // content id
 
-      // Check if we're the liaison for the content
+      // Check if content exists
       const roleAddress = runtime.identities.key.address
       const providerId = runtime.storageProviderId
       let dataObject
+
       try {
-        dataObject = await runtime.assets.checkLiaisonForDataObject(providerId, id)
+        dataObject = await runtime.assets.getDataObject(id)
       } catch (err) {
         errorHandler(res, err, 403)
         return
       }
 
+      if (!dataObject) {
+        res.status(404).send({ message: 'Content Not Found' })
+        return
+      }
+
       // Early filtering on content_length..do not wait for fileInfo
       // ensure its less than max allowed by node policy.
       const filterResult = filter({}, req.headers)
@@ -156,7 +162,6 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
             if (filterResult.code !== 200) {
               debug('Rejecting content')
               stream.cleanup()
-              await runtime.assets.rejectContent(roleAddress, providerId, id)
               res.status(400).send({ message: 'Rejecting content type' })
             } else {
               try {
@@ -181,14 +186,21 @@ module.exports = function (storage, runtime, ipfsHttpGatewayUrl, anonymous) {
 
           try {
             debug('accepting Content')
-            await runtime.assets.acceptContent(roleAddress, providerId, id)
+            // Only if judegment is Pending
+            if (dataObject.liaison_judgement.type === 'Pending') {
+              await runtime.assets.acceptContent(roleAddress, providerId, id)
+            }
 
-            debug('creating storage relationship for newly uploaded content')
-            // Create storage relationship and flip it to ready.
-            const dosrId = await runtime.assets.createStorageRelationship(roleAddress, providerId, id)
+            // Is there any real value in updating this state? Nobody uses it!
+            const { relationshipId } = await runtime.assets.getStorageRelationshipAndId(providerId, id)
+            if (!relationshipId) {
+              debug('creating storage relationship for newly uploaded content')
+              // Create storage relationship and flip it to ready.
+              const dosrId = await runtime.assets.createStorageRelationship(roleAddress, providerId, id)
 
-            debug('toggling storage relationship for newly uploaded content')
-            await runtime.assets.toggleStorageRelationshipReady(roleAddress, providerId, dosrId, true)
+              debug('toggling storage relationship for newly uploaded content')
+              await runtime.assets.toggleStorageRelationshipReady(roleAddress, providerId, dosrId, true)
+            }
           } catch (err) {
             debug(`${err.message}`)
           }

storage-node/packages/runtime-api/assets.js

@@ -102,15 +102,6 @@ class AssetsApi {
     return this.base.signAndSend(providerAccoundId, tx)
   }
 
-  /*
-   * Sets the data object liaison judgement to Rejected
-   */
-  async rejectContent(providerAccountId, storageProviderId, contentId) {
-    contentId = parseContentId(contentId)
-    const tx = this.base.api.tx.dataDirectory.rejectContent(storageProviderId, contentId)
-    return this.base.signAndSend(providerAccountId, tx)
-  }
-
   /*
    * Gets storage relationship for contentId for the given provider
    */

types/augment-codec/augment-api-tx.ts

@@ -368,35 +368,55 @@ declare module '@polkadot/api/types/submittable' {
     dataDirectory: {
       /**
        * Storage provider accepts a content. Requires signed storage provider account and its id.
-       * The LiaisonJudgement can be updated, but only by the liaison.
+       * The LiaisonJudgement can only be updated once from Pending to Accepted.
+       * Subsequent calls are a no-op.
        **/
       acceptContent: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, contentId: ContentId | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Adds the content to the system. The created DataObject
-       * awaits liaison to accept or reject it.
+       * awaits liaison to accept it.
        **/
       addContent: AugmentedSubmittable<(owner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, content: Vec<ContentParameters> | (ContentParameters | { content_id?: any; type_id?: any; ipfs_content_id?: any } | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Storage provider rejects a content. Requires signed storage provider account and its id.
-       * The LiaisonJudgement can be updated, but only by the liaison.
-       **/
-      rejectContent: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, contentId: ContentId | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Remove the content from the system.
        **/
       removeContent: AugmentedSubmittable<(owner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, contentIds: Vec<ContentId> | (ContentId | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Set the default owner voucher
+       **/
+      setDefaultVoucher: AugmentedSubmittable<(sizeLimit: u64 | AnyNumber | Uint8Array, objectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets global voucher objects limit. Requires root privileges.
+       * New limit cannot be less that used value.
+       **/
+      setGlobalVoucherObjectsLimit: AugmentedSubmittable<(newObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets global voucher size limit. Requires root privileges.
+       * New limit cannot be less that used value.
+       **/
+      setGlobalVoucherSizeLimit: AugmentedSubmittable<(newSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets VoucherObjectsLimitUpperBound. Requires root privileges.
+       **/
+      setVoucherObjectsLimitUpperBound: AugmentedSubmittable<(newUpperBound: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets VoucherSizeLimitUpperBound. Requires root privileges.
+       **/
+      setVoucherSizeLimitUpperBound: AugmentedSubmittable<(newUpperBound: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Locks / unlocks content uploading
        **/
       updateContentUploadingStatus: AugmentedSubmittable<(isBlocked: bool | boolean | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Updates storage object owner voucher objects limit. Requires leader privileges.
+       * New limit cannot be less that used value.
        **/
-      updateStorageObjectOwnerVoucherObjectsLimit: AugmentedSubmittable<(abstractOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      updateStorageObjectOwnerVoucherObjectsLimit: AugmentedSubmittable<(objectOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Updates storage object owner voucher size limit. Requires leader privileges.
+       * New limit cannot be less that used value.
        **/
-      updateStorageObjectOwnerVoucherSizeLimit: AugmentedSubmittable<(abstractOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      updateStorageObjectOwnerVoucherSizeLimit: AugmentedSubmittable<(objectOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
     };
     dataObjectStorageRegistry: {
       /**

types/augment/all/defs.json

@@ -485,8 +485,7 @@
     "LiaisonJudgement": {
         "_enum": [
             "Pending",
-            "Accepted",
-            "Rejected"
+            "Accepted"
         ]
     },
     "DataObject": {
@@ -494,7 +493,7 @@
         "added_at": "BlockAndTime",
         "type_id": "DataObjectTypeId",
         "size": "u64",
-        "liaison": "StorageProviderId",
+        "liaison": "Option<StorageProviderId>",
         "liaison_judgement": "LiaisonJudgement",
         "ipfs_content_id": "Text"
     },

types/augment/all/types.ts

@@ -346,7 +346,7 @@ export interface DataObject extends Struct {
   readonly owner: StorageObjectOwner;
   readonly added_at: BlockAndTime;
   readonly type_id: DataObjectTypeId;
-  readonly liaison: StorageProviderId;
+  readonly liaison: Option<StorageProviderId>;
   readonly liaison_judgement: LiaisonJudgement;
   readonly ipfs_content_id: Text;
 }
@@ -532,7 +532,6 @@ export interface LeadId extends Null {}
 export interface LiaisonJudgement extends Enum {
   readonly isPending: boolean;
   readonly isAccepted: boolean;
-  readonly isRejected: boolean;
 }
 
 /** @name LookupSource */

types/augment/augment-api-tx.ts

@@ -368,35 +368,55 @@ declare module '@polkadot/api/types/submittable' {
     dataDirectory: {
       /**
        * Storage provider accepts a content. Requires signed storage provider account and its id.
-       * The LiaisonJudgement can be updated, but only by the liaison.
+       * The LiaisonJudgement can only be updated once from Pending to Accepted.
+       * Subsequent calls are a no-op.
        **/
       acceptContent: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, contentId: ContentId | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Adds the content to the system. The created DataObject
-       * awaits liaison to accept or reject it.
+       * awaits liaison to accept it.
        **/
       addContent: AugmentedSubmittable<(owner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, content: Vec<ContentParameters> | (ContentParameters | { content_id?: any; type_id?: any; ipfs_content_id?: any } | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
-      /**
-       * Storage provider rejects a content. Requires signed storage provider account and its id.
-       * The LiaisonJudgement can be updated, but only by the liaison.
-       **/
-      rejectContent: AugmentedSubmittable<(storageProviderId: StorageProviderId | AnyNumber | Uint8Array, contentId: ContentId | string | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Remove the content from the system.
        **/
       removeContent: AugmentedSubmittable<(owner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, contentIds: Vec<ContentId> | (ContentId | string | Uint8Array)[]) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Set the default owner voucher
+       **/
+      setDefaultVoucher: AugmentedSubmittable<(sizeLimit: u64 | AnyNumber | Uint8Array, objectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets global voucher objects limit. Requires root privileges.
+       * New limit cannot be less that used value.
+       **/
+      setGlobalVoucherObjectsLimit: AugmentedSubmittable<(newObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets global voucher size limit. Requires root privileges.
+       * New limit cannot be less that used value.
+       **/
+      setGlobalVoucherSizeLimit: AugmentedSubmittable<(newSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets VoucherObjectsLimitUpperBound. Requires root privileges.
+       **/
+      setVoucherObjectsLimitUpperBound: AugmentedSubmittable<(newUpperBound: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      /**
+       * Sets VoucherSizeLimitUpperBound. Requires root privileges.
+       **/
+      setVoucherSizeLimitUpperBound: AugmentedSubmittable<(newUpperBound: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Locks / unlocks content uploading
        **/
       updateContentUploadingStatus: AugmentedSubmittable<(isBlocked: bool | boolean | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Updates storage object owner voucher objects limit. Requires leader privileges.
+       * New limit cannot be less that used value.
        **/
-      updateStorageObjectOwnerVoucherObjectsLimit: AugmentedSubmittable<(abstractOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      updateStorageObjectOwnerVoucherObjectsLimit: AugmentedSubmittable<(objectOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherObjectsLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
       /**
        * Updates storage object owner voucher size limit. Requires leader privileges.
+       * New limit cannot be less that used value.
        **/
-      updateStorageObjectOwnerVoucherSizeLimit: AugmentedSubmittable<(abstractOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
+      updateStorageObjectOwnerVoucherSizeLimit: AugmentedSubmittable<(objectOwner: ObjectOwner | { Member: any } | { Channel: any } | { DAO: any } | { Council: any } | { WorkingGroup: any } | string | Uint8Array, newVoucherSizeLimit: u64 | AnyNumber | Uint8Array) => SubmittableExtrinsic<ApiType>>;
     };
     dataObjectStorageRegistry: {
       /**

types/src/storage.ts

@@ -1,4 +1,4 @@
-import { BTreeMap, u64, bool, Text, Null, Bytes } from '@polkadot/types'
+import { BTreeMap, u64, bool, Text, Null, Bytes, Option } from '@polkadot/types'
 import { BlockAndTime, JoyEnum, JoyStructDecorated, Hash, ChannelId, DAOId, WorkingGroup } from './common'
 import { MemberId } from './members'
 import { StorageProviderId } from './working-group' // this should be in discovery really
@@ -32,7 +32,6 @@ export class DataObjectStorageRelationshipId extends u64 {}
 export const LiaisonJudgementDef = {
   Pending: Null,
   Accepted: Null,
-  Rejected: Null,
 } as const
 export type LiaisonJudgementKey = keyof typeof LiaisonJudgementDef
 export class LiaisonJudgement extends JoyEnum(LiaisonJudgementDef) {}
@@ -62,7 +61,7 @@ export class DataObject extends JoyStructDecorated({
   added_at: BlockAndTime,
   type_id: DataObjectTypeId,
   size: u64,
-  liaison: StorageProviderId,
+  liaison: Option.with(StorageProviderId),
   liaison_judgement: LiaisonJudgement,
   ipfs_content_id: Text,
 }) {