runtime: storage-v2: Add storage buckets picking for a dynamic bag.

Cargo.lock

@@ -4285,6 +4285,7 @@ dependencies = [
  "pallet-balances",
  "pallet-common",
  "pallet-membership",
+ "pallet-randomness-collective-flip",
  "pallet-timestamp",
  "parity-scale-codec",
  "serde",

runtime-modules/storage-v2/Cargo.toml

@@ -23,6 +23,7 @@ frame-benchmarking = { package = 'frame-benchmarking', default-features = false,
 [dev-dependencies]
 sp-io = { package = 'sp-io', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '2cd20966cc09b059817c3ebe12fc130cdd850d62'}
 sp-core = { package = 'sp-core', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '2cd20966cc09b059817c3ebe12fc130cdd850d62'}
+randomness-collective-flip = { package = 'pallet-randomness-collective-flip', default-features = false, git = 'https://github.com/paritytech/substrate.git', rev = '2cd20966cc09b059817c3ebe12fc130cdd850d62'}
 
 [features]
 default = ['std']

runtime-modules/storage-v2/src/lib.rs

@@ -6,8 +6,6 @@
 #![warn(missing_docs)]
 
 // TODO: Remove old Storage pallet.
-// TODO: authentication_key
-// TODO: Check dynamic bag existence.
 // TODO: add dynamic bag creation policy.
 // TODO: add module comment
 // TODO: make public methods as root extrinsics to enable storage-node dev mode.
@@ -21,10 +19,11 @@ mod tests;
 mod benchmarking;
 
 mod bag_manager;
+mod storage_bucket_picker;
 
 use codec::{Codec, Decode, Encode};
 use frame_support::dispatch::{DispatchError, DispatchResult};
-use frame_support::traits::{Currency, ExistenceRequirement, Get};
+use frame_support::traits::{Currency, ExistenceRequirement, Get, Randomness};
 use frame_support::{decl_error, decl_event, decl_module, decl_storage, ensure, Parameter};
 #[cfg(feature = "std")]
 use serde::{Deserialize, Serialize};
@@ -42,6 +41,7 @@ use common::origin::ActorOriginValidator;
 use common::working_group::WorkingGroup;
 
 use bag_manager::BagManager;
+use storage_bucket_picker::StorageBucketPicker;
 
 //TODO: Prepare types for moving to common module for the DataObjectStorage.
 
@@ -165,6 +165,15 @@ pub trait Trait: frame_system::Trait + balances::Trait + membership::Trait {
     /// "Storage buckets per bag" value constraint.
     type StorageBucketsPerBagValueConstraint: Get<StorageBucketsPerBagValueConstraint>;
 
+    /// Defines initial storage buckets number assigned to a new dynamic bag.
+    type InitialStorageBucketsNumberForDynamicBag: Get<u64>; //TODO: remove?
+
+    /// Defines max random iteration number (eg.: when picking the storage buckets).
+    type MaxRandomIterationNumber: Get<u64>;
+
+    /// Something that provides randomness in the runtime.
+    type Randomness: Randomness<Self::Hash>;
+
     /// Demand the working group leader authorization.
     /// TODO: Refactor after merging with the Olympia release.
     fn ensure_working_group_leader_origin(origin: Self::Origin) -> DispatchResult;
@@ -962,6 +971,10 @@ decl_module! {
         const StorageBucketsPerBagValueConstraint: StorageBucketsPerBagValueConstraint =
             T::StorageBucketsPerBagValueConstraint::get();
 
+        /// Exports const - initial storage buckets number assigned to a new dynamic bag.
+        const InitialStorageBucketsNumberForDynamicBag: u64 =
+            T::InitialStorageBucketsNumberForDynamicBag::get();
+
         // ===== Storage Lead actions =====
 
         /// Delete storage bucket. Must be empty. Storage operator must be missing.
@@ -2204,4 +2217,11 @@ impl<T: Trait> Module<T> {
             let _ = Balances::<T>::slash(account_id, fee);
         }
     }
+
+    // Selects storage bucket ID sets to assign to the storage bucket.
+    pub(crate) fn pick_storage_buckets_for_dynamic_bag(
+        _bag_type: DynamicBagType,
+    ) -> BTreeSet<T::StorageBucketId> {
+        StorageBucketPicker::<T>::pick_storage_buckets()
+    }
 }

runtime-modules/storage-v2/src/storage_bucket_picker.rs

@@ -0,0 +1,170 @@
+#![warn(missing_docs)]
+
+use frame_support::traits::{Get, Randomness};
+use sp_arithmetic::traits::{One, Zero};
+use sp_runtime::SaturatedConversion;
+use sp_std::collections::btree_set::BTreeSet;
+use sp_std::marker::PhantomData;
+
+use crate::{Module, Trait};
+
+// Generates storage bucket IDs to assign to a new dynamic bag.
+pub(crate) struct StorageBucketPicker<T> {
+    trait_marker: PhantomData<T>,
+}
+
+impl<T: Trait> StorageBucketPicker<T> {
+    // Selects storage bucket ID sets to assign to the storage bucket.
+    // At first it tries to generate random bucket IDs. If acquired random IDs number is not enough
+    // it tries to get additional IDs starting from zero up to total number of the possible IDs.
+    // The function filters deleted buckets and disabled buckets (accepting_new_bags == false)
+    // Total number of possible IDs is limited by the dynamic bag settings.
+    // Returns the accumulated bucket ID set or empty set.
+    pub(crate) fn pick_storage_buckets() -> BTreeSet<T::StorageBucketId> {
+        let bucket_num = T::InitialStorageBucketsNumberForDynamicBag::get();
+
+        RandomStorageBucketIdIterator::<T>::new()
+            .chain(SequentialStorageBucketIdIterator::<T>::new())
+            .filter(Self::check_storage_bucket_id_for_bag_assigning)
+            .scan(BTreeSet::new(), |bucket_ids, bucket_id| {
+                if !bucket_ids.contains(&bucket_id) {
+                    bucket_ids.insert(bucket_id);
+                }
+
+                Some(bucket_ids.clone())
+            })
+            .take(bucket_num as usize)
+            .last()
+            .unwrap_or_default()
+    }
+
+    // Verifies storage bucket ID (non-deleted and accepting new bags).
+    pub(crate) fn check_storage_bucket_id_for_bag_assigning(
+        bucket_id: &T::StorageBucketId,
+    ) -> bool {
+        // Check bucket for existence (return false if not). Check `accepting_new_bags`.
+        Module::<T>::ensure_storage_bucket_exists(bucket_id)
+            .ok()
+            .map(|bucket| bucket.accepting_new_bags)
+            .unwrap_or(false)
+    }
+}
+
+// Iterator for random storage bucket IDs. It uses Substrate Randomness trait
+// (and possibly randomness_collective_flip pallet for implementation).
+// Its maximum iterations are bounded.
+struct RandomStorageBucketIdIterator<T: Trait> {
+    // Trait marker.
+    trait_marker: PhantomData<T>,
+
+    // Current Iterator step number.
+    current_iteration: u64,
+
+    // Maximum allowed iteration number.
+    max_iteration_number: u64,
+
+    // Current seed for the randomness generator.
+    current_seed: T::Hash,
+}
+
+impl<T: Trait> Iterator for RandomStorageBucketIdIterator<T> {
+    type Item = T::StorageBucketId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Cannot create randomness in the initial block (Substrate error).
+        if <frame_system::Module<T>>::block_number() == Zero::zero() {
+            return None;
+        }
+
+        if self.current_iteration > self.max_iteration_number {
+            return None;
+        }
+
+        let random_storage_bucket_id = self.random_storage_bucket_id();
+
+        self.current_iteration += 1;
+        self.current_seed = T::Randomness::random(self.current_seed.as_ref());
+
+        Some(random_storage_bucket_id)
+    }
+}
+
+impl<T: Trait> RandomStorageBucketIdIterator<T> {
+    // Generate random storage bucket ID using next_storage_bucket_id() as upper_bound.
+    // Deleted storage bucket ID are included.
+    fn random_storage_bucket_id(&self) -> T::StorageBucketId {
+        let total_buckets_number = Module::<T>::next_storage_bucket_id();
+
+        let random_bucket_id: T::StorageBucketId = self
+            .random_index(total_buckets_number.saturated_into())
+            .saturated_into();
+
+        random_bucket_id
+    }
+
+    // Generate random number from zero to upper_bound (excluding).
+    fn random_index(&self, upper_bound: u64) -> u64 {
+        if upper_bound == 0 {
+            return upper_bound;
+        }
+
+        let mut rand: u64 = 0;
+        for offset in 0..8 {
+            rand += (self.current_seed.as_ref()[offset] as u64) << offset;
+        }
+        rand % upper_bound
+    }
+
+    // Creates new iterator.
+    fn new() -> Self {
+        // Cannot create randomness in the initial block (Substrate error).
+        let seed = if <frame_system::Module<T>>::block_number() == Zero::zero() {
+            Default::default()
+        } else {
+            T::Randomness::random_seed()
+        };
+
+        Self {
+            current_iteration: 0,
+            max_iteration_number: T::MaxRandomIterationNumber::get(),
+            trait_marker: PhantomData,
+            current_seed: seed,
+        }
+    }
+}
+
+// Iterator for sequential storage bucket IDs. It starts from the first possible storage bucket ID
+// (zero) and goes up to the last storage bucket IDs (next_storage_bucket_id - excluding).
+struct SequentialStorageBucketIdIterator<T: Trait> {
+    // Trait marker.
+    trait_marker: PhantomData<T>,
+
+    // Storage bucket ID for the current iteration.
+    current_bucket_id: T::StorageBucketId,
+}
+
+impl<T: Trait> Iterator for SequentialStorageBucketIdIterator<T> {
+    type Item = T::StorageBucketId;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_bucket_id >= Module::<T>::next_storage_bucket_id() {
+            return None;
+        }
+
+        let result = self.current_bucket_id;
+
+        self.current_bucket_id += One::one();
+
+        Some(result)
+    }
+}
+
+impl<T: Trait> SequentialStorageBucketIdIterator<T> {
+    // Creates new iterator.
+    fn new() -> Self {
+        Self {
+            current_bucket_id: Zero::zero(),
+            trait_marker: PhantomData,
+        }
+    }
+}

runtime-modules/storage-v2/src/tests/fixtures.rs

@@ -5,7 +5,7 @@ use frame_system::{EventRecord, Phase, RawOrigin};
 use sp_std::collections::btree_set::BTreeSet;
 
 use super::mocks::{
-    Balances, Storage, System, Test, TestEvent, DEFAULT_MEMBER_ACCOUNT_ID,
+    Balances, CollectiveFlip, Storage, System, Test, TestEvent, DEFAULT_MEMBER_ACCOUNT_ID,
     DEFAULT_STORAGE_PROVIDER_ACCOUNT_ID, WG_LEADER_ACCOUNT_ID,
 };
 
@@ -19,9 +19,11 @@ use crate::{
 pub fn run_to_block(n: u64) {
     while System::block_number() < n {
         <System as OnFinalize<u64>>::on_finalize(System::block_number());
+        <CollectiveFlip as OnFinalize<u64>>::on_finalize(System::block_number());
         <Storage as OnFinalize<u64>>::on_finalize(System::block_number());
         System::set_block_number(System::block_number() + 1);
         <System as OnInitialize<u64>>::on_initialize(System::block_number());
+        <CollectiveFlip as OnInitialize<u64>>::on_initialize(System::block_number());
         <Storage as OnInitialize<u64>>::on_initialize(System::block_number());
     }
 }

runtime-modules/storage-v2/src/tests/mocks.rs

@@ -50,13 +50,15 @@ impl balances::Trait for Test {
 }
 
 parameter_types! {
-    pub const MaxStorageBucketNumber: u64 = 2;
+    pub const MaxStorageBucketNumber: u64 = 1000;
     pub const MaxNumberOfDataObjectsPerBag: u64 = 4;
     pub const DataObjectDeletionPrize: u64 = 10;
     pub const StorageModuleId: ModuleId = ModuleId(*b"mstorage"); // module storage
     pub const BlacklistSizeLimit: u64 = 1;
     pub const StorageBucketsPerBagValueConstraint: crate::StorageBucketsPerBagValueConstraint =
         crate::StorageBucketsPerBagValueConstraint {min: 3, max_min_diff: 7};
+    pub const InitialStorageBucketsNumberForDynamicBag: u64 = 3;
+    pub const MaxRandomIterationNumber: u64 = 3;
 }
 
 pub const WG_LEADER_ACCOUNT_ID: u64 = 100001;
@@ -76,6 +78,9 @@ impl crate::Trait for Test {
     type ModuleId = StorageModuleId;
     type MemberOriginValidator = ();
     type StorageBucketsPerBagValueConstraint = StorageBucketsPerBagValueConstraint;
+    type InitialStorageBucketsNumberForDynamicBag = InitialStorageBucketsNumberForDynamicBag;
+    type Randomness = CollectiveFlip;
+    type MaxRandomIterationNumber = MaxRandomIterationNumber;
 
     fn ensure_working_group_leader_origin(origin: Self::Origin) -> DispatchResult {
         let account_id = ensure_signed(origin)?;
@@ -179,3 +184,4 @@ pub fn build_test_externalities() -> sp_io::TestExternalities {
 pub type Storage = crate::Module<Test>;
 pub type System = frame_system::Module<Test>;
 pub type Balances = balances::Module<Test>;
+pub type CollectiveFlip = randomness_collective_flip::Module<Test>;

runtime-modules/storage-v2/src/tests/mod.rs

@@ -15,12 +15,13 @@ use common::working_group::WorkingGroup;
 
 use crate::{
     AssignedDataObject, BagId, DataObject, DataObjectCreationParameters, DataObjectStorage,
-    DynamicBagId, Error, ModuleAccount, ObjectsInBagParams, RawEvent, StaticBagId,
+    DynamicBagId, DynamicBagType, Error, ModuleAccount, ObjectsInBagParams, RawEvent, StaticBagId,
     StorageBucketOperatorStatus, StorageTreasury, UploadParameters, Voucher,
 };
 
 use mocks::{
-    build_test_externalities, Balances, DataObjectDeletionPrize, MaxNumberOfDataObjectsPerBag,
+    build_test_externalities, Balances, DataObjectDeletionPrize,
+    InitialStorageBucketsNumberForDynamicBag, MaxNumberOfDataObjectsPerBag, MaxStorageBucketNumber,
     Storage, Test, DEFAULT_MEMBER_ACCOUNT_ID, DEFAULT_MEMBER_ID,
     DEFAULT_STORAGE_PROVIDER_ACCOUNT_ID, DEFAULT_STORAGE_PROVIDER_ID, WG_LEADER_ACCOUNT_ID,
 };
@@ -133,13 +134,8 @@ fn create_storage_bucket_fails_with_non_leader_origin() {
 #[test]
 fn create_storage_bucket_fails_with_exceeding_max_storage_bucket_limit() {
     build_test_externalities().execute_with(|| {
-        CreateStorageBucketFixture::default()
-            .with_origin(RawOrigin::Signed(WG_LEADER_ACCOUNT_ID))
-            .call_and_assert(Ok(()));
-
-        CreateStorageBucketFixture::default()
-            .with_origin(RawOrigin::Signed(WG_LEADER_ACCOUNT_ID))
-            .call_and_assert(Ok(()));
+        let buckets_number = MaxStorageBucketNumber::get();
+        create_storage_buckets(buckets_number);
 
         CreateStorageBucketFixture::default()
             .with_origin(RawOrigin::Signed(WG_LEADER_ACCOUNT_ID))
@@ -3132,3 +3128,87 @@ fn create_dynamic_bag(dynamic_bag_id: &DynamicBagId<Test>) {
         .with_bag_id(dynamic_bag_id.clone())
         .call_and_assert(Ok(()));
 }
+
+#[test]
+fn test_storage_bucket_picking_for_bag_non_random() {
+    build_test_externalities().execute_with(|| {
+        // Randomness disabled on the initial block.
+
+        let initial_buckets_number = InitialStorageBucketsNumberForDynamicBag::get();
+        // No buckets
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+        assert_eq!(bucket_ids, BTreeSet::new());
+
+        // Less then initial buckets number
+        let buckets_number = initial_buckets_number - 1;
+        let created_buckets = create_storage_buckets(buckets_number);
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+
+        assert_eq!(bucket_ids, created_buckets);
+
+        // More then initial buckets number
+        let buckets_number = 5;
+        create_storage_buckets(buckets_number);
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+
+        assert_eq!(
+            bucket_ids,
+            BTreeSet::from_iter((0u64..initial_buckets_number).into_iter())
+        );
+    });
+}
+
+#[test]
+fn test_storage_bucket_picking_for_bag_with_randomness() {
+    build_test_externalities().execute_with(|| {
+        // Randomness disabled on the initial block.
+        let starting_block = 1;
+        run_to_block(starting_block);
+
+        let initial_buckets_number = InitialStorageBucketsNumberForDynamicBag::get();
+        // No buckets
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+        assert_eq!(bucket_ids, BTreeSet::new());
+
+        // Less then initial buckets number
+        let buckets_number = initial_buckets_number - 1;
+        let created_buckets = create_storage_buckets(buckets_number);
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+
+        println!("{:?}", Storage::next_storage_bucket_id());
+        assert_eq!(bucket_ids, created_buckets);
+
+        // More then initial buckets number
+        let buckets_number = 5;
+        create_storage_buckets(buckets_number);
+        let bucket_ids = Storage::pick_storage_buckets_for_dynamic_bag(DynamicBagType::Member);
+
+        assert_eq!(
+            bucket_ids,
+            BTreeSet::from_iter((0u64..initial_buckets_number).into_iter())
+        );
+    });
+}
+
+fn create_storage_buckets(buckets_number: u64) -> BTreeSet<u64> {
+    set_max_voucher_limits();
+
+    let objects_limit = 1;
+    let size_limit = 100;
+
+    let mut bucket_ids = BTreeSet::new();
+
+    for _ in 0..buckets_number {
+        let bucket_id = CreateStorageBucketFixture::default()
+            .with_origin(RawOrigin::Signed(WG_LEADER_ACCOUNT_ID))
+            .with_invite_worker(None)
+            .with_objects_limit(objects_limit)
+            .with_size_limit(size_limit)
+            .call_and_assert(Ok(()))
+            .unwrap();
+
+        bucket_ids.insert(bucket_id);
+    }
+
+    bucket_ids
+}

runtime/src/lib.rs

@@ -678,9 +678,11 @@ parameter_types! {
     pub const MaxNumberOfDataObjectsPerBag: u64 = 1000; //TODO: adjust value
     pub const DataObjectDeletionPrize: Balance = 10; //TODO: adjust value
     pub const BlacklistSizeLimit: u64 = 10000; //TODO: adjust value
+    pub const MaxRandomIterationNumber: u64 = 30; //TODO: adjust value
     pub const StorageModuleId: ModuleId = ModuleId(*b"mstorage"); // module storage
     pub const StorageBucketsPerBagValueConstraint: storage_v2::StorageBucketsPerBagValueConstraint =
         storage_v2::StorageBucketsPerBagValueConstraint {min: 3, max_min_diff: 7}; //TODO: adjust value
+    pub const InitialStorageBucketsNumberForDynamicBag: u64 = 5; //TODO: adjust value
 }
 
 impl storage_v2::Trait for Runtime {
@@ -696,6 +698,9 @@ impl storage_v2::Trait for Runtime {
     type ModuleId = StorageModuleId;
     type MemberOriginValidator = MembershipOriginValidator<Self>;
     type StorageBucketsPerBagValueConstraint = StorageBucketsPerBagValueConstraint;
+    type InitialStorageBucketsNumberForDynamicBag = InitialStorageBucketsNumberForDynamicBag;
+    type Randomness = RandomnessCollectiveFlip;
+    type MaxRandomIterationNumber = MaxRandomIterationNumber;
 
     fn ensure_working_group_leader_origin(origin: Self::Origin) -> DispatchResult {
         StorageWorkingGroup::ensure_origin_is_active_leader(origin)