runtime: storage: Add random distribution bucket selection.

runtime-modules/storage/src/distribution_bucket_picker.rs

@@ -1,94 +0,0 @@
-#![warn(missing_docs)]
-
-use frame_support::traits::Randomness;
-use sp_arithmetic::traits::Zero;
-use sp_runtime::SaturatedConversion;
-use sp_std::cell::RefCell;
-use sp_std::collections::btree_set::BTreeSet;
-use sp_std::marker::PhantomData;
-use sp_std::rc::Rc;
-use sp_std::vec::Vec;
-
-use crate::{DynamicBagType, Module, Trait};
-
-// Generates distribution bucket IDs to assign to a new dynamic bag.
-pub(crate) struct DistributionBucketPicker<T> {
-    trait_marker: PhantomData<T>,
-}
-
-impl<T: Trait> DistributionBucketPicker<T> {
-    // Get random distribution buckets from distribution bucket families using the dynamic bag
-    // creation policy.
-    pub(crate) fn pick_distribution_buckets(
-        bag_type: DynamicBagType,
-    ) -> BTreeSet<T::DistributionBucketId> {
-        let creation_policy = Module::<T>::get_dynamic_bag_creation_policy(bag_type);
-
-        if creation_policy.no_distribution_buckets_required() {
-            return BTreeSet::new();
-        }
-
-        // Randomness for all bucket family.
-        // let random_seed = RefCell::new(Module::<T>::get_initial_random_seed());
-        let random_seed = Rc::new(RefCell::new(Module::<T>::get_initial_random_seed()));
-
-        creation_policy
-            .families
-            .iter()
-            .filter_map(|(family_id, bucket_num)| {
-                Module::<T>::ensure_distribution_bucket_family_exists(family_id)
-                    .ok()
-                    .map(|fam| (fam, bucket_num))
-            })
-            .map(|(family, bucket_num)| {
-                let filtered_ids = family
-                    .distribution_buckets
-                    .iter()
-                    .filter_map(|(id, bucket)| bucket.accepting_new_bags.then(|| *id))
-                    .collect::<Vec<_>>();
-
-                (filtered_ids, bucket_num)
-            })
-            .map(|(bucket_ids, bucket_num)| {
-                Self::get_random_distribution_buckets(bucket_ids, *bucket_num, random_seed.clone())
-            })
-            .flatten()
-            .collect::<BTreeSet<_>>()
-    }
-
-    // Get random bucket IDs from the ID collection.
-    pub fn get_random_distribution_buckets(
-        ids: Vec<T::DistributionBucketId>,
-        bucket_number: u32,
-        seed: Rc<RefCell<T::Hash>>, //     seed: RefCell<T::Hash>
-    ) -> BTreeSet<T::DistributionBucketId> {
-        let mut working_ids = ids;
-        let mut result_ids = BTreeSet::default();
-
-        for _ in 0..bucket_number {
-            if working_ids.is_empty() {
-                break;
-            }
-
-            let current_seed = Self::advance_random_seed(seed.clone());
-
-            let upper_bound = working_ids.len() as u64 - 1;
-            let index =
-                Module::<T>::random_index(current_seed.as_ref(), upper_bound).saturated_into();
-            result_ids.insert(working_ids.remove(index));
-        }
-
-        result_ids
-    }
-
-    // Changes the internal seed value of the container and returns new random seed.
-    fn advance_random_seed(seed: Rc<RefCell<T::Hash>>) -> T::Hash {
-        // Cannot create randomness in the initial block (Substrate error).
-        if <frame_system::Module<T>>::block_number() == Zero::zero() {
-            return Module::<T>::get_initial_random_seed();
-        }
-
-        let current_seed = *seed.borrow();
-        seed.replace(T::Randomness::random(current_seed.as_ref()))
-    }
-}

runtime-modules/storage/src/lib.rs

@@ -127,7 +127,7 @@ mod tests;
 mod benchmarking;
 
 //pub(crate) mod distribution_bucket_picker;
-pub(crate) mod storage_bucket_picker;
+pub(crate) mod random_buckets;
 
 use codec::{Codec, Decode, Encode};
 use frame_support::dispatch::{DispatchError, DispatchResult};
@@ -151,8 +151,8 @@ use common::constraints::BoundedValueConstraint;
 use common::origin::ActorOriginValidator;
 use common::working_group::WorkingGroup;
 
-//use distribution_bucket_picker::DistributionBucketPicker;
-use storage_bucket_picker::StorageBucketPicker;
+use random_buckets::DistributionBucketPicker;
+use random_buckets::StorageBucketPicker;
 
 /// Public interface for the storage module.
 pub trait DataObjectStorage<T: Trait> {
@@ -247,7 +247,9 @@ pub trait Trait: frame_system::Trait + balances::Trait + membership::Trait {
         + Default
         + Copy
         + MaybeSerialize
-        + PartialEq;
+        + PartialEq
+        + Into<u64>
+        + From<u64>;
 
     /// Distribution bucket index within a distribution bucket family type.
     type DistributionBucketIndex: Parameter
@@ -257,7 +259,9 @@ pub trait Trait: frame_system::Trait + balances::Trait + membership::Trait {
         + Default
         + Copy
         + MaybeSerialize
-        + PartialEq;
+        + PartialEq
+        + Into<u64>
+        + From<u64>;
 
     /// Distribution bucket family ID type.
     type DistributionBucketFamilyId: Parameter
@@ -3284,8 +3288,7 @@ impl<T: Trait> Module<T> {
     pub(crate) fn pick_distribution_buckets_for_dynamic_bag(
         bag_type: DynamicBagType,
     ) -> BTreeSet<DistributionBucketId<T>> {
-        unimplemented!()
-        // DistributionBucketPicker::<T>::pick_distribution_buckets(bag_type)
+        DistributionBucketPicker::<T>::pick_distribution_buckets(bag_type)
     }
 
     // Get default dynamic bag policy by bag type.

runtime-modules/storage/src/random_buckets/distribution_bucket_picker.rs

@@ -0,0 +1,78 @@
+#![warn(missing_docs)]
+
+use sp_std::cell::RefCell;
+use sp_std::collections::btree_set::BTreeSet;
+use sp_std::marker::PhantomData;
+use sp_std::vec::Vec;
+
+use crate::{DistributionBucketId, DynamicBagType, Module, Trait};
+
+pub(crate) use super::{RandomBucketIdIterator, SequentialBucketIdIterator};
+
+// Generates distribution bucket IDs to assign to a new dynamic bag.
+pub(crate) struct DistributionBucketPicker<T> {
+    trait_marker: PhantomData<T>,
+}
+
+impl<T: Trait> DistributionBucketPicker<T> {
+    // Get random distribution buckets from distribution bucket families using the dynamic bag
+    // creation policy.
+    pub(crate) fn pick_distribution_buckets(
+        bag_type: DynamicBagType,
+    ) -> BTreeSet<DistributionBucketId<T>> {
+        let creation_policy = Module::<T>::get_dynamic_bag_creation_policy(bag_type);
+
+        if creation_policy.no_distribution_buckets_required() {
+            return BTreeSet::new();
+        }
+
+        // Distribution bucket IDs accumulator.
+        let bucket_ids_cell = RefCell::new(BTreeSet::<T::DistributionBucketIndex>::new());
+
+        creation_policy
+            .families
+            .iter()
+            .filter_map(|(family_id, bucket_num)| {
+                Module::<T>::ensure_distribution_bucket_family_exists(family_id)
+                    .ok()
+                    .map(|fam| (family_id, fam, bucket_num))
+            })
+            .map(|(family_id, family, bucket_num)| {
+                RandomBucketIdIterator::<T, T::DistributionBucketIndex>::new(
+                    family.next_distribution_bucket_index,
+                )
+                .chain(
+                    SequentialBucketIdIterator::<T, T::DistributionBucketIndex>::new(
+                        family.next_distribution_bucket_index,
+                    ),
+                )
+                .filter(|bucket_idx| {
+                    let bucket_id = DistributionBucketId::<T> {
+                        distribution_bucket_family_id: *family_id,
+                        distribution_bucket_index: *bucket_idx,
+                    };
+
+                    Module::<T>::ensure_distribution_bucket_exists(&bucket_id)
+                        .ok()
+                        .map(|bucket| bucket.accepting_new_bags)
+                        .unwrap_or(false)
+                })
+                .filter(|bucket_idx| {
+                    let bucket_ids = bucket_ids_cell.borrow();
+
+                    // Skips the iteration on existing ID.
+                    !bucket_ids.contains(bucket_idx)
+                })
+                .map(|bucket_idx| DistributionBucketId::<T> {
+                    distribution_bucket_family_id: *family_id,
+                    distribution_bucket_index: bucket_idx,
+                })
+                .take(*bucket_num as usize)
+                .collect::<Vec<_>>()
+
+                // rename buckets
+            })
+            .flatten()
+            .collect::<BTreeSet<_>>()
+    }
+}

runtime-modules/storage/src/random_buckets/mod.rs

@@ -0,0 +1,131 @@
+use frame_support::traits::{Get, Randomness};
+use sp_arithmetic::traits::{BaseArithmetic, One, Zero};
+use sp_runtime::traits::Bounded;
+use sp_runtime::SaturatedConversion;
+use sp_std::marker::PhantomData;
+
+use crate::{Module, Trait};
+
+pub(crate) mod distribution_bucket_picker;
+pub(crate) mod storage_bucket_picker;
+
+pub(crate) use distribution_bucket_picker::DistributionBucketPicker;
+pub(crate) use storage_bucket_picker::StorageBucketPicker;
+
+// A meta trait for defining generic bucket ID.
+pub(crate) trait BucketId:
+    Bounded + BaseArithmetic + From<u64> + Into<u64> + Clone + PartialOrd
+{
+}
+impl<T: Bounded + BaseArithmetic + From<u64> + Into<u64> + Clone + PartialOrd> BucketId for T {}
+
+// Iterator for random storage or distribution bucket IDs. It uses Substrate Randomness trait
+// (and possibly randomness_collective_flip pallet for implementation).
+// Its maximum iterations number is bounded.
+pub(crate) struct RandomBucketIdIterator<T: Trait, Id: BucketId> {
+    // Trait marker.
+    trait_marker: PhantomData<T>,
+
+    // Current Iterator step number.
+    current_iteration: u64,
+
+    // Maximum allowed iteration number.
+    max_iteration_number: u64,
+
+    // Current seed for the randomness generator.
+    current_seed: T::Hash,
+
+    // Next possible id for the buckets.
+    next_id: Id,
+}
+
+impl<T: Trait, Id: BucketId> Iterator for RandomBucketIdIterator<T, Id> {
+    type Item = Id;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Cannot create randomness in the initial block (Substrate error).
+        if <frame_system::Module<T>>::block_number() == Zero::zero() {
+            return None;
+        }
+
+        if self.current_iteration >= self.max_iteration_number {
+            return None;
+        }
+
+        let random_bucket_id = self.random_bucket_id();
+
+        self.current_iteration += 1;
+        self.current_seed = T::Randomness::random(self.current_seed.as_ref());
+
+        Some(random_bucket_id)
+    }
+}
+
+impl<T: Trait, Id: BucketId> RandomBucketIdIterator<T, Id> {
+    // Generate random storage or distribution bucket ID using next_id as an upper_bound.
+    // Deleted bucket IDs are included.
+    fn random_bucket_id(&self) -> Id {
+        let total_buckets_number: u64 = self.next_id.clone().into();
+
+        let random_bucket_id: Id = Module::<T>::random_index(
+            self.current_seed.as_ref(),
+            total_buckets_number.saturated_into(),
+        )
+        .saturated_into();
+
+        random_bucket_id
+    }
+
+    // Creates new iterator.
+    pub(crate) fn new(next_id: Id) -> Self {
+        let seed = Module::<T>::get_initial_random_seed();
+
+        Self {
+            current_iteration: 0,
+            max_iteration_number: T::MaxRandomIterationNumber::get(),
+            trait_marker: PhantomData,
+            current_seed: seed,
+            next_id,
+        }
+    }
+}
+
+// Iterator for sequential storage or distribution bucket IDs. It starts from the first possible storage bucket ID
+// (zero) and goes up to the last storage bucket IDs (next_storage_bucket_id - excluding).
+pub(crate) struct SequentialBucketIdIterator<T: Trait, Id: BucketId> {
+    // Trait marker.
+    trait_marker: PhantomData<T>,
+
+    // Bucket ID for the current iteration.
+    current_bucket_id: Id,
+
+    // Next possible id for the buckets.
+    next_id: Id,
+}
+
+impl<T: Trait, Id: BucketId> Iterator for SequentialBucketIdIterator<T, Id> {
+    type Item = Id;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_bucket_id >= self.next_id {
+            return None;
+        }
+
+        let result = self.current_bucket_id.clone();
+
+        self.current_bucket_id += One::one();
+
+        Some(result)
+    }
+}
+
+impl<T: Trait, Id: BucketId> SequentialBucketIdIterator<T, Id> {
+    // Creates new iterator.
+    pub(crate) fn new(next_id: Id) -> Self {
+        Self {
+            current_bucket_id: Zero::zero(),
+            trait_marker: PhantomData,
+            next_id,
+        }
+    }
+}

runtime-modules/storage/src/random_buckets/storage_bucket_picker.rs

@@ -0,0 +1,65 @@
+#![warn(missing_docs)]
+
+use sp_std::cell::RefCell;
+use sp_std::collections::btree_set::BTreeSet;
+use sp_std::marker::PhantomData;
+
+pub(crate) use super::{RandomBucketIdIterator, SequentialBucketIdIterator};
+use crate::{DynamicBagType, Module, Trait};
+
+// Generates storage bucket IDs to assign to a new dynamic bag.
+pub(crate) struct StorageBucketPicker<T> {
+    trait_marker: PhantomData<T>,
+}
+
+impl<T: Trait> StorageBucketPicker<T> {
+    // Selects storage bucket ID sets to assign to the storage bucket.
+    // At first, it tries to generate random bucket IDs. If acquired random IDs number is not enough
+    // it tries to get additional IDs starting from zero up to the total number of the possible IDs.
+    // The function filters deleted buckets and disabled buckets (accepting_new_bags == false)
+    // Total number of possible IDs is limited by the dynamic bag settings.
+    // Returns an accumulated bucket ID set or an empty set.
+    pub(crate) fn pick_storage_buckets(bag_type: DynamicBagType) -> BTreeSet<T::StorageBucketId> {
+        let creation_policy = Module::<T>::get_dynamic_bag_creation_policy(bag_type);
+
+        if creation_policy.no_storage_buckets_required() {
+            return BTreeSet::new();
+        }
+
+        let required_bucket_num = creation_policy.number_of_storage_buckets as usize;
+
+        // Storage bucket IDs accumulator.
+        let bucket_ids_cell = RefCell::new(BTreeSet::new());
+        let next_storage_bucket_id = Module::<T>::next_storage_bucket_id();
+        RandomBucketIdIterator::<T, T::StorageBucketId>::new(next_storage_bucket_id)
+            .chain(SequentialBucketIdIterator::<T, T::StorageBucketId>::new(
+                next_storage_bucket_id,
+            ))
+            .filter(Self::check_storage_bucket_is_valid_for_bag_assigning)
+            .filter(|bucket_id| {
+                let bucket_ids = bucket_ids_cell.borrow();
+
+                // Skips the iteration on existing ID.
+                !bucket_ids.contains(bucket_id)
+            })
+            .take(required_bucket_num)
+            .for_each(|bucket_id| {
+                let mut bucket_ids = bucket_ids_cell.borrow_mut();
+
+                bucket_ids.insert(bucket_id);
+            });
+
+        bucket_ids_cell.into_inner()
+    }
+
+    // Verifies storage bucket ID (non-deleted and accepting new bags).
+    pub(crate) fn check_storage_bucket_is_valid_for_bag_assigning(
+        bucket_id: &T::StorageBucketId,
+    ) -> bool {
+        // Check bucket for existence (return false if not). Check `accepting_new_bags`.
+        Module::<T>::ensure_storage_bucket_exists(bucket_id)
+            .ok()
+            .map(|bucket| bucket.accepting_new_bags)
+            .unwrap_or(false)
+    }
+}

runtime-modules/storage/src/storage_bucket_picker.rs

@@ -1,168 +0,0 @@
-#![warn(missing_docs)]
-
-use frame_support::traits::{Get, Randomness};
-use sp_arithmetic::traits::{One, Zero};
-use sp_runtime::SaturatedConversion;
-use sp_std::cell::RefCell;
-use sp_std::collections::btree_set::BTreeSet;
-use sp_std::marker::PhantomData;
-
-use crate::{DynamicBagType, Module, Trait};
-
-// Generates storage bucket IDs to assign to a new dynamic bag.
-pub(crate) struct StorageBucketPicker<T> {
-    trait_marker: PhantomData<T>,
-}
-
-impl<T: Trait> StorageBucketPicker<T> {
-    // Selects storage bucket ID sets to assign to the storage bucket.
-    // At first, it tries to generate random bucket IDs. If acquired random IDs number is not enough
-    // it tries to get additional IDs starting from zero up to the total number of the possible IDs.
-    // The function filters deleted buckets and disabled buckets (accepting_new_bags == false)
-    // Total number of possible IDs is limited by the dynamic bag settings.
-    // Returns an accumulated bucket ID set or an empty set.
-    pub(crate) fn pick_storage_buckets(bag_type: DynamicBagType) -> BTreeSet<T::StorageBucketId> {
-        let creation_policy = Module::<T>::get_dynamic_bag_creation_policy(bag_type);
-
-        if creation_policy.no_storage_buckets_required() {
-            return BTreeSet::new();
-        }
-
-        let required_bucket_num = creation_policy.number_of_storage_buckets as usize;
-
-        // Storage bucket IDs accumulator.
-        let bucket_ids_cell = RefCell::new(BTreeSet::new());
-
-        RandomStorageBucketIdIterator::<T>::new()
-            .chain(SequentialStorageBucketIdIterator::<T>::new())
-            .filter(Self::check_storage_bucket_is_valid_for_bag_assigning)
-            .filter(|bucket_id| {
-                let bucket_ids = bucket_ids_cell.borrow();
-
-                // Skips the iteration on existing ID.
-                !bucket_ids.contains(bucket_id)
-            })
-            .take(required_bucket_num)
-            .for_each(|bucket_id| {
-                let mut bucket_ids = bucket_ids_cell.borrow_mut();
-
-                bucket_ids.insert(bucket_id);
-            });
-
-        bucket_ids_cell.into_inner()
-    }
-
-    // Verifies storage bucket ID (non-deleted and accepting new bags).
-    pub(crate) fn check_storage_bucket_is_valid_for_bag_assigning(
-        bucket_id: &T::StorageBucketId,
-    ) -> bool {
-        // Check bucket for existence (return false if not). Check `accepting_new_bags`.
-        Module::<T>::ensure_storage_bucket_exists(bucket_id)
-            .ok()
-            .map(|bucket| bucket.accepting_new_bags)
-            .unwrap_or(false)
-    }
-}
-
-// Iterator for random storage bucket IDs. It uses Substrate Randomness trait
-// (and possibly randomness_collective_flip pallet for implementation).
-// Its maximum iterations are bounded.
-pub(crate) struct RandomStorageBucketIdIterator<T: Trait> {
-    // Trait marker.
-    trait_marker: PhantomData<T>,
-
-    // Current Iterator step number.
-    current_iteration: u64,
-
-    // Maximum allowed iteration number.
-    max_iteration_number: u64,
-
-    // Current seed for the randomness generator.
-    current_seed: T::Hash,
-}
-
-impl<T: Trait> Iterator for RandomStorageBucketIdIterator<T> {
-    type Item = T::StorageBucketId;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // Cannot create randomness in the initial block (Substrate error).
-        if <frame_system::Module<T>>::block_number() == Zero::zero() {
-            return None;
-        }
-
-        if self.current_iteration >= self.max_iteration_number {
-            return None;
-        }
-
-        let random_storage_bucket_id = self.random_storage_bucket_id();
-
-        self.current_iteration += 1;
-        self.current_seed = T::Randomness::random(self.current_seed.as_ref());
-
-        Some(random_storage_bucket_id)
-    }
-}
-
-impl<T: Trait> RandomStorageBucketIdIterator<T> {
-    // Generate random storage bucket ID using next_storage_bucket_id() as upper_bound.
-    // Deleted storage bucket ID are included.
-    fn random_storage_bucket_id(&self) -> T::StorageBucketId {
-        let total_buckets_number = Module::<T>::next_storage_bucket_id();
-
-        let random_bucket_id: T::StorageBucketId = Module::<T>::random_index(
-            self.current_seed.as_ref(),
-            total_buckets_number.saturated_into(),
-        )
-        .saturated_into();
-
-        random_bucket_id
-    }
-
-    // Creates new iterator.
-    pub(crate) fn new() -> Self {
-        let seed = Module::<T>::get_initial_random_seed();
-
-        Self {
-            current_iteration: 0,
-            max_iteration_number: T::MaxRandomIterationNumber::get(),
-            trait_marker: PhantomData,
-            current_seed: seed,
-        }
-    }
-}
-
-// Iterator for sequential storage bucket IDs. It starts from the first possible storage bucket ID
-// (zero) and goes up to the last storage bucket IDs (next_storage_bucket_id - excluding).
-pub(crate) struct SequentialStorageBucketIdIterator<T: Trait> {
-    // Trait marker.
-    trait_marker: PhantomData<T>,
-
-    // Storage bucket ID for the current iteration.
-    current_bucket_id: T::StorageBucketId,
-}
-
-impl<T: Trait> Iterator for SequentialStorageBucketIdIterator<T> {
-    type Item = T::StorageBucketId;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.current_bucket_id >= Module::<T>::next_storage_bucket_id() {
-            return None;
-        }
-
-        let result = self.current_bucket_id;
-
-        self.current_bucket_id += One::one();
-
-        Some(result)
-    }
-}
-
-impl<T: Trait> SequentialStorageBucketIdIterator<T> {
-    // Creates new iterator.
-    pub(crate) fn new() -> Self {
-        Self {
-            current_bucket_id: Zero::zero(),
-            trait_marker: PhantomData,
-        }
-    }
-}

runtime-modules/storage/src/tests/mod.rs

@@ -3312,12 +3312,12 @@ fn test_storage_bucket_iterators() {
         let buckets_number = 5;
         create_storage_buckets(buckets_number);
 
-        use crate::storage_bucket_picker::{
-            RandomStorageBucketIdIterator as Rand, SequentialStorageBucketIdIterator as Seq,
+        use crate::random_buckets::storage_bucket_picker::{
+            RandomBucketIdIterator as Rand, SequentialBucketIdIterator as Seq,
         };
 
-        let ids = Rand::<Test>::new()
-            .chain(Seq::<Test>::new())
+        let ids = Rand::<Test, u64>::new(Storage::next_storage_bucket_id())
+            .chain(Seq::<Test, u64>::new(Storage::next_storage_bucket_id()))
             .collect::<Vec<_>>();
 
         // Check combined iterator length.
@@ -4324,6 +4324,8 @@ fn distribution_bucket_family_pick_during_dynamic_bag_creation_succeeded() {
         let picked_bucket_ids =
             Storage::pick_distribution_buckets_for_dynamic_bag(dynamic_bag_type);
 
+        println!("{:?}", picked_bucket_ids);
+
         assert_eq!(picked_bucket_ids.len(), (new_bucket_number * 2) as usize); // buckets from two families
 
         let total_ids1 = BTreeSet::from_iter(