Skip to content

Commit

Permalink
Runtime Migrations (#170)
Browse files Browse the repository at this point in the history 
* add migration.md to track the process

* remove migration md (moved to issue #164)

* WIP add signaling hasher migration and (failing) test

* fix migration test

* add voting hasher change test and fix migration

* add WIP state grafting script

* actually execute migration for voting pallet

* use live chain for grafting and reformt code

* use apopiak-migrations substrate branch

* update grafting script to skip staking

* remove berlin.json loading line

* add mainnet-dev hybrid chainspec for migration testing

* update Cargo.lock

* add custom migration that orders balances and account migration correctly

* update cargo.lock
  • Loading branch information
@apopiak
apopiak authored Jun 23, 2020
1 parent 2697410 commit 0a01a69
Show file tree
Hide file tree
Showing 20 changed files with 9,869 additions and 648 deletions.
929 changes: 477 additions & 452 deletions Cargo.lock
View file

Large diffs are not rendered by default.

8,956 changes: 8,956 additions & 0 deletions chains/hybrid-mainnet-dev.json
View file

Large diffs are not rendered by default.

14 changes: 7 additions & 7 deletions modules/edge-signaling/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,16 @@ serde = { version = "1.0", default-features = false, optional = true }
serde_derive = { version = "1.0", optional = true }
safe-mix = { version = "1.0", default-features = false }
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
voting = { package ="edge-voting", path = "../edge-voting", default-features = false }

[dev-dependencies]
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }

[features]
default = ["std"]
Expand Down
25 changes: 23 additions & 2 deletions modules/edge-signaling/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
#[cfg(test)]
mod tests;

use frame_support::traits::{Currency, ReservableCurrency,};
use frame_support::traits::{Currency, Get, ReservableCurrency,};
use sp_std::prelude::*;

use frame_system::{self as system, ensure_signed};
use frame_support::dispatch::DispatchResult;
use frame_support::{dispatch::DispatchResult, weights::Weight};
use codec::{Decode, Encode};

use sp_runtime::RuntimeDebug;
Expand Down Expand Up @@ -66,6 +66,11 @@ decl_module! {
type Error = Error<T>;
fn deposit_event() = default;

fn on_runtime_upgrade() -> Weight {
migration::migrate::<T>();
T::MaximumBlockWeight::get()
}

/// Creates a new signaling proposal.
#[weight = 0]
pub fn create_proposal(
Expand Down Expand Up @@ -259,3 +264,19 @@ decl_storage! {
pub ProposalCreationBond get(fn proposal_creation_bond) config(): BalanceOf<T>;
}
}

mod migration {
use super::*;

pub fn migrate<T: Trait>() {
for (hash, _n) in InactiveProposals::<T>::get() {
ProposalOf::<T>::migrate_key_from_blake(hash);
}
for (hash, _n) in ActiveProposals::<T>::get() {
ProposalOf::<T>::migrate_key_from_blake(hash);
}
for (hash, _n) in CompletedProposals::<T>::get() {
ProposalOf::<T>::migrate_key_from_blake(hash);
}
}
}
75 changes: 72 additions & 3 deletions modules/edge-signaling/src/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ use super::*;

use frame_support::{
parameter_types, impl_outer_origin, assert_err, assert_ok, weights::Weight,
traits::{OnFinalize}
traits::{OnFinalize}, Twox128, Blake2_256, StorageHasher,
storage::{unhashed, generator::StorageMap as GeneratorMap},
};
use sp_core::{H256, Blake2Hasher, Hasher};
use sp_runtime::{
Expand All @@ -43,14 +44,18 @@ parameter_types! {
pub const AvailableBlockRatio: Perbill = Perbill::one();
pub const MaximumExtrinsicWeight: Weight = 1024;
}

type AccountId = u64;
type BlockNumber = u64;

impl frame_system::Trait for Test {
type Origin = Origin;
type Index = u64;
type BlockNumber = u64;
type BlockNumber = BlockNumber;
type Call = ();
type Hash = H256;
type Hashing = BlakeTwo256;
type AccountId = u64;
type AccountId = AccountId;
type Lookup = IdentityLookup<Self::AccountId>;
type Header = Header;
type Event = ();
Expand Down Expand Up @@ -92,6 +97,7 @@ impl Trait for Test {

pub type Balances = pallet_balances::Module<Test>;
pub type System = frame_system::Module<Test>;
pub type Voting = voting::Module<Test>;
pub type Signaling = Module<Test>;

const BOND: u128 = 10;
Expand Down Expand Up @@ -553,4 +559,67 @@ fn propose_multichoice_should_work() {
})
);
});
}

#[test]
fn change_hasher_migration() {
mod deprecated {
use sp_std::prelude::*;

use codec::{Encode, Decode};
use frame_support::{decl_module, decl_storage};

use crate::{Trait, ProposalRecord};

decl_module! {
pub struct Module<T: Trait> for enum Call where origin: T::Origin { }
}
decl_storage! {
trait Store for Module<T: Trait> as Signaling {
pub ProposalOf get(fn proposal_of): map hasher(opaque_blake2_256)
T::Hash => Option<ProposalRecord<T::AccountId, T::BlockNumber>>;
}
}
}

new_test_ext().execute_with(|| {
System::set_block_number(1);
// build proposal, vote and record
let public = get_test_key();
let (title, proposal) = generate_proposal();
let hash = build_proposal_hash(public, &proposal);
let outcomes = vec![YES_VOTE, NO_VOTE];
let index = ProposalCount::get();
let vote_id = Voting::create_vote(
public.clone(),
VoteType::Binary,
false, // not commit-reveal
TallyType::OneCoin,
outcomes,
).expect("Voting::create_vote failed");
let transition_time = System::block_number() + Signaling::voting_length();
let record = ProposalRecord {
index: index,
author: public.clone(),
stage: VoteStage::PreVoting,
transition_time: transition_time,
title: title.to_vec(),
contents: proposal.to_vec(),
vote_id: vote_id,
};
// insert the record with the old hasher
deprecated::ProposalOf::<Test>::insert(hash, &record);
InactiveProposals::<Test>::mutate(|proposals| proposals.push((hash, transition_time)));
assert!(
Signaling::proposal_of(hash).is_none(),
"proposal should not (yet) be available with the new hasher"
);
// do the migration
crate::migration::migrate::<Test>();
let maybe_prop = Signaling::proposal_of(hash);
// check that it was successfull
assert!(maybe_prop.is_some());
let prop = maybe_prop.unwrap();
assert_eq!(prop, record);
});
}
26 changes: 13 additions & 13 deletions modules/edge-treasury-reward/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@ edition = "2018"
serde = { version = "1.0", default-features = false, optional = true }
safe-mix = { version = "1.0", default-features = false }
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-staking = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-treasury = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-staking = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-treasury = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }

[dev-dependencies]
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-staking = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-staking-reward-curve = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-session = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-timestamp = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-staking = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-staking-reward-curve = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-session = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-timestamp = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }

[features]
default = ["std"]
Expand Down
14 changes: 7 additions & 7 deletions modules/edge-voting/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ edition = "2018"
serde = { version = "1.0", default-features = false, optional = true }
safe-mix = { version = "1.0", default-features = false }
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-std = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-runtime = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-support = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
frame-system = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
pallet-balances = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }

[dev-dependencies]
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "time-travel", default-features = false }
sp-io = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }
sp-core = { git = "https://github.com/hicommonwealth/substrate.git", branch = "apopiak-migrations", default-features = false }

[features]
default = ["std"]
Expand Down
19 changes: 17 additions & 2 deletions modules/edge-voting/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ mod tests;
use sp_std::prelude::*;
use sp_std::result;
use frame_system::{self as system, ensure_signed};
use frame_support::dispatch::DispatchResult;
use frame_support::{dispatch::DispatchResult, traits::Get, weights::Weight};
use codec::{Decode, Encode};

use sp_runtime::RuntimeDebug;
Expand Down Expand Up @@ -110,6 +110,11 @@ decl_module! {
type Error = Error<T>;
fn deposit_event() = default;

fn on_runtime_upgrade() -> Weight {
migration::migrate::<T>();
T::MaximumBlockWeight::get()
}

/// A function for commit-reveal voting schemes that adds a vote commitment.
///
/// A vote commitment is formatted using the native hash function. There
Expand Down Expand Up @@ -290,8 +295,18 @@ decl_event!(
decl_storage! {
trait Store for Module<T: Trait> as Voting {
/// The map of all vote records indexed by id
pub VoteRecords get(fn vote_records): map hasher(twox_64_concat) u64 => Option<VoteRecord<T::AccountId>>;
pub VoteRecords get(fn vote_records): map hasher(twox_64_concat) u64 => Option<VoteRecord<T::AccountId>>;
/// The number of vote records that have been created
pub VoteRecordCount get(fn vote_record_count): u64;
}
}

mod migration {
use super::*;

pub fn migrate<T: Trait>() {
for idx in 0..(VoteRecordCount::get() + 1) {
VoteRecords::<T>::migrate_key_from_blake(idx);
}
}
}
60 changes: 60 additions & 0 deletions modules/edge-voting/src/tests.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,3 +561,63 @@ fn commit_reveal_ranked_choice_vote_should_work() {
assert_ok!(reveal(public, 1, vote.3.to_vec(), Some(SECRET)));
});
}

#[test]
fn change_hasher_migration() {
mod deprecated {
use sp_std::prelude::*;

use codec::{Encode, Decode};
use frame_support::{decl_module, decl_storage};

use crate::{Trait, VoteRecord};

decl_module! {
pub struct Module<T: Trait> for enum Call where origin: T::Origin { }
}
decl_storage! {
trait Store for Module<T: Trait> as Voting {
pub VoteRecords get(fn vote_records): map hasher(opaque_blake2_256)
u64 => Option<VoteRecord<T::AccountId>>;
}
}
}

new_test_ext().execute_with(|| {
System::set_block_number(1);
// build vote record
let public = get_test_key();
let yes_vote: [u8; 32] = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1];
let no_vote: [u8; 32] = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
let outcomes = vec![yes_vote, no_vote];
let id = VoteRecordCount::get() + 1;
let record = VoteRecord {
id: id,
commitments: vec![],
reveals: vec![],
outcomes: outcomes,
data: VoteData {
initiator: public.clone(),
stage: VoteStage::PreVoting,
vote_type: VoteType::Binary,
tally_type: TallyType::OneCoin,
is_commit_reveal: false,
},
};

// insert the record with the old hasher
deprecated::VoteRecords::<Test>::insert(id, &record);
VoteRecordCount::mutate(|i| *i += 1);
assert!(
Voting::vote_records(id).is_none(),
"proposal should not (yet) be available with the new hasher"
);
// do the migration
crate::migration::migrate::<Test>();
let maybe_vote = Voting::vote_records(id);
// check that it was successfull
assert!(maybe_vote.is_some());
let vote = maybe_vote.unwrap();
assert_eq!(vote, record);
});
}
Loading

0 comments on commit 0a01a69

Please sign in to comment.