Overcomplicated first approach to defining migrations.

microrm-macros/src/schema.rs

@@ -45,25 +45,14 @@ pub fn derive(tokens: proc_macro::TokenStream) -> proc_macro::TokenStream {
         let item_type = &field.1;
 
         quote! {
-            <#item_type as ::microrm::schema::DatabaseItem>::accept_item_visitor(v);
-        }
-    });
-
-    let build_method = items.iter().map(|field| {
-        let item_name = &field.0;
-        let item_type = type_to_expression_context_type(&field.1);
-        quote! {
-            #item_name : #item_type :: build()
+            <#item_type as Default>::default().accept_item_visitor(v);
         }
     });
 
     quote! {
         impl ::microrm::schema::Schema for #db_ident {
-            fn build() -> Self where Self: Sized {
-                Self { #(#build_method),* }
-            }
-
-            fn accept_item_visitor(v: &mut impl ::microrm::schema::DatabaseItemVisitor) {
+            fn accept_item_visitor(&self, v: &mut impl ::microrm::schema::DatabaseItemVisitor) {
+                use ::microrm::schema::DatabaseItem;
                 #(#visit_items)*
             }
         }

microrm/src/db.rs

@@ -312,12 +312,6 @@ impl<'l> From<&'l String> for ConnectionPoolConfig<'l> {
     }
 }
 
-/*impl ConnectionLeaser for ConnectionPool {
-    fn lease<'s>(&'s self) -> impl AsMut<ConnectionLease<'s>> {
-        self.acquire().expect("implicit lease acquisition failure")
-    }
-}*/
-
 #[derive(Clone)]
 /// Multithreading-safe database connection pool.
 pub struct ConnectionPool {
@@ -352,14 +346,16 @@ impl ConnectionPool {
 
 pub(crate) struct Transaction<'l, 'd: 'l> {
     db: &'l mut ConnectionLease<'d>,
+    name: &'l str,
     committed: bool,
 }
 
 impl<'l, 'd: 'l> Transaction<'l, 'd> {
-    pub fn new(db: &'l mut ConnectionLease<'d>) -> DBResult<Self> {
-        db.execute_raw_sql("BEGIN TRANSACTION")?;
+    pub fn new(db: &'l mut ConnectionLease<'d>, name: &'l str) -> DBResult<Self> {
+        db.execute_raw_sql(format!("SAVEPOINT '{name}'"))?;
         Ok(Self {
             db,
+            name,
             committed: false,
         })
     }
@@ -371,7 +367,8 @@ impl<'l, 'd: 'l> Transaction<'l, 'd> {
     pub fn commit(mut self) -> DBResult<()> {
         self.committed = true;
 
-        self.db.execute_raw_sql("COMMIT")
+        self.db
+            .execute_raw_sql(format!("RELEASE SAVEPOINT '{}'", self.name))
     }
 }
 
@@ -383,12 +380,6 @@ impl<'l, 'd: 'l> Drop for Transaction<'l, 'd> {
     }
 }
 
-/*impl<'l> ConnectionLeaser for Transaction<'l> {
-    fn lease<'s>(&'s self) -> impl AsRef<ConnectionLease<'s>> {
-        self.db
-    }
-}*/
-
 struct Statement {
     #[allow(unused)]
     sqlite: *mut sq::sqlite3,

microrm/src/glue.rs

@@ -18,7 +18,7 @@ use crate::{
 impl<T: Entity> Stored<T> {
     /// Synchronize the wrapped value with the corresponding database row.
     pub fn sync(&mut self, lease: &mut ConnectionLease) -> DBResult<()> {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_sync")?;
         query::base_queries::update_entity(txn.lease(), self)?;
         txn.commit()
     }
@@ -53,14 +53,14 @@ impl<'a, T: Entity> Queryable for &'a IDMap<T> {
 
 impl<T: Entity> Insertable<T> for IDMap<T> {
     fn insert(&self, lease: &mut ConnectionLease, value: T) -> DBResult<T::ID> {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_insert")?;
         let out = query::base_queries::insert(txn.lease(), &value)?;
         txn.commit()?;
         Ok(out)
     }
 
     fn insert_and_return(&self, lease: &mut ConnectionLease, value: T) -> DBResult<Stored<T>> {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_insert_and_return")?;
         let out = query::base_queries::insert_and_return(txn.lease(), value)?;
         txn.commit()?;
         Ok(out)

microrm/src/lib.rs

@@ -3,18 +3,16 @@
 //! Unlike many fancier ORM systems, microrm is designed to be lightweight, both in terms of
 //! runtime overhead and developer LoC. By necessity, it sacrifices flexibility towards these
 //! goals, and so can be thought of as more opinionated than, say,
-//! [SeaORM](https://www.sea-ql.org/SeaORM/) or [Diesel](https://diesel.rs/). Major limitations of
-//! microrm are:
-//! - lack of database migration support
-//! - limited vocabulary for describing object-to-object relations
-//!
-//! There are four externally-facing components in microrm:
-//! - Object modelling (via the [`Datum`](schema/datum/trait.Datum.html) and
-//!   [`Entity`](schema/entity/trait.Entity.html) traits)
-//! - Database querying (via [`Queryable`](prelude/trait.Queryable.html),
-//!   [`RelationInterface`](prelude/trait.RelationInterface.html) and
-//!   [`Insertable`](prelude/trait.Insertable.html) traits)
-//! - Migrations (TODO)
+//! [SeaORM](https://www.sea-ql.org/SeaORM/) or [Diesel](https://diesel.rs/). The major limitation of
+//! microrm is somewhat limited vocabulary for describing object-to-object relations, though it is
+//! powerful enough for most usecases where sqlite is appropriate.
+//!
+//! There are three externally-facing components in microrm:
+//! - Schema modelling (mostly by the [`Datum`](schema/datum/trait.Datum.html) and
+//!   [`Entity`](schema/entity/trait.Entity.html) traits) and migrations (see
+//!   [`schema::migration::Migrator`]
+//! - Database querying (via [`query::Queryable`], [`query::RelationInterface`] and
+//!   [`query::Insertable`] traits)
 //! - Command-line interface generation via the [`clap`](https://docs.rs/clap/latest/clap/) crate
 //!   (see [`cli::Autogenerate`] and [`cli::EntityInterface`]; requires the optional crate feature `clap`)
 //!
@@ -232,8 +230,7 @@ extern crate self as microrm;
 
 /// SQLite database interaction functions.
 pub mod db;
-pub mod migration;
-mod query;
+pub mod query;
 pub mod schema;
 
 mod glue;

microrm/src/migration.rs

@@ -1,20 +0,0 @@
-#![allow(missing_docs)]
-
-use crate::schema::entity::EntityList;
-
-mod new;
-
-pub use new::New;
-
-pub trait Migration {
-    type OldEntities: EntityList;
-    type NewEntities: EntityList;
-}
-
-pub struct Migrator {}
-
-impl Migrator {
-    pub fn new() -> Self {
-        todo!()
-    }
-}

microrm/src/query.rs

@@ -1,3 +1,7 @@
+//! Database query interface.
+//!
+//!
+
 use itertools::Itertools;
 
 use crate::{
@@ -59,7 +63,7 @@ pub(crate) enum QueryPart {
 }
 
 #[derive(Debug)]
-pub struct Query<'l> {
+pub(crate) struct Query<'l> {
     parts: HashMap<QueryPart, Vec<QueryPartData<'l>>>,
 }
 
@@ -221,7 +225,7 @@ pub trait RelationInterface {
         let rdata = self.get_data()?;
         let an = RelationNames::collect::<Self>(self)?;
 
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_connect_to")?;
 
         base_queries::do_connect::<Self::RemoteEntity>(txn.lease(), rdata, an, remote_id)?;
 
@@ -240,7 +244,7 @@ pub trait RelationInterface {
         let rdata = self.get_data()?;
         let an = RelationNames::collect::<Self>(self)?;
 
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_disconnect_from")?;
 
         // second, add to the relation table
         txn.lease().with_prepared(
@@ -294,7 +298,7 @@ impl<AI: RelationInterface> Insertable<AI::RemoteEntity> for AI {
         let rdata = self.get_data()?;
         let an = RelationNames::collect::<Self>(self)?;
 
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_insert")?;
 
         // so first, into the remote table
         let remote_id = base_queries::insert(txn.lease(), &value)?;
@@ -321,7 +325,7 @@ impl<AI: RelationInterface> Insertable<AI::RemoteEntity> for AI {
         let rdata = self.get_data()?;
         let an = RelationNames::collect::<Self>(self)?;
 
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_insert_and_return")?;
 
         // so first, into the remote table
         let remote = base_queries::insert_and_return(txn.lease(), value)?;
@@ -363,7 +367,7 @@ pub trait Queryable: Clone {
     where
         Self: Sized,
     {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_count")?;
         struct CountTag;
         let out = txn.lease().with_prepared(
             std::any::TypeId::of::<(Self::StaticVersion, CountTag)>(),
@@ -397,7 +401,7 @@ pub trait Queryable: Clone {
     where
         Self: Sized,
     {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_get")?;
         struct GetTag;
         let out = txn.lease().with_prepared(
             std::any::TypeId::of::<(Self::StaticVersion, GetTag)>(),
@@ -421,7 +425,7 @@ pub trait Queryable: Clone {
     where
         Self: Sized,
     {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_get_ids")?;
         struct GetIDTag;
         let out = txn.lease().with_prepared(
             std::any::TypeId::of::<(Self::StaticVersion, GetIDTag)>(),
@@ -451,7 +455,7 @@ pub trait Queryable: Clone {
     where
         Self: Sized,
     {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_delete")?;
         struct DeleteTag;
         txn.lease().with_prepared(
             std::any::TypeId::of::<(Self::StaticVersion, DeleteTag)>(),
@@ -484,7 +488,7 @@ pub trait Queryable: Clone {
     where
         Self: Sized,
     {
-        let mut txn = Transaction::new(lease)?;
+        let mut txn = Transaction::new(lease, "_microrm_remove")?;
         struct DeleteTag;
         let out = txn.lease().with_prepared(
             std::any::TypeId::of::<(Self::StaticVersion, DeleteTag)>(),

microrm/src/schema.rs

@@ -28,6 +28,8 @@ pub mod relation;
 /// Types related to indexes.
 pub mod index;
 
+pub mod migration;
+
 mod build;
 mod collect;
 pub(crate) mod meta;
@@ -284,7 +286,7 @@ impl<T: Entity> IDMap<T> {
 }
 
 impl<E: Entity> DatabaseItem for IDMap<E> {
-    fn accept_item_visitor(visitor: &mut impl DatabaseItemVisitor) {
+    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor) {
         visitor.visit_idmap::<E>();
     }
 }
@@ -292,7 +294,7 @@ impl<E: Entity> DatabaseItem for IDMap<E> {
 /// Represents a logical item in a database schema, be it an index, table, or logical grouping.
 pub trait DatabaseItem {
     /// Accept an entity visitor for entity discovery.
-    fn accept_item_visitor(visitor: &mut impl DatabaseItemVisitor);
+    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor);
 }
 
 /// Visitor trait for iterating across the types in a [`Schema`] tree.
@@ -308,13 +310,13 @@ pub trait DatabaseItemVisitor {
 }
 
 /// A root structure for the database specification graph.
-pub trait Schema: Default {
+pub trait Schema {
     /// Install this schema into a database
     fn install(&self, lease: &mut ConnectionLease) -> DBResult<()>
     where
         Self: Sized,
     {
-        let schema = build::collect_from_database::<Self>();
+        let schema = build::collect_from_database(self);
         match schema.check(lease) {
             // schema checks out
             Some(true) => {},
@@ -328,22 +330,17 @@ pub trait Schema: Default {
         Ok(())
     }
 
-    #[doc(hidden)]
-    fn build() -> Self
-    where
-        Self: Sized;
-
     /// Accept a visitor for iteration.
-    fn accept_item_visitor(visitor: &mut impl DatabaseItemVisitor)
+    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
     where
         Self: Sized;
 }
 
 impl<S: Schema> DatabaseItem for S {
-    fn accept_item_visitor(visitor: &mut impl DatabaseItemVisitor)
+    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
     where
         Self: Sized,
     {
-        <S as Schema>::accept_item_visitor(visitor);
+        <Self as Schema>::accept_item_visitor(self, visitor);
     }
 }

microrm/src/schema/build.rs

@@ -1,5 +1,7 @@
+use std::collections::HashMap;
+
 use crate::{
-    db::ConnectionLease,
+    db::{ConnectionLease, Transaction},
     query::{Insertable, Queryable},
     schema::{
         collect::{EntityStateContainer, PartType},
@@ -9,6 +11,8 @@ use crate::{
     DBResult,
 };
 
+use sha2::Digest;
+
 use super::Schema;
 
 #[derive(Debug)]
@@ -92,13 +96,27 @@ impl IndexInfo {
     }
 }
 
+#[derive(Debug)]
 pub(crate) struct DatabaseSchema {
-    signature: u64,
-    queries: Vec<String>,
+    signature: String,
+    table_queries: HashMap<String, String>,
+    index_queries: HashMap<String, String>,
 }
 
 impl DatabaseSchema {
-    const SCHEMA_SIGNATURE_KEY: &'static str = "schema_signature";
+    pub(crate) const SCHEMA_SIGNATURE_KEY: &'static str = "schema_signature";
+
+    pub fn signature(&self) -> &str {
+        self.signature.as_str()
+    }
+
+    pub fn table_queries(&self) -> &HashMap<String, String> {
+        &self.table_queries
+    }
+
+    pub fn index_queries(&self) -> &HashMap<String, String> {
+        &self.index_queries
+    }
 
     /// Three possible results:
     /// - yes, this is a schema match (true)
@@ -106,7 +124,7 @@ impl DatabaseSchema {
     /// - there is no schema that we know of (None)
     pub fn check(&self, lease: &mut ConnectionLease) -> Option<bool> {
         // attempt to use connection as a MetadataDB database
-        let metadb = meta::MetadataDB::build();
+        let metadb = meta::MetadataDB::default();
 
         // check to see if the signature exists and matches
         metadb
@@ -115,39 +133,47 @@ impl DatabaseSchema {
             .get(lease)
             .ok()
             .flatten()
-            .map(|kv| kv.value.parse::<u64>().unwrap_or(0) == self.signature)
+            .map(|kv| kv.value == self.signature)
+        // .map(|kv| kv.value.parse::<u64>().unwrap_or(0) == self.signature)
     }
 
     pub fn create(&self, lease: &mut ConnectionLease) -> DBResult<()> {
-        lease.execute_raw_sql("BEGIN TRANSACTION")?;
-        for query in self.queries.iter() {
-            log::trace!("Running creation query {query}");
-            lease.execute_raw_sql(query)?;
+        let mut txn = Transaction::new(lease, "_microrm_create")?;
+        for query in self.table_queries.iter() {
+            log::trace!("Running creation query {}", query.1);
+            txn.lease().execute_raw_sql(query.1)?;
+        }
+        for query in self.index_queries.iter() {
+            log::trace!("Running creation query {}", query.1);
+            txn.lease().execute_raw_sql(query.1)?;
         }
 
         // attempt to use connection as a MetadataDB database
-        let metadb = meta::MetadataDB::build();
+        let metadb = meta::MetadataDB::default();
 
-        for query in collect_from_database::<meta::MetadataDB>().queries.iter() {
-            lease.execute_raw_sql(query)?;
+        for query in collect_from_database(&meta::MetadataDB::default())
+            .table_queries
+            .iter()
+        {
+            txn.lease().execute_raw_sql(query.1)?;
         }
 
-        lease.execute_raw_sql("COMMIT")?;
-
         // store signature
         metadb.metastore.insert(
-            lease,
+            txn.lease(),
             meta::MicrormMeta {
                 key: Self::SCHEMA_SIGNATURE_KEY.into(),
                 value: format!("{}", self.signature),
             },
         )?;
 
+        txn.commit()?;
+
         Ok(())
     }
 }
 
-pub(crate) fn collect_from_database<DB: Schema>() -> DatabaseSchema {
+pub(crate) fn collect_from_database<DB: Schema>(schema: &DB) -> DatabaseSchema {
     struct IV(EntityStateContainer, Vec<IndexInfo>);
 
     impl DatabaseItemVisitor for IV {
@@ -183,7 +209,7 @@ pub(crate) fn collect_from_database<DB: Schema>() -> DatabaseSchema {
 
     let mut iv = IV(EntityStateContainer::default(), Default::default());
 
-    DB::accept_item_visitor(&mut iv);
+    schema.accept_item_visitor(&mut iv);
 
     // now to turn all that into a set of tables
     let mut tables = std::collections::HashMap::new();
@@ -280,13 +306,11 @@ pub(crate) fn collect_from_database<DB: Schema>() -> DatabaseSchema {
         tables.insert(table_name, table);
     }
 
-    // this must be a stable hash function, so we very explicitly want to use a SipHasher with
-    // known parameters
-    #[allow(deprecated)]
-    let mut signature_hasher = std::hash::SipHasher::new();
-    use std::hash::{Hash, Hasher};
+    // this must be a stable hash function, so we use sha2
+    let mut hasher = sha2::Sha256::new();
 
-    let mut queries = vec![];
+    let mut table_queries = vec![];
+    let mut index_queries = vec![];
 
     // create sorted table list
     let mut sorted_tables: Vec<_> = tables.into_iter().collect();
@@ -295,19 +319,23 @@ pub(crate) fn collect_from_database<DB: Schema>() -> DatabaseSchema {
     for (table_name, table) in sorted_tables {
         let create_sql = table.build_creation_query();
 
-        table_name.hash(&mut signature_hasher);
-        create_sql.hash(&mut signature_hasher);
-        queries.push(create_sql);
+        hasher.update(&table_name);
+        hasher.update(&create_sql);
+        table_queries.push((table_name, create_sql));
     }
 
     for iinfo in iv.1.into_iter() {
-        iinfo.hash(&mut signature_hasher);
-        queries.push(iinfo.build_creation_query());
+        hasher.update(&iinfo.table_name);
+        let query = iinfo.build_creation_query();
+        hasher.update(&query);
+        index_queries.push((iinfo.table_name, query));
     }
 
-    let signature = signature_hasher.finish();
-
-    log::trace!("Schema signature: {signature:x}");
+    let digest = hasher.finalize();
 
-    DatabaseSchema { signature, queries }
+    DatabaseSchema {
+        signature: digest.into_iter().map(|v| format!("{:02x}", v)).collect(),
+        table_queries: HashMap::from_iter(table_queries.into_iter()),
+        index_queries: HashMap::from_iter(index_queries.into_iter()),
+    }
 }

microrm/src/schema/datum.rs

@@ -104,16 +104,23 @@ pub trait DatumDiscriminatorRef {
 
 /// A fixed-length list of EntityDatums, usually a tuple.
 pub trait DatumList: Clone {
+    /// Recursion control: true if this is an empty list.
     const IS_EMPTY: bool = false;
+    /// The first [`Datum`] in this list, or a sentinel for an empty list.
     type ListHead: Datum;
+    /// Another `DatumList` containing all but the first datums in this list.
     type ListTail: DatumList;
-
+    /// A version of this [`DatumList`] where each element is replaced by a reference to this
+    /// [`DatumList`], ie turning `&(T1, T2)` into `(&T1, &T2)`.
     type RefList<'l>: DatumList
     where
         Self: 'l;
 
+    /// Get an instance of the first element of this list.
     fn list_head(&self) -> &Self::ListHead;
-    fn list_tail(&self) -> Self::ListTail; // { todo!() }
+    /// Create a copy of the tail of this list.
+    fn list_tail(&self) -> Self::ListTail;
+    /// Create a referenced version of the current list; see [`Self::RefList`].
     fn into_ref<'l>(&'l self) -> Self::RefList<'l> {
         todo!()
     }

microrm/src/schema/entity.rs

@@ -140,7 +140,16 @@ pub trait EntityVisitor {
 
 /// A list of [`Entity`] types.
 pub trait EntityList {
-    const LEN: usize;
+    /// The first entity in this list, or a sentinel for an empty list.
+    type ListHead: Entity;
+    /// An [`EntityList`] instance containing all entities but the first.
+    type ListTail: EntityList;
 
+    /// For recursion control: is this an empty list?
+    const IS_EMPTY: bool = false;
+
+    /// Accept a visitor to iterate across all entities in this list.
     fn accept_visitor<EV: EntityVisitor>(visitor: &mut EV);
 }
+
+mod entity_list;

microrm/src/schema/entity/entity_list.rs

@@ -0,0 +1,51 @@
+use super::{Entity, EntityList, EntityVisitor};
+
+#[derive(microrm_macros::Entity)]
+pub struct FakeEntity {}
+
+impl EntityList for () {
+    type ListHead = FakeEntity;
+    type ListTail = ();
+    const IS_EMPTY: bool = true;
+
+    fn accept_visitor<EV: EntityVisitor>(_visitor: &mut EV) {}
+}
+
+impl<E0: Entity> EntityList for E0 {
+    type ListHead = E0;
+    type ListTail = ();
+    fn accept_visitor<EV: EntityVisitor>(visitor: &mut EV) {
+        visitor.visit::<E0>();
+    }
+}
+
+macro_rules! entity_list {
+    ( $e0:ident, $($e:ident),* ) => {
+        impl< $e0: Entity, $( $e : Entity ),*> EntityList for ( $e0, $( $e ),*) {
+            type ListHead = $e0;
+            #[allow(unused_parens)]
+            type ListTail = ( $( $e ),* );
+            fn accept_visitor<EV: EntityVisitor>(visitor: &mut EV) {
+                visitor.visit::< $e0 >();
+                $( visitor.visit::< $e >(); )*
+            }
+        }
+    }
+}
+
+entity_list!(E0,);
+entity_list!(E0, E1);
+entity_list!(E0, E1, E2);
+entity_list!(E0, E1, E2, E3);
+entity_list!(E0, E1, E2, E3, E4);
+entity_list!(E0, E1, E2, E3, E4, E5);
+entity_list!(E0, E1, E2, E3, E4, E5, E6);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12, E13);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12, E13, E14);
+entity_list!(E0, E1, E2, E3, E4, E5, E6, E7, E8, E9, E10, E11, E12, E13, E14, E15);

microrm/src/schema/index.rs

@@ -44,7 +44,7 @@ impl<E: Entity, EPL: EntityPartList<Entity = E>> Index<E, EPL> {
 }
 
 impl<E: Entity, EPL: EntityPartList<Entity = E>> super::DatabaseItem for Index<E, EPL> {
-    fn accept_item_visitor(visitor: &mut impl super::DatabaseItemVisitor) {
+    fn accept_item_visitor(&self, visitor: &mut impl super::DatabaseItemVisitor) {
         visitor.visit_index::<E, EPL>();
     }
 }

microrm/src/schema/migration.rs

@@ -0,0 +1,164 @@
+// this is overcomplicated.
+// just give a function that has access to each entity type and allow it to copy whatever it likes?
+
+#![allow(missing_docs)]
+
+use std::collections::HashSet;
+
+use crate::{
+    db::ConnectionLease,
+    query::Queryable,
+    schema::{entity::EntityList, Schema},
+    DBResult,
+};
+
+mod new;
+
+pub use new::New;
+use new::WrapSchema;
+
+use super::{
+    build::{collect_from_database, DatabaseSchema},
+    meta::MetadataDB,
+};
+
+pub trait Migration: 'static + Default {
+    type OldSchema: Schema + Default;
+    type NewSchema: Schema + Default;
+}
+
+pub trait MigrationRef {
+    // fn with_new_schema(&self, changed: HashSet<String>) -> Box<&dyn Schema>;
+    fn collect_intermediate_schema(&self, towrap: HashSet<String>) -> Box<&dyn Schema>;
+}
+impl<M: Migration> MigrationRef for M {
+    fn collect_intermediate_schema(&self, towrap: HashSet<String>) -> Box<&dyn Schema> {
+        let wrap_schema = WrapSchema::<M::NewSchema>::new(M::NewSchema::default(), towrap);
+
+        println!("collection: {:?}", collect_from_database(&wrap_schema));
+        todo!()
+    }
+}
+
+struct MigrationEntry {
+    old_schema: DatabaseSchema,
+    old_signature: String,
+    new_schema: DatabaseSchema,
+    new_signature: String,
+    migration: Box<dyn MigrationRef>,
+}
+
+#[derive(microrm_macros::Schema, Default)]
+struct EmptySchema {}
+
+pub struct Migrator {
+    base_schema: DatabaseSchema,
+    migrations: Vec<MigrationEntry>,
+}
+
+impl Migrator {
+    pub fn new<Base: Schema + Default>() -> Self {
+        Self {
+            base_schema: collect_from_database(&Base::default()),
+            migrations: vec![],
+        }
+    }
+
+    pub fn add<M: Migration>(&mut self) {
+        let old_schema = collect_from_database(&M::OldSchema::default());
+        let new_schema = collect_from_database(&M::NewSchema::default());
+        self.migrations.push(MigrationEntry {
+            old_signature: old_schema.signature().into(),
+            new_signature: new_schema.signature().into(),
+            old_schema,
+            new_schema,
+            migration: Box::new(M::default()),
+        });
+    }
+
+    pub fn apply(&self, lease: &mut ConnectionLease) -> DBResult<()> {
+        log::debug!("Applying migrations");
+        // try getting data from the metaschema
+        let existing_schema = MetadataDB::default()
+            .metastore
+            .keyed(DatabaseSchema::SCHEMA_SIGNATURE_KEY)
+            .get(lease)
+            .ok()
+            .flatten()
+            .map(|v| v.wrapped().value);
+
+        // early-out: do we match the latest migration?
+        if existing_schema.is_some()
+            && self.migrations.last().map(|v| &v.new_signature) == existing_schema.as_ref()
+        {
+            log::trace!("no migrations to apply");
+            return Ok(());
+        }
+
+        // early-out: do we have any existing schema at all?
+        if existing_schema.is_none() {
+            log::trace!("no existing schema found");
+            return self.create_from_scratch(lease);
+        }
+
+        // we have work to do, time to get started
+        let to_apply = self
+            .migrations
+            .iter()
+            .skip_while(|m| Some(&m.old_signature) != existing_schema.as_ref());
+
+        for migration in to_apply {
+            log::debug!(
+                "Applying migration from {}... to {}...",
+                &migration.old_signature[0..8],
+                &migration.new_signature[0..8]
+            );
+
+            // what entities have *changed* between the two schemas? exclude new/removed
+            let table_differences = migration
+                .old_schema
+                .table_queries()
+                .iter()
+                .map(|(tname, oq)| {
+                    Some((
+                        tname,
+                        oq,
+                        migration.new_schema.table_queries().get(tname.as_str())?,
+                    ))
+                })
+                .flatten()
+                .collect::<Vec<_>>();
+
+            let changed = table_differences
+                .iter()
+                .map(|v| v.0)
+                .cloned()
+                .collect::<HashSet<_>>();
+
+            let nschema = migration.migration.collect_intermediate_schema(changed);
+
+            // collect_from_database(nschema.as_ref());
+
+            log::trace!("nschema built");
+
+            println!("table_differences: {table_differences:?}");
+            todo!()
+        }
+
+        log::info!("Database migrations applied");
+
+        Ok(())
+    }
+
+    fn create_from_scratch(&self, lease: &mut ConnectionLease) -> DBResult<()> {
+        let latest_schema = self
+            .migrations
+            .last()
+            .map(|m| &m.new_schema)
+            .unwrap_or(&self.base_schema);
+
+        latest_schema.create(lease)?;
+
+        Ok(())
+    }
+}

microrm/src/migration/new.rs → microrm/src/schema/migration/new.rs

@@ -1,4 +1,9 @@
-use std::{collections::HashMap, hash::Hash, sync::Mutex};
+use std::{
+    collections::{HashMap, HashSet},
+    hash::Hash,
+    marker::PhantomData,
+    sync::Mutex,
+};
 
 use crate::{
     db::{StatementContext, StatementRow},
@@ -6,10 +11,15 @@ use crate::{
         datum::{ConcreteDatum, Datum, DatumList},
         entity::{Entity, EntityID, EntityPart, EntityPartList, EntityPartVisitor},
         relation::RelationData,
+        DatabaseItemVisitor, Schema,
     },
     DBResult,
 };
 
+// ----------------------------------------------------------------------
+// Entity-related New<> adapter
+// ----------------------------------------------------------------------
+
 pub struct NewID<OE: Entity> {
     value: OE::ID,
 }
@@ -257,3 +267,45 @@ impl<E: Entity> Entity for New<E> {
         self.value.accept_part_visitor_mut(&mut pv);
     }
 }
+
+// ----------------------------------------------------------------------
+// Schema-level adaptation
+// ----------------------------------------------------------------------
+pub struct WrapSchema<S: Schema> {
+    base: S,
+    towrap: HashSet<String>,
+}
+
+impl<S: Schema> WrapSchema<S> {
+    pub fn new(base: S, towrap: HashSet<String>) -> Self {
+        Self { base, towrap }
+    }
+}
+
+impl<S: Schema> Schema for WrapSchema<S> {
+    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
+    where
+        Self: Sized,
+    {
+        struct InterceptingVisitor<'a, S: Schema, V: DatabaseItemVisitor>(
+            &'a WrapSchema<S>,
+            &'a mut V,
+        );
+        impl<'a, S: Schema, V: DatabaseItemVisitor> DatabaseItemVisitor for InterceptingVisitor<'a, S, V> {
+            fn visit_idmap<T: Entity>(&mut self)
+            where
+                Self: Sized,
+            {
+            }
+
+            fn visit_index<T: Entity, PL: EntityPartList<Entity = T>>(&mut self)
+            where
+                Self: Sized,
+            {
+            }
+        }
+
+        let mut iv = InterceptingVisitor(self, visitor);
+        self.base.accept_item_visitor(&mut iv);
+    }
+}

microrm/tests/common/mod.rs

@@ -1,13 +1,9 @@
 use microrm::prelude::*;
 
-pub fn open_test_db_helper<DB: Schema>(identifier: &'static str) -> (microrm::ConnectionPool, DB) {
-    let path = format!(
-        "{tmpdir}/{identifier}.db",
-        tmpdir = std::env!("CARGO_TARGET_TMPDIR"),
-    );
-    let path = path.replace("::", "_");
-    let _ = std::fs::remove_file(path.as_str());
-    let pool = microrm::ConnectionPool::new(path.as_str()).expect("couldn't connect to database");
+pub fn open_test_db_helper<DB: Schema + Default>(
+    identifier: &'static str,
+) -> (microrm::ConnectionPool, DB) {
+    let pool = open_test_pool_helper(identifier);
     let db = DB::default();
 
     db.install(&mut pool.acquire().expect("couldn't acquire lease"))
@@ -16,6 +12,23 @@ pub fn open_test_db_helper<DB: Schema>(identifier: &'static str) -> (microrm::Co
     (pool, db)
 }
 
+pub fn open_test_pool_helper(identifier: &'static str) -> microrm::ConnectionPool {
+    let path = test_db_path(identifier);
+    log::info!("path to test database: {path}");
+    let _ = std::fs::remove_file(path.as_str());
+    let pool = microrm::ConnectionPool::new(path.as_str()).expect("couldn't connect to database");
+
+    pool
+}
+
+fn test_db_path(identifier: &'static str) -> String {
+    let path = format!(
+        "{tmpdir}/{identifier}.db",
+        tmpdir = std::env!("CARGO_TARGET_TMPDIR"),
+    );
+    path.replace("::", "_")
+}
+
 mod macros {
     macro_rules! open_test_db {
         () => {{
@@ -24,7 +37,16 @@ mod macros {
         }};
     }
 
+    macro_rules! open_test_pool {
+        () => {{
+            use stdext::function_name;
+            $crate::common::open_test_pool_helper(function_name!())
+        }};
+    }
+
     pub(crate) use open_test_db;
+    pub(crate) use open_test_pool;
 }
 
 pub(crate) use macros::open_test_db;
+pub(crate) use macros::open_test_pool;

microrm/tests/migration.rs

@@ -0,0 +1,130 @@
+use microrm::{
+    prelude::*,
+    schema::migration::{Migration, Migrator},
+    ConnectionPool,
+};
+use test_log::test;
+
+mod common;
+
+mod s1 {
+    use microrm::prelude::*;
+    #[derive(Entity)]
+    pub struct E {
+        pub val: String,
+    }
+
+    #[derive(Default, Schema)]
+    pub struct S {
+        pub emap: microrm::IDMap<E>,
+    }
+}
+use s1::{E as E1, S as S1};
+
+mod s2 {
+    use microrm::prelude::*;
+    #[derive(PartialEq, Entity)]
+    pub struct E {
+        pub val: String,
+        pub other_val: String,
+    }
+
+    #[derive(Default, Schema)]
+    pub struct S {
+        pub emap: microrm::IDMap<E>,
+    }
+}
+use s2::{E as E2, S as S2};
+
+#[derive(Default)]
+struct S1S2;
+
+impl Migration for S1S2 {
+    type OldSchema = S1;
+    type NewSchema = S2;
+}
+
+#[test]
+fn from_empty() {
+    let cpool = common::open_test_pool!();
+    let mut lease = cpool.acquire().unwrap();
+
+    let mut migrator = Migrator::new::<S1>();
+    migrator.add::<S1S2>();
+
+    migrator.apply(&mut lease).unwrap();
+}
+
+#[test]
+fn multistep_schema() {
+    let cpool = common::open_test_pool!();
+    let mut lease = cpool.acquire().unwrap();
+
+    // first apply S1
+    let mut migrator = Migrator::new::<S1>();
+    migrator.apply(&mut lease).unwrap();
+
+    // then apply S1S2
+    migrator.add::<S1S2>();
+    migrator.apply(&mut lease).unwrap();
+
+    S2::default()
+        .emap
+        .insert(
+            &mut lease,
+            E2 {
+                val: String::from("some_val"),
+                other_val: String::from("some_other_val"),
+            },
+        )
+        .unwrap();
+
+    let all_entities = S2::default().emap.get(&mut lease).unwrap();
+    assert_eq!(
+        all_entities
+            .into_iter()
+            .map(|v| v.wrapped())
+            .collect::<Vec<_>>(),
+        vec![E2 {
+            val: String::from("some_val"),
+            other_val: String::from("some_other_val")
+        },]
+    );
+}
+
+#[test]
+fn migrated_data() {
+    let cpool = common::open_test_pool!(); // ConnectionPool::new(":memory:").unwrap();
+    let mut lease = cpool.acquire().unwrap();
+
+    // first apply S1
+    let mut migrator = Migrator::new::<S1>();
+    migrator.apply(&mut lease).unwrap();
+
+    // add some data
+    S1::default()
+        .emap
+        .insert(
+            &mut lease,
+            E1 {
+                val: String::from("some_val"),
+            },
+        )
+        .unwrap();
+
+    // then apply S1S2
+    migrator.add::<S1S2>();
+    migrator.apply(&mut lease).unwrap();
+
+    let all_entities = S2::default().emap.get(&mut lease).unwrap();
+    assert_eq!(
+        all_entities
+            .into_iter()
+            .map(|v| v.wrapped())
+            .collect::<Vec<_>>(),
+        vec![E2 {
+            val: String::from("some_val"),
+            other_val: String::from("migrated_value")
+        },]
+    );
+}