Working very simple schema migration!

microrm-macros/src/entity.rs

@@ -263,7 +263,7 @@ pub fn derive(tokens: proc_macro::TokenStream) -> proc_macro::TokenStream {
 
             fn desc() -> Option<&'static str> { None }
 
-            fn get_datum(from: &Self::Entity) -> &Self::Datum {
+            fn get_datum(_from: &Self::Entity) -> &Self::Datum {
                 unreachable!()
             }
         }

microrm-macros/src/schema.rs

@@ -20,6 +20,7 @@ pub fn derive(tokens: proc_macro::TokenStream) -> proc_macro::TokenStream {
 
     let db_ident = input.ident;
 
+    let item_types = items.iter().map(|field| field.1.clone());
     let visit_items = items.iter().map(|field| {
         let item_type = &field.1;
 
@@ -29,12 +30,15 @@ pub fn derive(tokens: proc_macro::TokenStream) -> proc_macro::TokenStream {
     });
 
     quote! {
-        impl ::microrm::schema::Schema for #db_ident {
+        impl ::microrm::schema::DatabaseItem for #db_ident {
             fn accept_item_visitor(&self, v: &mut impl ::microrm::schema::DatabaseItemVisitor) {
                 use ::microrm::schema::DatabaseItem;
                 #(#visit_items)*
             }
+
+            type Subitems = ( #(#item_types),* , );
         }
+        impl ::microrm::schema::Schema for #db_ident {}
     }
     .into()
 }

microrm/src/query/base_queries.rs

@@ -11,6 +11,93 @@ use crate::{
 
 use super::RelationNames;
 
+struct PartNameVisitor<'a, E: Entity>(&'a mut String, &'a mut String, std::marker::PhantomData<E>);
+impl<'a, E: Entity> EntityPartVisitor for PartNameVisitor<'a, E> {
+    type Entity = E;
+    fn visit<EP: EntityPart>(&mut self) {
+        // if this is a set-relation, then we don't actually want to do anything
+        // with it here; it doesn't have a column
+        if is_relation::<EP>() {
+            return;
+        }
+
+        if !self.0.is_empty() {
+            self.0.push_str(", ");
+            self.1.push_str(", ");
+        }
+        self.0.push('`');
+        self.0.push_str(EP::part_name());
+        self.0.push('`');
+        self.1.push('?');
+    }
+}
+
+struct PartBinder<'a, 'b, E: Entity>(
+    &'a mut StatementContext<'b>,
+    i32,
+    std::marker::PhantomData<E>,
+);
+impl<'a, 'b, E: Entity> EntityPartVisitor for PartBinder<'a, 'b, E> {
+    type Entity = E;
+    fn visit_datum<EP: EntityPart>(&mut self, datum: &EP::Datum) {
+        // skip relations, as with the query preparation above
+        if is_relation::<EP>() {
+            return;
+        }
+
+        datum.bind_to(self.0, self.1);
+        self.1 += 1;
+    }
+}
+
+pub(crate) fn insert_exact<E: Entity>(
+    conn: &mut ConnectionLease,
+    value: &E,
+    id: E::ID,
+) -> DBResult<()> {
+    struct InsertExactQuery<E: Entity>(std::marker::PhantomData<E>);
+
+    conn.with_prepared(
+        std::any::TypeId::of::<InsertExactQuery<E>>(),
+        || {
+            let mut part_names = String::new();
+            let mut placeholders = String::new();
+
+            E::accept_part_visitor(&mut PartNameVisitor(
+                &mut part_names,
+                &mut placeholders,
+                Default::default(),
+            ));
+
+            if part_names.len() == 0 {
+                part_names.clear();
+                part_names.push_str("id");
+                placeholders.clear();
+                placeholders.push_str("?");
+            } else {
+                part_names = format!("`id`, {part_names}");
+                placeholders = format!("?, {placeholders}");
+            }
+
+            format!(
+                "INSERT INTO `{}` ({}) VALUES ({})",
+                E::entity_name(),
+                part_names,
+                placeholders
+            )
+        },
+        |mut ctx| {
+            id.bind_to(&mut ctx, 1);
+            value.accept_part_visitor_ref(&mut PartBinder(&mut ctx, 2, Default::default()));
+
+            ctx.run()?;
+            Ok(())
+        },
+    )?;
+
+    Ok(())
+}
+
 pub(crate) fn insert<E: Entity>(conn: &mut ConnectionLease, value: &E) -> DBResult<E::ID> {
     struct InsertQuery<E: Entity>(std::marker::PhantomData<E>);
 
@@ -19,30 +106,6 @@ pub(crate) fn insert<E: Entity>(conn: &mut ConnectionLease, value: &E) -> DBResu
         || {
             let mut part_names = String::new();
             let mut placeholders = String::new();
-            struct PartNameVisitor<'a, E: Entity>(
-                &'a mut String,
-                &'a mut String,
-                std::marker::PhantomData<E>,
-            );
-            impl<'a, E: Entity> EntityPartVisitor for PartNameVisitor<'a, E> {
-                type Entity = E;
-                fn visit<EP: EntityPart>(&mut self) {
-                    // if this is a set-relation, then we don't actually want to do anything
-                    // with it here; it doesn't have a column
-                    if is_relation::<EP>() {
-                        return;
-                    }
-
-                    if !self.0.is_empty() {
-                        self.0.push_str(", ");
-                        self.1.push_str(", ");
-                    }
-                    self.0.push('`');
-                    self.0.push_str(EP::part_name());
-                    self.0.push('`');
-                    self.1.push('?');
-                }
-            }
 
             E::accept_part_visitor(&mut PartNameVisitor(
                 &mut part_names,
@@ -58,24 +121,6 @@ pub(crate) fn insert<E: Entity>(conn: &mut ConnectionLease, value: &E) -> DBResu
             )
         },
         |mut ctx| {
-            struct PartBinder<'a, 'b, E: Entity>(
-                &'a mut StatementContext<'b>,
-                i32,
-                std::marker::PhantomData<E>,
-            );
-            impl<'a, 'b, E: Entity> EntityPartVisitor for PartBinder<'a, 'b, E> {
-                type Entity = E;
-                fn visit_datum<EP: EntityPart>(&mut self, datum: &EP::Datum) {
-                    // skip relations, as with the query preparation above
-                    if is_relation::<EP>() {
-                        return;
-                    }
-
-                    datum.bind_to(self.0, self.1);
-                    self.1 += 1;
-                }
-            }
-
             value.accept_part_visitor_ref(&mut PartBinder(&mut ctx, 1, Default::default()));
 
             ctx.run()?

microrm/src/schema.rs

@@ -34,6 +34,8 @@ pub(crate) mod meta;
 
 pub mod migration;
 
+mod detail;
+
 // ----------------------------------------------------------------------
 // API types
 // ----------------------------------------------------------------------
@@ -289,18 +291,24 @@ impl<E: Entity> DatabaseItem for IDMap<E> {
     fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor) {
         visitor.visit_idmap::<E>();
     }
+    type Subitems = ();
 }
 
 /// Represents a logical item in a database schema, be it an index, table, or logical grouping.
 pub trait DatabaseItem {
-    /// Accept an entity visitor for entity discovery.
+    /// Accept an entity visitor for (local) entity discovery.
     fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor);
+
+    /// Ordered list of DatabaseItems that are direct children of this item.
+    type Subitems: DatabaseItemList;
 }
 
 /// A special sentinel DatabaseItem for DatabaseItemList.
+#[derive(Default, Debug, Clone, Copy)]
 pub struct SentinelDatabaseItem;
 impl DatabaseItem for SentinelDatabaseItem {
     fn accept_item_visitor(&self, _visitor: &mut impl DatabaseItemVisitor) {}
+    type Subitems = ();
 }
 
 /// Representation of a list of DatabaseItems.
@@ -319,6 +327,11 @@ impl DatabaseItemList for () {
     const EMPTY: bool = true;
 }
 
+impl<DI0: DatabaseItem> DatabaseItemList for (DI0,) {
+    type Head = DI0;
+    type Tail = ();
+}
+
 /// Visitor trait for iterating across the types in a [`Schema`] tree.
 pub trait DatabaseItemVisitor {
     /// Visit an `IDMap<T>` type.
@@ -332,13 +345,13 @@ pub trait DatabaseItemVisitor {
 }
 
 /// A root structure for the database specification graph.
-pub trait Schema: 'static + Default {
+pub trait Schema: 'static + Default + DatabaseItem {
     /// Install this schema into a database
     fn install(&self, lease: &mut ConnectionLease) -> DBResult<()>
     where
         Self: Sized,
     {
-        let schema = build::collect_from_database(self);
+        let schema = build::generate_from_schema::<Self>();
         match schema.check(lease) {
             // schema checks out
             Some(true) => {},
@@ -351,18 +364,4 @@ pub trait Schema: 'static + Default {
         }
         Ok(())
     }
-
-    /// Accept a visitor for iteration.
-    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
-    where
-        Self: Sized;
-}
-
-impl<S: Schema> DatabaseItem for S {
-    fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
-    where
-        Self: Sized,
-    {
-        <Self as Schema>::accept_item_visitor(self, visitor);
-    }
 }

microrm/src/schema/build.rs

@@ -97,13 +97,13 @@ impl IndexInfo {
 }
 
 #[derive(Debug)]
-pub(crate) struct DatabaseSchema {
+pub(crate) struct GeneratedSchema {
     signature: String,
     table_queries: HashMap<String, String>,
     index_queries: HashMap<String, String>,
 }
 
-impl DatabaseSchema {
+impl GeneratedSchema {
     pub(crate) const SCHEMA_SIGNATURE_KEY: &'static str = "schema_signature";
 
     // the following functions are for use in migrations
@@ -155,7 +155,7 @@ impl DatabaseSchema {
         // attempt to use connection as a MetadataDB database
         let metadb = meta::MetadataDB::default();
 
-        for query in collect_from_database(&meta::MetadataDB::default())
+        for query in generate_from_schema::<meta::MetadataDB>()
             .table_queries
             .iter()
         {
@@ -177,7 +177,110 @@ impl DatabaseSchema {
     }
 }
 
-pub(crate) fn collect_from_database<DB: Schema>(schema: &DB) -> DatabaseSchema {
+fn process_state(tables: &mut HashMap<String, TableInfo>, state: &super::collect::EntityState) {
+    let table_name = state.name.to_string();
+    // we may end up visiting duplicate entities; skip them if so
+    if tables.contains_key(&table_name) {
+        return;
+    }
+
+    let mut table = TableInfo::new(table_name.clone());
+    for part in state.parts.iter() {
+        match &part.ty {
+            PartType::Datum(dtype) => table.columns.push(ColumnInfo {
+                name: part.name,
+                ty: dtype.to_string(),
+                fkey: None,
+                unique: part.unique,
+            }),
+            PartType::IDReference(entity_name) => table.columns.push(ColumnInfo {
+                name: part.name,
+                ty: "int".into(),
+                fkey: Some(format!("`{}`(`id`)", entity_name)),
+                unique: part.unique,
+            }),
+            PartType::RelationDomain {
+                table_name: relation_table_name,
+                range_name,
+                injective,
+            } => {
+                let mut relation_table = TableInfo::new(relation_table_name.clone());
+
+                relation_table.columns.push(ColumnInfo {
+                    name: "domain",
+                    ty: "int".into(),
+                    fkey: Some(format!("`{}`(`id`)", table_name)),
+                    unique: false,
+                });
+
+                relation_table.columns.push(ColumnInfo {
+                    name: "range",
+                    ty: "int".into(),
+                    fkey: Some(format!("`{}`(`id`)", range_name)),
+                    unique: *injective,
+                });
+
+                relation_table
+                    .constraints
+                    .push("unique(`range`, `domain`)".to_string());
+
+                tables.insert(relation_table_name.clone(), relation_table);
+            },
+            PartType::RelationRange {
+                table_name: relation_table_name,
+                domain_name,
+                injective,
+            } => {
+                let mut relation_table = TableInfo::new(relation_table_name.clone());
+
+                relation_table.columns.push(ColumnInfo {
+                    name: "domain",
+                    ty: "int".into(),
+                    fkey: Some(format!("`{}`(`id`)", domain_name)),
+                    unique: false,
+                });
+
+                relation_table.columns.push(ColumnInfo {
+                    name: "range",
+                    ty: "int".into(),
+                    fkey: Some(format!("`{}`(`id`)", table_name)),
+                    unique: *injective,
+                });
+
+                relation_table
+                    .constraints
+                    .push("unique(`range`, `domain`)".to_string());
+                tables.insert(relation_table_name.clone(), relation_table);
+            },
+        }
+    }
+
+    let key = state.parts.iter().filter(|p| p.key).collect::<Vec<_>>();
+    if !key.is_empty() {
+        table.constraints.push(format!(
+            "/* keying index */ unique({})",
+            key.into_iter()
+                .map(|s| format!("`{}`", s.name))
+                .reduce(|a, b| format!("{},{}", a, b))
+                .unwrap()
+        ));
+    }
+
+    tables.insert(table_name, table);
+}
+
+pub(crate) fn generate_single_entity_table<E: Entity>() -> String {
+    let mut esc = EntityStateContainer::default();
+    esc.visit_idmap::<E>();
+
+    let mut tables = std::collections::HashMap::new();
+    esc.iter_states()
+        .for_each(|state| process_state(&mut tables, state));
+
+    tables[E::entity_name()].build_creation_query()
+}
+
+pub(crate) fn generate_from_schema<S: Schema>() -> GeneratedSchema {
     struct IV(EntityStateContainer, Vec<IndexInfo>);
 
     impl DatabaseItemVisitor for IV {
@@ -213,102 +316,13 @@ pub(crate) fn collect_from_database<DB: Schema>(schema: &DB) -> DatabaseSchema {
 
     let mut iv = IV(EntityStateContainer::default(), Default::default());
 
-    schema.accept_item_visitor(&mut iv);
+    S::default().accept_item_visitor(&mut iv);
 
     // now to turn all that into a set of tables
     let mut tables = std::collections::HashMap::new();
 
-    for state in iv.0.iter_states() {
-        let table_name = state.name.to_string();
-        // we may end up visiting duplicate entities; skip them if so
-        if tables.contains_key(&table_name) {
-            continue;
-        }
-
-        let mut table = TableInfo::new(table_name.clone());
-        for part in state.parts.iter() {
-            match &part.ty {
-                PartType::Datum(dtype) => table.columns.push(ColumnInfo {
-                    name: part.name,
-                    ty: dtype.to_string(),
-                    fkey: None,
-                    unique: part.unique,
-                }),
-                PartType::IDReference(entity_name) => table.columns.push(ColumnInfo {
-                    name: part.name,
-                    ty: "int".into(),
-                    fkey: Some(format!("`{}`(`id`)", entity_name)),
-                    unique: part.unique,
-                }),
-                PartType::RelationDomain {
-                    table_name: relation_table_name,
-                    range_name,
-                    injective,
-                } => {
-                    let mut relation_table = TableInfo::new(relation_table_name.clone());
-
-                    relation_table.columns.push(ColumnInfo {
-                        name: "domain",
-                        ty: "int".into(),
-                        fkey: Some(format!("`{}`(`id`)", table_name)),
-                        unique: false,
-                    });
-
-                    relation_table.columns.push(ColumnInfo {
-                        name: "range",
-                        ty: "int".into(),
-                        fkey: Some(format!("`{}`(`id`)", range_name)),
-                        unique: *injective,
-                    });
-
-                    relation_table
-                        .constraints
-                        .push("unique(`range`, `domain`)".to_string());
-
-                    tables.insert(relation_table_name.clone(), relation_table);
-                },
-                PartType::RelationRange {
-                    table_name: relation_table_name,
-                    domain_name,
-                    injective,
-                } => {
-                    let mut relation_table = TableInfo::new(relation_table_name.clone());
-
-                    relation_table.columns.push(ColumnInfo {
-                        name: "domain",
-                        ty: "int".into(),
-                        fkey: Some(format!("`{}`(`id`)", domain_name)),
-                        unique: false,
-                    });
-
-                    relation_table.columns.push(ColumnInfo {
-                        name: "range",
-                        ty: "int".into(),
-                        fkey: Some(format!("`{}`(`id`)", table_name)),
-                        unique: *injective,
-                    });
-
-                    relation_table
-                        .constraints
-                        .push("unique(`range`, `domain`)".to_string());
-                    tables.insert(relation_table_name.clone(), relation_table);
-                },
-            }
-        }
-
-        let key = state.parts.iter().filter(|p| p.key).collect::<Vec<_>>();
-        if !key.is_empty() {
-            table.constraints.push(format!(
-                "/* keying index */ unique({})",
-                key.into_iter()
-                    .map(|s| format!("`{}`", s.name))
-                    .reduce(|a, b| format!("{},{}", a, b))
-                    .unwrap()
-            ));
-        }
-
-        tables.insert(table_name, table);
-    }
+    iv.0.iter_states()
+        .for_each(|state| process_state(&mut tables, state));
 
     // this must be a stable hash function, so we use sha2
     let mut hasher = sha2::Sha256::new();
@@ -337,7 +351,7 @@ pub(crate) fn collect_from_database<DB: Schema>(schema: &DB) -> DatabaseSchema {
 
     let digest = hasher.finalize();
 
-    DatabaseSchema {
+    GeneratedSchema {
         signature: digest.into_iter().fold(String::new(), |mut a, v| {
             a += &format!("{:02x}", v);
             a

microrm/src/schema/detail.rs

@@ -0,0 +1,106 @@
+use super::{
+    migration::{MigratableItem, SchemaList},
+    DatabaseItem, DatabaseItemList, Schema,
+};
+
+impl<DI0: DatabaseItem, DI1: DatabaseItem> DatabaseItemList for (DI0, DI1) {
+    type Head = DI0;
+    type Tail = (DI1,);
+}
+
+impl<DI0: DatabaseItem, DI1: DatabaseItem, DI2: DatabaseItem> DatabaseItemList for (DI0, DI1, DI2) {
+    type Head = DI0;
+    type Tail = (DI1, DI2);
+}
+
+impl<DI0: DatabaseItem, DI1: DatabaseItem, DI2: DatabaseItem, DI3: DatabaseItem> DatabaseItemList
+    for (DI0, DI1, DI2, DI3)
+{
+    type Head = DI0;
+    type Tail = (DI1, DI2, DI3);
+}
+
+impl<
+        DI0: DatabaseItem,
+        DI1: DatabaseItem,
+        DI2: DatabaseItem,
+        DI3: DatabaseItem,
+        DI4: DatabaseItem,
+    > DatabaseItemList for (DI0, DI1, DI2, DI3, DI4)
+{
+    type Head = DI0;
+    type Tail = (DI1, DI2, DI3, DI4);
+}
+
+impl<
+        DI0: DatabaseItem,
+        DI1: DatabaseItem,
+        DI2: DatabaseItem,
+        DI3: DatabaseItem,
+        DI4: DatabaseItem,
+        DI5: DatabaseItem,
+    > DatabaseItemList for (DI0, DI1, DI2, DI3, DI4, DI5)
+{
+    type Head = DI0;
+    type Tail = (DI1, DI2, DI3, DI4, DI5);
+}
+
+impl<
+        DI0: DatabaseItem,
+        DI1: DatabaseItem,
+        DI2: DatabaseItem,
+        DI3: DatabaseItem,
+        DI4: DatabaseItem,
+        DI5: DatabaseItem,
+        DI6: DatabaseItem,
+    > DatabaseItemList for (DI0, DI1, DI2, DI3, DI4, DI5, DI6)
+{
+    type Head = DI0;
+    type Tail = (DI1, DI2, DI3, DI4, DI5, DI6);
+}
+
+impl<
+        DI0: DatabaseItem,
+        DI1: DatabaseItem,
+        DI2: DatabaseItem,
+        DI3: DatabaseItem,
+        DI4: DatabaseItem,
+        DI5: DatabaseItem,
+        DI6: DatabaseItem,
+        DI7: DatabaseItem,
+    > DatabaseItemList for (DI0, DI1, DI2, DI3, DI4, DI5, DI6, DI7)
+{
+    type Head = DI0;
+    type Tail = (DI1, DI2, DI3, DI4, DI5, DI6, DI7);
+}
+
+// ----------------------------------------------------------------------
+// SchemaList
+// ----------------------------------------------------------------------
+
+impl<S0: Schema, S1: Schema> SchemaList for (S0, S1)
+where
+    S1: MigratableItem<S0>,
+{
+    type Head = S1;
+    type Tail = (S0,);
+}
+
+impl<S0: Schema, S1: Schema, S2: Schema> SchemaList for (S0, S1, S2)
+where
+    S1: MigratableItem<S0>,
+    S2: MigratableItem<S1>,
+{
+    type Head = S2;
+    type Tail = (S0, S1);
+}
+
+impl<S0: Schema, S1: Schema, S2: Schema, S3: Schema> SchemaList for (S0, S1, S2, S3)
+where
+    S1: MigratableItem<S0>,
+    S2: MigratableItem<S1>,
+    S3: MigratableItem<S2>,
+{
+    type Head = S3;
+    type Tail = (S0, S1, S2);
+}

microrm/src/schema/index.rs

@@ -47,6 +47,8 @@ impl<E: Entity, EPL: EntityPartList<Entity = E>> super::DatabaseItem for Index<E
     fn accept_item_visitor(&self, visitor: &mut impl super::DatabaseItemVisitor) {
         visitor.visit_index::<E, EPL>();
     }
+
+    type Subitems = ();
 }
 
 macro_rules! entity_index {

microrm/src/schema/migration.rs

@@ -1,7 +1,15 @@
+// IP: in-progress
+
 #![allow(missing_docs)]
 
+use std::marker::PhantomData;
+
+use crate::query::Insertable;
 use crate::schema::{
-    self, build::collect_from_database, entity::Entity, DatabaseItem, DatabaseItemVisitor, Schema,
+    self,
+    build::{generate_from_schema, generate_single_entity_table, GeneratedSchema},
+    entity::{Entity, EntityID, EntityPart, EntityPartList, EntityPartVisitor},
+    DatabaseItem, Schema, SentinelDatabaseItem,
 };
 use crate::{ConnectionLease, DBResult, Error};
 
@@ -20,61 +28,363 @@ impl<T: 'static + Entity> MigratableEntity<T> for T {
     }
 }
 
-pub trait MigratableItem<From: DatabaseItem>: 'static + DatabaseItem {
-    fn run_migration(lease: &mut ConnectionLease) -> DBResult<()>;
-}
-
-/*
-pub trait MigratableSchema<From: Schema>: 'static + Schema {
-    fn run_migration(lease: &mut ConnectionLease) -> DBResult<()> {
-        let i = Self::default();
-
-        struct Visitor;
-        impl DatabaseItemVisitor for Visitor {
-            fn visit_idmap<T: Entity>(&mut self)
-            where
-                Self: Sized,
-            {
-                todo!()
-            }
-            fn visit_index<T: Entity, PL: schema::entity::EntityPartList<Entity = T>>(&mut self)
-            where
-                Self: Sized,
-            {
-                todo!()
-            }
+pub struct MigrationContext<'a, 'b> {
+    lease: &'a mut ConnectionLease<'b>,
+    from_gen: GeneratedSchema,
+    into_gen: GeneratedSchema,
+    in_progress: Vec<(&'static str, &'static str)>,
+}
+
+// impl<'a, 'b, 'c, From: DatabaseItem, Into: DatabaseItem> MigrationContext<'a, 'b, 'c, From
+impl<'a, 'b> MigrationContext<'a, 'b> {
+    pub fn in_progress<E: Entity>(&mut self) -> DBResult<MigrateMap<E>> {
+        // TODO: check that this is, like, part of the new schema...
+
+        let table_sql = generate_single_entity_table::<IPEntity<E>>();
+        log::info!("table_sql: {table_sql}");
+
+        self.lease.execute_raw_sql(table_sql.as_str())?;
+
+        self.in_progress
+            .push((E::entity_name(), IPEntity::<E>::entity_name()));
+
+        Ok(MigrateMap::<E> {
+            ..Default::default()
+        })
+    }
+
+    pub fn lease(&mut self) -> &mut ConnectionLease<'b> {
+        self.lease
+    }
+
+    fn finish(self) -> DBResult<()> {
+        log::trace!("in progress: {:?}", self.in_progress);
+
+        // handle each in-progress table
+        for (basename, ipname) in self.in_progress {
+            // first drop the old one
+            self.lease
+                .execute_raw_sql(format!("DROP TABLE {basename};"))?;
+            // then put the new one in place
+            self.lease
+                .execute_raw_sql(format!("ALTER TABLE {ipname} RENAME TO {basename}"))?;
         }
 
-        let mut v = Visitor;
-        i.accept_item_visitor(&mut v);
-        todo!()
+        Ok(())
+    }
+}
+
+pub trait MigratableItem<From: DatabaseItem>: 'static + DatabaseItem {
+    fn run_migration(ctx: &mut MigrationContext) -> DBResult<()>
+    where
+        Self: Sized;
+}
+
+#[derive(Debug)]
+pub struct IPEntityID<OE: Entity>(i64, PhantomData<OE>);
+
+impl<OE: Entity> Clone for IPEntityID<OE> {
+    fn clone(&self) -> Self {
+        Self(self.0, PhantomData)
+    }
+}
+
+impl<OE: Entity> Copy for IPEntityID<OE> {}
+
+impl<OE: Entity> PartialEq for IPEntityID<OE> {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.eq(&other.0)
+    }
+}
+
+impl<OE: Entity> Eq for IPEntityID<OE> {}
+
+impl<OE: Entity> Ord for IPEntityID<OE> {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.0.cmp(&other.0)
+    }
+}
+
+impl<OE: Entity> PartialOrd for IPEntityID<OE> {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl<OE: Entity> std::hash::Hash for IPEntityID<OE> {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.0.hash(state)
+    }
+}
+
+impl<OE: Entity> schema::datum::Datum for IPEntityID<OE> {
+    fn sql_type() -> &'static str {
+        i64::sql_type()
+    }
+    fn bind_to(&self, stmt: &mut crate::db::StatementContext, index: i32) {
+        self.0.bind_to(stmt, index)
+    }
+    fn build_from(
+        adata: schema::relation::RelationData,
+        stmt: &mut crate::db::StatementRow,
+        index: &mut i32,
+    ) -> DBResult<Self>
+    where
+        Self: Sized,
+    {
+        Ok(Self(i64::build_from(adata, stmt, index)?, PhantomData))
+    }
+    fn accept_discriminator(d: &mut impl schema::datum::DatumDiscriminator)
+    where
+        Self: Sized,
+    {
+        d.visit_entity_id::<IPEntity<OE>>();
+    }
+    fn accept_discriminator_ref(&self, d: &mut impl schema::datum::DatumDiscriminatorRef)
+    where
+        Self: Sized,
+    {
+        d.visit_entity_id::<IPEntity<OE>>(self);
+    }
+}
+
+impl<OE: Entity> schema::datum::ConcreteDatum for IPEntityID<OE> {}
+
+impl<OE: Entity> EntityID for IPEntityID<OE> {
+    type Entity = IPEntity<OE>;
+
+    fn from_raw(id: i64) -> Self {
+        Self(id, PhantomData)
+    }
+    fn into_raw(self) -> i64 {
+        self.0
+    }
+}
+
+pub struct IPEntityIDPart<OE: Entity>(PhantomData<OE>);
+impl<OE: Entity> Clone for IPEntityIDPart<OE> {
+    fn clone(&self) -> Self {
+        Self(PhantomData)
+    }
+}
+impl<OE: Entity> Copy for IPEntityIDPart<OE> {}
+impl<OE: Entity> Default for IPEntityIDPart<OE> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<OE: Entity> EntityPart for IPEntityIDPart<OE> {
+    type Datum = IPEntityID<OE>;
+    type Entity = IPEntity<OE>;
+    fn desc() -> Option<&'static str> {
+        None
+    }
+    fn unique() -> bool {
+        false
+    }
+    fn part_name() -> &'static str {
+        "id"
+    }
+    fn get_datum(_: &Self::Entity) -> &Self::Datum {
+        unreachable!()
+    }
+}
+
+pub struct IPEntityPart<OE: Entity, OEP: EntityPart<Entity = OE>>(PhantomData<OEP>);
+impl<OE: Entity, OEP: EntityPart<Entity = OE>> Clone for IPEntityPart<OE, OEP> {
+    fn clone(&self) -> Self {
+        Self(PhantomData)
+    }
+}
+impl<OE: Entity, OEP: EntityPart<Entity = OE>> Default for IPEntityPart<OE, OEP> {
+    fn default() -> Self {
+        Self(PhantomData)
+    }
+}
+
+impl<OE: Entity, OEP: EntityPart<Entity = OE>> EntityPart for IPEntityPart<OE, OEP> {
+    type Datum = OEP::Datum;
+    type Entity = IPEntity<OE>;
+    fn desc() -> Option<&'static str> {
+        OEP::desc()
+    }
+    fn unique() -> bool {
+        OEP::unique()
+    }
+    fn part_name() -> &'static str {
+        OEP::part_name()
+    }
+    fn get_datum(from: &Self::Entity) -> &Self::Datum {
+        OEP::get_datum(&from.0)
+    }
+}
+
+pub struct IPEntityPartList<OE: Entity, OEPL: EntityPartList<Entity = OE>>(PhantomData<OEPL>);
+
+struct PartListVisitorWrapper<'a, OE: Entity, EPV: EntityPartVisitor<Entity = IPEntity<OE>>>(
+    &'a mut EPV,
+);
+
+impl<'a, OE: Entity, EPV: EntityPartVisitor<Entity = IPEntity<OE>>> EntityPartVisitor
+    for PartListVisitorWrapper<'a, OE, EPV>
+{
+    type Entity = OE;
+
+    // this is only for the datum versions
+    fn visit<EP: EntityPart<Entity = Self::Entity>>(&mut self) {
+        unreachable!()
+    }
+
+    fn visit_datum<EP: EntityPart<Entity = Self::Entity>>(&mut self, datum: &EP::Datum) {
+        self.0.visit_datum::<IPEntityPart<OE, EP>>(datum);
+    }
+    fn visit_datum_mut<EP: EntityPart<Entity = Self::Entity>>(&mut self, datum: &mut EP::Datum) {
+        self.0.visit_datum_mut::<IPEntityPart<OE, EP>>(datum);
     }
 }
 
-/// identity implementations
-// impl<T: 'static + Schema> MigratableSchema<T> for T { }
+impl<OE: Entity, OEPL: EntityPartList<Entity = OE>> EntityPartList for IPEntityPartList<OE, OEPL> {
+    type Entity = IPEntity<OE>;
+    type ListHead = IPEntityPart<OE, OEPL::ListHead>;
+    type ListTail = IPEntityPartList<OE, OEPL::ListTail>;
+    type DatumList = OEPL::DatumList;
 
-*/
+    fn build_datum_list(stmt: &mut crate::db::StatementRow) -> DBResult<Self::DatumList> {
+        OEPL::build_datum_list(stmt)
+    }
+    fn accept_part_visitor(
+        visitor: &mut impl schema::entity::EntityPartVisitor<Entity = Self::Entity>,
+    ) {
+        if !OEPL::IS_EMPTY {
+            visitor.visit::<Self::ListHead>();
+            Self::ListTail::accept_part_visitor(visitor);
+        }
+    }
+    fn accept_part_visitor_ref(
+        datum_list: &Self::DatumList,
+        visitor: &mut impl schema::entity::EntityPartVisitor<Entity = Self::Entity>,
+    ) {
+        OEPL::accept_part_visitor_ref(datum_list, &mut PartListVisitorWrapper(visitor));
+    }
+}
 
-#[derive(Default, microrm_macros::Schema)]
-pub struct EmptySchema {}
+#[derive(Debug)]
+pub struct IPEntity<OE: Entity>(OE);
 
-impl MigratableItem<EmptySchema> for EmptySchema {
-    fn run_migration(_lease: &mut ConnectionLease) -> DBResult<()> { unreachable!() }
+lazy_static::lazy_static! {
+    static ref IP_NAME_MAP: std::sync::RwLock<std::collections::HashMap<&'static str, &'static str>> = {
+        std::sync::RwLock::new(std::collections::HashMap::new())
+    };
 }
 
-impl<T: 'static + DatabaseItem> MigratableItem<()> for T {
-    fn run_migration(_lease: &mut ConnectionLease) -> DBResult<()> { unreachable!() }
+impl<OE: Entity> Entity for IPEntity<OE> {
+    type ID = IPEntityID<OE>;
+    type Parts = IPEntityPartList<OE, OE::Parts>;
+    type Keys = IPEntityPartList<OE, OE::Keys>;
+    type IDPart = IPEntityIDPart<OE>;
+
+    fn entity_name() -> &'static str {
+        // try lookup
+        let nmap = IP_NAME_MAP.read().unwrap();
+        if let Some(ename) = nmap.get(OE::entity_name()) {
+            ename
+        } else {
+            drop(nmap);
+            let mut nwmap = IP_NAME_MAP.write().unwrap();
+            let ename: &'static str =
+                Box::leak(format!("_IP_{}", OE::entity_name()).into_boxed_str());
+            nwmap.insert(OE::entity_name(), ename);
+            ename
+        }
+    }
+
+    fn build(values: <Self::Parts as EntityPartList>::DatumList) -> Self {
+        Self(OE::build(values))
+    }
+
+    fn accept_part_visitor(visitor: &mut impl schema::entity::EntityPartVisitor<Entity = Self>) {
+        Self::Parts::accept_part_visitor(visitor)
+    }
+
+    fn accept_part_visitor_ref(
+        &self,
+        visitor: &mut impl schema::entity::EntityPartVisitor<Entity = Self>,
+    ) {
+        self.0
+            .accept_part_visitor_ref(&mut PartListVisitorWrapper(visitor));
+    }
+
+    fn accept_part_visitor_mut(
+        &mut self,
+        visitor: &mut impl schema::entity::EntityPartVisitor<Entity = Self>,
+    ) {
+        self.0
+            .accept_part_visitor_ref(&mut PartListVisitorWrapper(visitor));
+    }
 }
 
-impl Schema for () {
-    fn accept_item_visitor(&self, _visitor: &mut impl schema::DatabaseItemVisitor)
+impl<T: 'static + DatabaseItem> MigratableItem<super::SentinelDatabaseItem> for T {
+    fn run_migration(_ctx: &mut MigrationContext) -> DBResult<()>
     where
         Self: Sized,
     {
+        unreachable!()
+    }
+}
+
+// XXX: this should definitely not be default
+pub struct MigrateMap<OE: Entity>(PhantomData<OE>);
+
+impl<OE: Entity> Default for MigrateMap<OE> {
+    fn default() -> Self {
+        Self(PhantomData)
     }
 }
 
+impl<OE: Entity> Insertable<OE> for MigrateMap<OE> {
+    fn insert(&self, lease: &mut ConnectionLease, value: OE) -> DBResult<OE::ID> {
+        use crate::IDMap;
+        let id = IDMap::<IPEntity<OE>>::insert(&IDMap::build(), lease, IPEntity(value))?;
+        Ok(<OE::ID>::from_raw(id.0))
+    }
+    fn insert_and_return(
+        &self,
+        lease: &mut ConnectionLease,
+        value: OE,
+    ) -> DBResult<crate::Stored<OE>> {
+        use crate::IDMap;
+        let rval =
+            IDMap::<IPEntity<OE>>::insert_and_return(&IDMap::build(), lease, IPEntity(value))?;
+        let id = rval.id();
+        Ok(crate::Stored::new(
+            <OE::ID>::from_raw(id.into_raw()),
+            rval.wrapped().0,
+        ))
+    }
+}
+
+impl<OE: Entity> MigrateMap<OE> {
+    pub fn insert_matching<ME: Entity>(
+        &self,
+        lease: &mut ConnectionLease,
+        value: OE,
+        old_value: crate::Stored<ME>,
+    ) -> DBResult<()> {
+        let id = old_value.id().into_raw();
+        let mut txn = crate::db::Transaction::new(lease, "_microrm_insert_exact")?;
+        crate::query::base_queries::insert_exact(
+            txn.lease(),
+            &IPEntity(value),
+            IPEntityID::from_raw(id),
+        )?;
+        txn.commit()?;
+        Ok(())
+    }
+}
+
+impl Schema for super::SentinelDatabaseItem {}
+
 pub trait SchemaList {
     type Head: Schema + MigratableItem<<Self::Tail as SchemaList>::Head>;
     type Tail: SchemaList;
@@ -82,52 +392,25 @@ pub trait SchemaList {
 }
 
 impl SchemaList for () {
-    type Head = ();
+    type Head = SentinelDatabaseItem;
     type Tail = ();
     const EMPTY: bool = true;
 }
 
 impl<S0: Schema> SchemaList for (S0,)
 where
-    S0: MigratableItem<()>,
+    S0: MigratableItem<SentinelDatabaseItem>,
 {
     type Head = S0;
     type Tail = ();
 }
 
-impl<S0: Schema, S1: Schema> SchemaList for (S0, S1)
-where
-    S1: MigratableItem<S0>,
-{
-    type Head = S1;
-    type Tail = (S0,);
-}
-
-impl<S0: Schema, S1: Schema, S2: Schema> SchemaList for (S0, S1, S2)
-where
-    S1: MigratableItem<S0>,
-    S2: MigratableItem<S1>,
-{
-    type Head = S2;
-    type Tail = (S0, S1);
-}
-
-impl<S0: Schema, S1: Schema, S2: Schema, S3: Schema> SchemaList for (S0, S1, S2, S3)
-where
-    S1: MigratableItem<S0>,
-    S2: MigratableItem<S1>,
-    S3: MigratableItem<S2>,
-{
-    type Head = S3;
-    type Tail = (S0, S1, S2);
-}
-
 fn migration_helper<A: SchemaList>(lease: &mut ConnectionLease) -> DBResult<()> {
     if A::EMPTY {
         return Err(Error::IncompatibleSchema);
     }
 
-    let built = collect_from_database(&<A::Head>::default());
+    let built = generate_from_schema::<A::Head>();
     match built.check(lease) {
         Some(true) => Ok(()),
         Some(false) => {
@@ -136,20 +419,26 @@ fn migration_helper<A: SchemaList>(lease: &mut ConnectionLease) -> DBResult<()>
             // migrate from (A::Tail::Head) to A::Head
             type MigrateTo<A> = <A as SchemaList>::Head;
             type MigrateFrom<A> = <<A as SchemaList>::Tail as SchemaList>::Head;
-            <MigrateTo<A> as MigratableItem<MigrateFrom<A>>>::run_migration(lease)
+
+            let mut context = MigrationContext {
+                lease,
+                into_gen: built,
+                from_gen: generate_from_schema::<MigrateFrom<A>>(),
+                in_progress: vec![],
+            };
+
+            <MigrateTo<A> as MigratableItem<MigrateFrom<A>>>::run_migration(&mut context)?;
+
+            context.finish()
         },
         None => Err(Error::EmptyDatabase),
     }
 }
 
 pub fn run_migration<A: SchemaList>(lease: &mut ConnectionLease) -> DBResult<()> {
-    // let metadb = meta::MetadataDB::default();
-
-    let _ = migration_helper::<A>(lease)?;
-
-    // easy case: see if the head is the most recent
+    let mut upgrade_txn = crate::db::Transaction::new(lease, "_microrm_migration")?;
+    // find the earliest matching schema, and then run each migration in sequence if needed
+    migration_helper::<A>(upgrade_txn.lease())?;
 
-    // super::build::collect_from_database(
-    // metadb.metastore.get(
-    Ok(())
+    upgrade_txn.commit()
 }

microrm/tests/manual_construction.rs

@@ -125,15 +125,18 @@ struct SimpleDatabase {
     strings: IDMap<SimpleEntity>,
 }
 
-impl Schema for SimpleDatabase {
+impl DatabaseItem for SimpleDatabase {
     fn accept_item_visitor(&self, visitor: &mut impl DatabaseItemVisitor)
     where
         Self: Sized,
     {
         <IDMap<SimpleEntity> as DatabaseItem>::accept_item_visitor(&IDMap::default(), visitor);
     }
+    type Subitems = (IDMap<SimpleEntity>,);
 }
 
+impl Schema for SimpleDatabase {}
+
 #[test]
 fn part_visitor() {
     struct V<E: Entity> {

microrm/tests/migration.rs

@@ -5,43 +5,122 @@ use microrm::schema::migration::*;
 
 mod common;
 
-#[derive(Entity)]
-struct KVEntity1 {
-    k: usize,
-    v: String,
-}
+mod schema1 {
+    use microrm::prelude::*;
 
-#[derive(Default, Schema)]
-struct Schema1 {
-    kv: microrm::IDMap<KVEntity1>,
-}
+    #[derive(Entity)]
+    pub struct KVEntity {
+        pub k: usize,
+        pub v: String,
+    }
 
-#[derive(Entity)]
-struct KVEntity2 {
-    k: String,
-    v: String,
+    #[derive(Default, Schema)]
+    pub struct Schema {
+        pub kv: microrm::IDMap<KVEntity>,
+    }
 }
 
-#[derive(Default, Schema)]
-struct Schema2 {
-    kv: microrm::IDMap<KVEntity2>,
+mod schema2 {
+    use microrm::prelude::*;
+
+    #[derive(Entity)]
+    pub struct KVEntity {
+        pub k: String,
+        pub v: String,
+    }
+
+    #[derive(Default, Schema)]
+    pub struct Schema {
+        pub kv: microrm::IDMap<KVEntity>,
+    }
 }
 
-impl MigratableEntity<KVEntity1> for KVEntity2 {
-    fn migrate(from: KVEntity1) -> microrm::DBResult<Option<Self>>
+impl MigratableItem<schema1::Schema> for schema2::Schema {
+    fn run_migration(ctx: &mut MigrationContext) -> microrm::DBResult<()>
     where
         Self: Sized,
     {
-        todo!()
+        // this is a very simple migration: convert the uints to Strings, which is always possible.
+
+        let ipkv2 = ctx.in_progress::<schema2::KVEntity>()?;
+
+        for kv in schema1::Schema::default().kv.get(ctx.lease())? {
+            ipkv2.insert_matching(
+                ctx.lease(),
+                schema2::KVEntity {
+                    k: kv.k.to_string(),
+                    v: kv.v.clone(),
+                },
+                kv,
+            )?;
+        }
+
+        // insert bonus extra KV entry
+        ipkv2.insert(
+            ctx.lease(),
+            schema2::KVEntity {
+                k: "s".to_string(),
+                v: "v".to_string(),
+            },
+        )?;
+
+        Ok(())
     }
 }
 
-impl MigratableSchema<Schema1> for Schema2 {}
+type Schemas = (schema1::Schema, schema2::Schema);
 
 #[test]
 fn run_empty_migration() {
-    let (pool, _db): (_, Schema1) = common::open_test_db!();
+    let (pool, _db): (_, schema2::Schema) = common::open_test_db!();
 
     let mut lease = pool.acquire().unwrap();
-    run_migration::<(Schema1, Schema2)>(&mut lease).expect("migration result");
+    run_migration::<Schemas>(&mut lease).expect("migration result");
+}
+
+#[test]
+fn run_simple_migration() {
+    let (pool, _db): (_, schema1::Schema) = common::open_test_db!();
+
+    let mut lease = pool.acquire().unwrap();
+    run_migration::<Schemas>(&mut lease).expect("migration result");
+}
+
+#[test]
+fn check_simple_migration() {
+    let (pool, db): (_, schema1::Schema) = common::open_test_db!();
+    let mut lease = pool.acquire().unwrap();
+
+    db.kv
+        .insert(
+            &mut lease,
+            schema1::KVEntity {
+                k: 42,
+                v: "sixtimesnine".to_string(),
+            },
+        )
+        .expect("insert Schema1 kv");
+
+    run_migration::<Schemas>(&mut lease).expect("migration result");
+
+    let db2 = schema2::Schema::default();
+    let migrated = db2
+        .kv
+        .with(schema2::KVEntity::K, "42")
+        .first()
+        .get(&mut lease)
+        .expect("get migrated entity")
+        .expect("no migrated entity");
+    assert_eq!(migrated.k, "42");
+    assert_eq!(migrated.v, "sixtimesnine");
+
+    let migrated = db2
+        .kv
+        .with(schema2::KVEntity::K, "s")
+        .first()
+        .get(&mut lease)
+        .expect("get migration-inserted entity")
+        .expect("no migration-inserted entity");
+    assert_eq!(migrated.k, "s");
+    assert_eq!(migrated.v, "v");
 }