fix: drop type_key column from text_fields and datetime_fields tables

docs/library-changes.md

@@ -148,9 +148,9 @@ Migration from the legacy JSON format is provided via a walkthrough when opening
 
 #### Version 200
 
-| Used From | Format | Location                                        |
-| --------- | ------ | ----------------------------------------------- |
-| TBD       | SQLite | `<Library Folder>`/.TagStudio/ts_library.sqlite |
+| Used From                                                                                            | Format | Location                                        |
+| ---------------------------------------------------------------------------------------------------- | ------ | ----------------------------------------------- |
+| [c15e2b5](https://github.com/TagStudioDev/TagStudio/commit/c15e2b56eedd0a3c13391fa43571b8f8f7c7a91f) | SQLite | `<Library Folder>`/.TagStudio/ts_library.sqlite |
 
 - Adds `text_field_templates` and `date_field_templates` tables.
 - Drops `boolean_fields` and `value_type` tables.
@@ -162,3 +162,12 @@ Migration from the legacy JSON format is provided via a walkthrough when opening
     - Values are set to `TRUE` if the field row was previously a "TEXT_BOX" type.
 - Repairs existing "Description" fields inside the `text_fields` table to have their `is_multiline` column set to `TRUE` _(Previously done in [Version 7](#version-7))_.
 - Repairs existing "Comments" fields inside the `text_fields` table to have their `is_multiline` column set to `TRUE`.
+
+#### Version 201
+
+| Used From | Format | Location                                        |
+| --------- | ------ | ----------------------------------------------- |
+| TBD       | SQLite | `<Library Folder>`/.TagStudio/ts_library.sqlite |
+
+- Drops `type_key` columns from `text_fields` and `datetime_fields` tables.
+- Enforces column positions for `text_fields` and `datetime_fields` tables.

src/tagstudio/core/library/alchemy/constants.py

@@ -9,7 +9,7 @@
 
 DB_VERSION_CURRENT_KEY: str = "CURRENT"
 DB_VERSION_INITIAL_KEY: str = "INITIAL"
-DB_VERSION: int = 200
+DB_VERSION: int = 201
 
 TAG_CHILDREN_QUERY = text("""
 WITH RECURSIVE ChildTags AS (

src/tagstudio/core/library/alchemy/fields.py

@@ -20,15 +20,15 @@ class BaseField(Base):
 
     @declared_attr
     def id(self) -> Mapped[int]:
-        return mapped_column(primary_key=True, autoincrement=True)
+        return mapped_column(primary_key=True, autoincrement=True, sort_order=1)
 
     @declared_attr
     def name(self) -> Mapped[str]:
-        return mapped_column(nullable=False, default="")
+        return mapped_column(nullable=False, default="", sort_order=2)
 
     @declared_attr
     def entry_id(self) -> Mapped[int]:
-        return mapped_column(ForeignKey("entries.id"))
+        return mapped_column(ForeignKey("entries.id"), sort_order=3)
 
     @declared_attr
     def entry(self) -> Mapped[Entry]:
@@ -47,7 +47,7 @@ def clone_with_entry_id(self, entry_id: int) -> BaseField:  # pyright: ignore
 class TextField(BaseField):
     __tablename__ = "text_fields"
 
-    value: Mapped[str | None]
+    value: Mapped[str | None] = mapped_column(sort_order=4)
     is_multiline: Mapped[bool] = mapped_column(nullable=False, default=False)
 
     @override
@@ -75,7 +75,7 @@ def clone_with_entry_id(self, entry_id: int) -> TextField:
 class DatetimeField(BaseField):
     __tablename__ = "datetime_fields"
 
-    value: Mapped[str | None]
+    value: Mapped[str | None] = mapped_column(sort_order=4)
 
     @override
     def __eq__(self, other: object) -> bool:

src/tagstudio/core/library/alchemy/library.py

@@ -419,6 +419,7 @@ def open_sqlite_library(
         # Under -> sqlite-the-sqlite-dialect-now-uses-nullpool-for-file-based-databases
         poolclass = None if storage_path == ":memory:" else NullPool
         loaded_db_version: int = 0
+        initial_db_version: int = DB_VERSION
 
         logger.info(
             "[Library] Opening SQLite Library",
@@ -430,6 +431,7 @@ def open_sqlite_library(
             # Don't check DB version when creating new library
             if not is_new:
                 loaded_db_version = self.get_version(DB_VERSION_CURRENT_KEY)
+                initial_db_version = self.get_version(DB_VERSION_INITIAL_KEY)
 
                 # ======================== Library Database Version Checking =======================
                 # DB_VERSION 6 is the first supported SQLite DB version.
@@ -452,7 +454,7 @@ def open_sqlite_library(
                         ),
                     )
 
-            logger.info(f"[Library] DB_VERSION: {loaded_db_version}")
+            logger.info(f"[Library] Library DB version: {loaded_db_version}")
             make_tables(self.engine)
 
             if is_new:
@@ -571,6 +573,9 @@ def open_sqlite_library(
                     self.__apply_db104_migrations(session, library_dir)
                 if loaded_db_version < 200:
                     self.__apply_db200_migrations(session)
+                    # changes: field tables
+                if initial_db_version < 200 and loaded_db_version < 201:
+                    self.__apply_db201_migrations(session)
 
             session.execute(
                 text("CREATE INDEX IF NOT EXISTS idx_tags_name_shorthand ON tags (name, shorthand)")
@@ -588,6 +593,7 @@ def open_sqlite_library(
 
             # Update DB_VERSION
             if loaded_db_version < DB_VERSION:
+                logger.info(f"[Library] Library migrated to DB version {DB_VERSION}")
                 self.set_version(DB_VERSION_CURRENT_KEY, DB_VERSION)
 
         # everything is fine, set the library path
@@ -808,10 +814,6 @@ def __apply_db200_migrations(self, session: Session):
             session.execute(text("UPDATE datetime_fields SET name = type_key"))
             session.flush()
 
-            # TODO: Remove `type_key` columns from text_fields and datetime_fields tables.
-            # See issue with dropping columns foreign keys in SQLite:
-            # https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes
-
             # Change `name` values to title case
             logger.info("[Library][Migration][200] Normalizing TextField names...")
             for text_field in session.execute(select(TextField)).scalars():
@@ -863,6 +865,57 @@ def __apply_db200_migrations(self, session: Session):
 
             session.commit()
 
+    def __apply_db201_migrations(self, session: Session):
+        """Migrate DB to DB_VERSION 201."""
+        with session:
+            create_text_fields_table = text("""
+            CREATE TABLE text_fields_new (
+                id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                name VARCHAR NOT NULL,
+                entry_id INTEGER NOT NULL,
+                value VARCHAR,
+                is_multiline BOOLEAN NOT NULL,
+                FOREIGN KEY(entry_id) REFERENCES entries (id)
+            )
+            """)
+            create_datetime_fields_table = text("""
+            CREATE TABLE datetime_fields_new (
+                id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                name VARCHAR NOT NULL,
+                entry_id INTEGER NOT NULL,
+                value VARCHAR,
+                FOREIGN KEY(entry_id) REFERENCES entries (id)
+            )
+            """)
+
+            logger.info("[Library][Migration][201] Dropping type_key from text_fields table...")
+            session.execute(create_text_fields_table)
+            session.flush()
+            session.execute(
+                text("""
+                    INSERT INTO text_fields_new (id, name, entry_id, value, is_multiline)
+                    SELECT id, name, entry_id, value, is_multiline
+                    FROM text_fields
+                """)
+            )
+            session.execute(text("DROP TABLE text_fields"))
+            session.execute(text("ALTER TABLE text_fields_new RENAME TO text_fields"))
+
+            logger.info("[Library][Migration][201] Dropping type_key from datetime_fields table...")
+            session.execute(create_datetime_fields_table)
+            session.flush()
+            session.execute(
+                text("""
+                    INSERT INTO datetime_fields_new (id, name, entry_id, value)
+                    SELECT id, name, entry_id, value
+                    FROM datetime_fields
+                """)
+            )
+            session.execute(text("DROP TABLE datetime_fields"))
+            session.execute(text("ALTER TABLE datetime_fields_new RENAME TO datetime_fields"))
+
+            session.commit()
+
     @property
     def field_templates(self) -> Sequence[BaseFieldTemplate]:
         with Session(self.engine) as session:

tests/test_db_migrations.py

@@ -30,6 +30,7 @@
         # str(Path(CWD.parent / FIXTURES / EMPTY_LIBRARIES / "DB_VERSION_102")),
         str(Path(CWD.parent / FIXTURES / EMPTY_LIBRARIES / "DB_VERSION_103")),
         str(Path(CWD.parent / FIXTURES / EMPTY_LIBRARIES / "DB_VERSION_200")),
+        str(Path(CWD.parent / FIXTURES / EMPTY_LIBRARIES / "DB_VERSION_201")),
     ],
 )
 def test_library_migrations(path: str):