diff --git a/CHANGES.txt b/CHANGES.txt index 0e2c566c..05f52aa8 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,11 @@ Changelog ========= +0.22.0 +------ + +* Feat: Respect naming conventions for autogenerated spatial indexes @adrien-berchet (#628) + 0.21.0 ------ diff --git a/doc/alembic.rst b/doc/alembic.rst index 8bd6bc48..23879086 100644 --- a/doc/alembic.rst +++ b/doc/alembic.rst @@ -261,3 +261,57 @@ thus the ``env.py`` file should look like the following: render_item=alembic_helpers.render_item, ) # ... + + +.. _naming_conventions_for_spatial_indexes: + +Naming Conventions for Spatial Indexes +-------------------------------------- + +By default, GeoAlchemy 2 generates spatial index names using a fixed template (such as +``idx__``). When using SQLAlchemy metadata naming conventions (e.g. +``MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"})``), you can opt in to +having autogenerated spatial indexes follow this naming convention by setting +``follow_naming_convention=True`` on spatial types (:class:`~geoalchemy2.types.Geometry`, +:class:`~geoalchemy2.types.Geography`, or :class:`~geoalchemy2.types.Raster`): + +.. code-block:: python + + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Base = declarative_base(metadata=metadata) + + class Lake(Base): + __tablename__ = "lake" + id = Column(Integer, primary_key=True) + geom = Column( + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ) + ) + +When ``follow_naming_convention=True`` is enabled: + +1. **Naming Resolution and Fallback**: The index name is generated according to the ``"ix"`` key of the + naming convention (for example, ``ix_%(table_name)s_%(column_0_N_name)s`` produces ``ix_lake_geom``, + or ``ix_gis_lake_geom`` if a schema token is used). If the naming convention does not define an + ``"ix"`` key, the index name safely falls back to the default ``idx__``. +2. **Dialect DDL Behavior**: On PostgreSQL and MSSQL, the convention name is used directly. On MySQL, + where autogenerated spatial indexes are created unnamed by default, enabling the flag creates the + spatial index with its explicit convention name (``ALTER TABLE ADD SPATIAL INDEX ()``). +3. **Alembic Writer Requirement**: You must register the :obj:`~geoalchemy2.alembic_helpers.writer` + in ``env.py`` via ``process_revision_directives=alembic_helpers.writer``. The writer handles + spatial index operations and rewrites SQLite/GeoPackage table-modification directives to suppress + false diffs. +4. **SQLite / SpatiaLite and GeoPackage**: SpatiaLite implements spatial indexing via R-tree virtual + tables whose internal trigger and table names are fixed by SpatiaLite (``idx__``). + Because of this, naming conventions do not rename the underlying R-tree virtual tables on SpatiaLite. + While Alembic revision autogeneration through ``process_revision_directives=alembic_helpers.writer`` + filters out spurious index differences, direct calls to ``autogen.compare_metadata()`` without the + writer will detect the difference between the convention-named metadata index and the database R-tree. +5. **Switching on Existing Databases**: When switching ``follow_naming_convention=True`` on an existing + database (where the spatial index was previously created with the default ``idx__`` + name), Alembic autogenerate will propose dropping the old index and creating the new convention-named + spatial index. Ensure you test this migration before deploying. diff --git a/doc/core_tutorial.rst b/doc/core_tutorial.rst index 2094b455..fb4443ec 100644 --- a/doc/core_tutorial.rst +++ b/doc/core_tutorial.rst @@ -82,6 +82,14 @@ In that case every ``Table`` that's referenced to by ``metadata`` would be created in the database. The ``metadata`` object includes one ``Table`` here, our now well-known ``lake_table`` object. +.. note:: + + **Spatial Indexes** + + By default, GeoAlchemy 2 automatically creates a spatial index on spatial columns + (named ``idx__``). To configure naming conventions, custom + indexes, or dialect options, see the :ref:`spatial_indexes` guide. + Reflecting tables ----------------- diff --git a/doc/index.rst b/doc/index.rst index 1ec1702b..fb3c1a47 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -92,6 +92,7 @@ system. If you're new to GeoAlchemy 2 start with the :ref:`orm_tutorial` page. I orm_tutorial core_tutorial + spatial_indexes dialect_specific_features gallery/index alembic diff --git a/doc/orm_tutorial.rst b/doc/orm_tutorial.rst index 6d29d6fe..8d7c806d 100644 --- a/doc/orm_tutorial.rst +++ b/doc/orm_tutorial.rst @@ -118,6 +118,14 @@ If we wanted to drop the table we'd use:: >>> Lake.__table__.drop(engine) +.. note:: + + **Spatial Indexes** + + By default, GeoAlchemy 2 automatically creates a spatial index on spatial columns + (named ``idx__``). To configure naming conventions, custom + indexes, or dialect options, see the :ref:`spatial_indexes` guide. + Create an Instance of the Mapped Class -------------------------------------- diff --git a/doc/spatial_indexes.rst b/doc/spatial_indexes.rst new file mode 100644 index 00000000..745036f5 --- /dev/null +++ b/doc/spatial_indexes.rst @@ -0,0 +1,203 @@ +.. _spatial_indexes: + +Spatial Indexes +=============== + +Spatial databases use spatial indexes to accelerate queries involving spatial operators and functions +(such as bounding-box intersections, distance queries, and geometric containment). + +GeoAlchemy 2 simplifies spatial index management by automatically creating and managing spatial +indexes for :class:`~geoalchemy2.types.Geometry`, :class:`~geoalchemy2.types.Geography`, and +:class:`~geoalchemy2.types.Raster` columns. + + +Default Behavior +---------------- + +By default, any spatial column defined with ``spatial_index=True`` (the default) automatically +registers a spatial index in the table's metadata:: + + from sqlalchemy import Column, Integer, MetaData, Table + from geoalchemy2 import Geometry + + metadata = MetaData() + lake_table = Table( + "lake", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry("POLYGON")), + ) + +The autogenerated index is assigned a default name using the template ``idx__``:: + + >>> [idx.name for idx in lake_table.indexes] + ['idx_lake_geom'] + +When creating the table with ``lake_table.create(engine)`` or ``metadata.create_all(engine)``, +GeoAlchemy 2 automatically creates the corresponding spatial index using the native indexing mechanism +of the target database dialect. + + +Naming Conventions +------------------ + +SQLAlchemy allows you to define automated `constraint and index naming conventions +`_ on +:class:`~sqlalchemy.schema.MetaData`. + +By default, GeoAlchemy 2 uses its hardcoded ``idx__`` naming scheme for +backward compatibility. To instruct GeoAlchemy 2 to follow the naming convention configured on the +metadata, set ``follow_naming_convention=True`` on the spatial type. + +SQLAlchemy Core +^^^^^^^^^^^^^^^ + +In SQLAlchemy Core, pass ``follow_naming_convention=True`` when defining the column:: + + from sqlalchemy import Column, Integer, MetaData, Table + from geoalchemy2 import Geometry + + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + lake_table = Table( + "lake", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry("POLYGON", follow_naming_convention=True)), + ) + + >>> [idx.name for idx in lake_table.indexes] + ['ix_lake_geom'] + +SQLAlchemy ORM +^^^^^^^^^^^^^^ + +In the SQLAlchemy ORM, associate the naming convention with the declarative base's metadata:: + + from sqlalchemy import Column, Integer, MetaData + from sqlalchemy.orm import declarative_base + from geoalchemy2 import Geometry + + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Base = declarative_base(metadata=metadata) + + class Lake(Base): + __tablename__ = "lake" + id = Column(Integer, primary_key=True) + geom = Column(Geometry("POLYGON", follow_naming_convention=True)) + + >>> [idx.name for idx in Lake.__table__.indexes] + ['ix_lake_geom'] + +Or using the SQLAlchemy 2.0 ``DeclarativeBase`` class:: + + from sqlalchemy import MetaData + from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + from geoalchemy2 import Geometry, WKBElement + + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + + class Base(DeclarativeBase): + metadata = metadata + + class Lake(Base): + __tablename__ = "lake" + id: Mapped[int] = mapped_column(primary_key=True) + geom: Mapped[WKBElement] = mapped_column(Geometry("POLYGON", follow_naming_convention=True)) + +Resolution and Fallback Rules +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When ``follow_naming_convention=True`` is enabled: + +1. **Resolution**: The spatial index name is resolved using the ``"ix"`` key (or ``Index`` class key) + defined in the metadata's naming convention dictionary. +2. **Fallback**: If the naming convention does not define an ``"ix"`` key (or if no naming convention + is configured), the index name safely falls back to ``idx__``. + + +Custom User-Defined Spatial Indexes +----------------------------------- + +If you need full control over the spatial index (for example, to specify index storage parameters, +create a partial index with a ``WHERE`` clause, or index multiple columns), you should disable +automatic index generation by setting ``spatial_index=False`` and define an explicit +:class:`~sqlalchemy.schema.Index`:: + + from sqlalchemy import Column, Index, Integer, MetaData, Table + from geoalchemy2 import Geometry + + metadata = MetaData() + lake_table = Table( + "lake", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry("POLYGON", spatial_index=False)), + ) + + # Define a partial GiST spatial index on PostgreSQL + Index( + "idx_lake_active_geom", + lake_table.c.geom, + postgresql_using="gist", + postgresql_where=lake_table.c.id > 100, + ) + + +Dialect-Specific Details +------------------------ + +PostgreSQL / PostGIS +^^^^^^^^^^^^^^^^^^^^ + +- PostGIS spatial indexes are created as GiST (Generalized Search Tree) indexes using + ``USING gist``. +- **N-Dimensional Indexes**: By default, a 2D bounding-box index is created. For 3D or 4D geometries, + you can set ``use_N_D_index=True`` on the spatial type to generate an N-D index using the + ``gist_geometry_ops_nd`` operator class. +- **Managed Columns (`use_typmod=False`)**: When PostGIS type modifiers are disabled, columns are + added via ``AddGeometryColumn()`` after table creation, and the spatial index is created in an + ``after_create`` event using the resolved convention name. + +MySQL and MariaDB +^^^^^^^^^^^^^^^^^ + +- Spatial indexes are created as ``SPATIAL INDEX``. +- **Default naming**: In MySQL, autogenerated spatial indexes are created unnamed by default + (``ALTER TABLE
ADD SPATIAL INDEX ()``), which causes MySQL to assign the column's + name to the index. +- **Naming Conventions**: When ``follow_naming_convention=True`` is enabled, GeoAlchemy 2 explicitly + names the index in the DDL statement (``ALTER TABLE
ADD SPATIAL INDEX ()``). + +SQLite / SpatiaLite and GeoPackage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- SpatiaLite implements spatial indexing via R-tree virtual tables managed by C-extension functions + (``CreateSpatialIndex`` / ``DisableSpatialIndex``). +- The names of these virtual tables and their internal triggers are fixed by SpatiaLite + (``idx__``). As a result, naming conventions cannot rename SpatiaLite R-tree + virtual tables. +- GeoPackage likewise uses fixed R-tree virtual table names (``rtree__``) + managed via ``gpkgAddSpatialIndex``. + +MSSQL +^^^^^ + +- SQL Server requires spatial indexes to be created with a bounding box and tessellation grid scheme + (``GEOMETRY_GRID`` or ``GEOGRAPHY_GRID``). +- GeoAlchemy 2 automatically constructs and executes the required ``CREATE SPATIAL INDEX`` statement + after the table is created, applying the resolved naming convention. + + +Using with Alembic Migrations +----------------------------- + +When using Alembic for database migrations, autogenerated spatial indexes interact with Alembic's +autogenerate comparisons: + +- Ensure you register :obj:`~geoalchemy2.alembic_helpers.writer` in your Alembic ``env.py`` to handle + spatial index operations and suppress false diffs on SQLite/GeoPackage. +- When switching an existing database from default naming to ``follow_naming_convention=True``, Alembic + will generate a migration to drop the old ``idx__`` index and create the new + convention-named index. + +For detailed migration setup and examples, see :ref:`alembic_use`. diff --git a/geoalchemy2/admin/__init__.py b/geoalchemy2/admin/__init__.py index 3499458a..5774d365 100644 --- a/geoalchemy2/admin/__init__.py +++ b/geoalchemy2/admin/__init__.py @@ -83,6 +83,7 @@ def after_parent_attach(column, table): kwargs = { "postgresql_using": "gist", "_column_flag": True, + "info": {"geoalchemy2_spatial_index": True}, } col = column if _check_spatial_type(column.type, (Geometry, Geography)): @@ -91,13 +92,19 @@ def after_parent_attach(column, table): elif _check_spatial_type(column.type, Raster): col = func.ST_ConvexHull(column) - table.append_constraint( - Index( - _spatial_idx_name(table.name, column.name), - col, - **kwargs, - ) + name = ( + None + if getattr(column.type, "follow_naming_convention", False) + else _spatial_idx_name(table.name, column.name) + ) + idx = Index( + name, + col, + **kwargs, ) + table.append_constraint(idx) + if idx.name is None: + idx.name = _spatial_idx_name(table.name, column.name) @event.listens_for(Table, "column_reflect") def column_reflect(inspector, table, column_info): diff --git a/geoalchemy2/admin/dialects/common.py b/geoalchemy2/admin/dialects/common.py index 91c588fd..25c38138 100644 --- a/geoalchemy2/admin/dialects/common.py +++ b/geoalchemy2/admin/dialects/common.py @@ -22,6 +22,11 @@ def _spatial_idx_name(table_name, column_name): return f"idx_{table_name}_{column_name}" +def _is_generated_spatial_index(idx): + """Check whether an index is an autogenerated GeoAlchemy2 spatial index.""" + return bool(idx.info.get("geoalchemy2_spatial_index", False)) + + def _format_select_args(*args): if _SQLALCHEMY_VERSION_BEFORE_14: return [args] diff --git a/geoalchemy2/admin/dialects/geopackage.py b/geoalchemy2/admin/dialects/geopackage.py index 3a1d44d7..f05a1d63 100644 --- a/geoalchemy2/admin/dialects/geopackage.py +++ b/geoalchemy2/admin/dialects/geopackage.py @@ -15,7 +15,7 @@ from geoalchemy2 import functions from geoalchemy2.admin.dialects.common import _check_spatial_type from geoalchemy2.admin.dialects.common import _format_select_args -from geoalchemy2.admin.dialects.common import _spatial_idx_name +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import setup_create_drop from geoalchemy2.admin.dialects.sqlite import _SQLITE_FUNCTIONS from geoalchemy2.admin.dialects.sqlite import _compile_GeomFromWKB_SQLite @@ -195,15 +195,18 @@ def before_create(table, bind, **kw): # created during the table.create() step since the associated columns do not exist # at this time. table.info["_after_create_indexes"] = [] + generated = table.info.setdefault("_generated_spatial_indexes", {}) current_indexes = set(table.indexes) for idx in current_indexes: for col in table.info["_saved_columns"]: if _check_spatial_type(col.type, Geometry, dialect) and col in idx.columns.values(): table.indexes.remove(idx) - if idx.name != _spatial_idx_name(table.name, col.name) or not getattr( + if not _is_generated_spatial_index(idx) or not getattr( col.type, "spatial_index", False ): table.info["_after_create_indexes"].append(idx) + else: + generated[col.name] = idx if len(gis_cols) > 1: raise ValueError("Only one geometry column is allowed for a table stored in a GeoPackage.") diff --git a/geoalchemy2/admin/dialects/mssql.py b/geoalchemy2/admin/dialects/mssql.py index ada878fb..9e833290 100644 --- a/geoalchemy2/admin/dialects/mssql.py +++ b/geoalchemy2/admin/dialects/mssql.py @@ -24,6 +24,7 @@ from geoalchemy2 import _wkb_wkt from geoalchemy2 import functions from geoalchemy2.admin.dialects.common import _check_spatial_type +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import _spatial_idx_name from geoalchemy2.elements import WKBElement from geoalchemy2.elements import WKTElement @@ -497,10 +498,10 @@ def reflect_geometry_column(inspector, table, column_info): def _is_mssql_generated_spatial_index(idx, table, col): columns = list(idx.columns.values()) return ( - getattr(idx, "_column_flag", False) + _is_generated_spatial_index(idx) + and getattr(idx, "_column_flag", False) and len(columns) == 1 and columns[0] is col - and idx.name == _spatial_idx_name(table.name, col.name) and getattr(col.type, "spatial_index", False) ) @@ -518,6 +519,7 @@ def before_create(table, bind, **kw): ) table.info["_after_create_indexes"] = [] + generated = table.info.setdefault("_generated_spatial_indexes", {}) current_indexes = set(table.indexes) for idx in current_indexes: @@ -529,6 +531,8 @@ def before_create(table, bind, **kw): table.indexes.remove(idx) if not _is_mssql_generated_spatial_index(idx, table, col): table.info["_after_create_indexes"].append(idx) + else: + generated[col.name] = idx break @@ -541,7 +545,7 @@ def after_create(table, bind, **kw): for col in columns: if not _check_spatial_type(col.type, (Geometry, Geography), dialect): continue - if len(columns) == 1 or idx.name == _spatial_idx_name(table.name, col.name): + if len(columns) == 1: delayed_spatial_index_cols.add(col.name) for col in table.columns: @@ -553,7 +557,15 @@ def after_create(table, bind, **kw): and getattr(col.type, "spatial_index", False) and col.name not in delayed_spatial_index_cols ): - create_spatial_index(bind, table.name, col.name, col.type, schema=table.schema) + gen_idx = table.info.get("_generated_spatial_indexes", {}).get(col.name) + create_spatial_index( + bind, + table.name, + col.name, + col.type, + schema=table.schema, + index_name=gen_idx.name if gen_idx is not None else None, + ) for idx in after_create_indexes: table.indexes.add(idx) diff --git a/geoalchemy2/admin/dialects/mysql.py b/geoalchemy2/admin/dialects/mysql.py index 1a225d33..7f279768 100644 --- a/geoalchemy2/admin/dialects/mysql.py +++ b/geoalchemy2/admin/dialects/mysql.py @@ -23,7 +23,7 @@ from geoalchemy2 import functions from geoalchemy2._wkb_wkt import is_known_srid from geoalchemy2.admin.dialects.common import _check_spatial_type -from geoalchemy2.admin.dialects.common import _spatial_idx_name +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import compile_bin_literal from geoalchemy2.admin.dialects.common import setup_create_drop from geoalchemy2.admin.dialects.common import unwrap_wkb_constructor_clauses @@ -122,15 +122,18 @@ def before_create(table, bind, **kw): # created during the table.create() step since the associated columns do not exist # at this time. table.info["_after_create_indexes"] = [] + generated = table.info.setdefault("_generated_spatial_indexes", {}) current_indexes = set(table.indexes) for idx in current_indexes: for col in table.info["_saved_columns"]: if (_check_spatial_type(col.type, Geometry, dialect)) and col in idx.columns.values(): table.indexes.remove(idx) - if idx.name != _spatial_idx_name(table.name, col.name) or not getattr( + if not _is_generated_spatial_index(idx) or not getattr( col.type, "spatial_index", False ): table.info["_after_create_indexes"].append(idx) + else: + generated[col.name] = idx table.columns = table.info.pop("_saved_columns") @@ -150,11 +153,19 @@ def after_create(table, bind, **kw): and col.computed is None and not [i for i in table.indexes if col in i.columns.values()] ): - sql = f"ALTER TABLE {table.name} ADD SPATIAL INDEX({col.name});" + gen_idx = table.info.get("_generated_spatial_indexes", {}).get(col.name) + if getattr(col.type, "follow_naming_convention", False) and gen_idx is not None: + preparer = dialect.identifier_preparer + table_str = preparer.format_table(table) + index_str = preparer.format_index(gen_idx) + col_str = preparer.quote(col.name) + sql = f"ALTER TABLE {table_str} ADD SPATIAL INDEX {index_str}({col_str});" + else: + sql = f"ALTER TABLE {table.name} ADD SPATIAL INDEX({col.name});" q = text(sql) bind.execute(q) - for idx in table.info.pop("_after_create_indexes"): + for idx in table.info.pop("_after_create_indexes", []): table.indexes.add(idx) diff --git a/geoalchemy2/admin/dialects/postgresql.py b/geoalchemy2/admin/dialects/postgresql.py index 72c15fae..4bb3644e 100644 --- a/geoalchemy2/admin/dialects/postgresql.py +++ b/geoalchemy2/admin/dialects/postgresql.py @@ -16,6 +16,7 @@ from geoalchemy2 import functions from geoalchemy2.admin.dialects.common import _check_spatial_type from geoalchemy2.admin.dialects.common import _format_select_args +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import _spatial_idx_name from geoalchemy2.admin.dialects.common import compile_bin_literal from geoalchemy2.admin.dialects.common import setup_create_drop @@ -71,19 +72,23 @@ def check_management(column): return getattr(column.type, "use_typmod", None) is False -def create_spatial_index(bind, table, col): +def create_spatial_index(bind, table, col, index_name=None): """Create spatial index on the given column.""" postgresql_ops = {col.name: "gist_geometry_ops_nd"} if col.type.use_N_D_index else {} col_func = func.ST_ConvexHull(col) if _check_spatial_type(col.type, Raster) else col + name = index_name or _spatial_idx_name(table.name, col.name) idx = Index( - _spatial_idx_name(table.name, col.name), + name, col_func, postgresql_using="gist", postgresql_ops=postgresql_ops, _column_flag=True, + info={"geoalchemy2_spatial_index": True}, ) if bind is not None: idx.create(bind=bind) + table.indexes.discard(idx) + table.info.setdefault("_generated_spatial_indexes", {})[col.name] = idx return idx @@ -148,6 +153,7 @@ def before_create(table, bind, **kw): # created during the table.create() step since the associated columns do not exist # at this time. table.info["_after_create_indexes"] = [] + generated = table.info.setdefault("_generated_spatial_indexes", {}) current_indexes = set(table.indexes) for idx in current_indexes: for col in table.info["_saved_columns"]: @@ -155,10 +161,12 @@ def before_create(table, bind, **kw): _check_spatial_type(col.type, (Geometry, Raster), dialect) and check_management(col) ) and col in idx.columns.values(): table.indexes.remove(idx) - if idx.name != _spatial_idx_name(table.name, col.name) or not getattr( + if not _is_generated_spatial_index(idx) or not getattr( col.type, "spatial_index", False ): table.info["_after_create_indexes"].append(idx) + else: + generated[col.name] = idx def after_create(table, bind, **kw): @@ -188,7 +196,13 @@ def after_create(table, bind, **kw): and not [i for i in table.indexes if col in i.columns.values()] and check_management(col) ): - create_spatial_index(bind, table, col) + gen_idx = table.info.get("_generated_spatial_indexes", {}).get(col.name) + create_spatial_index( + bind, + table, + col, + index_name=gen_idx.name if gen_idx is not None else None, + ) for idx in table.info.pop("_after_create_indexes"): table.indexes.add(idx) diff --git a/geoalchemy2/admin/dialects/sqlite.py b/geoalchemy2/admin/dialects/sqlite.py index d8740dd7..38deae2a 100644 --- a/geoalchemy2/admin/dialects/sqlite.py +++ b/geoalchemy2/admin/dialects/sqlite.py @@ -14,6 +14,7 @@ from geoalchemy2 import functions from geoalchemy2.admin.dialects.common import _check_spatial_type from geoalchemy2.admin.dialects.common import _format_select_args +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import _spatial_idx_name from geoalchemy2.admin.dialects.common import compile_bin_literal from geoalchemy2.admin.dialects.common import setup_create_drop @@ -296,15 +297,18 @@ def before_create(table, bind, **kw): # created during the table.create() step since the associated columns do not exist # at this time. table.info["_after_create_indexes"] = [] + generated = table.info.setdefault("_generated_spatial_indexes", {}) current_indexes = set(table.indexes) for idx in current_indexes: for col in table.info["_saved_columns"]: if _check_spatial_type(col.type, Geometry, dialect) and col in idx.columns.values(): table.indexes.remove(idx) - if idx.name != _spatial_idx_name(table.name, col.name) or not getattr( + if not _is_generated_spatial_index(idx) or not getattr( col.type, "spatial_index", False ): table.info["_after_create_indexes"].append(idx) + else: + generated[col.name] = idx _setup_dummy_type(table, gis_cols) diff --git a/geoalchemy2/alembic_helpers.py b/geoalchemy2/alembic_helpers.py index 7d98db65..66567003 100644 --- a/geoalchemy2/alembic_helpers.py +++ b/geoalchemy2/alembic_helpers.py @@ -37,6 +37,7 @@ from geoalchemy2 import Raster from geoalchemy2.admin.dialects.common import _check_spatial_type from geoalchemy2.admin.dialects.common import _get_gis_cols +from geoalchemy2.admin.dialects.common import _is_generated_spatial_index from geoalchemy2.admin.dialects.common import _spatial_idx_name writer = rewriter.Rewriter() @@ -131,7 +132,10 @@ def spatial_behavior(self, connection, table_name, schema=None, **kw): "name": idx_name, "column_names": [idx_col], "unique": 0, - "dialect_options": {"_column_flag": True}, + "dialect_options": { + "_column_flag": True, + "info": {"geoalchemy2_spatial_index": True}, + }, } ) reflected_names.add(idx_name) @@ -155,6 +159,7 @@ def spatial_behavior(self, connection, table_name, schema=None, **kw): # Get spatial indexes has_index_query = f"""SELECT DISTINCT + INDEX_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = '{table_name}' AND INDEX_TYPE = 'SPATIAL'""" @@ -165,19 +170,26 @@ def spatial_behavior(self, connection, table_name, schema=None, **kw): if spatial_indexes: reflected_names = {i["name"] for i in indexes} for idx in spatial_indexes: - idx_col = idx[0] - idx_name = _spatial_idx_name(table_name, idx_col) - if idx_name in reflected_names: - continue - indexes.append( - { - "name": idx_name, - "column_names": [idx_col], - "unique": 0, - "dialect_options": {"_column_flag": True}, - } - ) - reflected_names.add(idx_name) + raw_idx_name = idx[0] + idx_col = idx[1] + # Only synthesize idx_t_c when INDEX_NAME == COLUMN_NAME + # (MySQL's default unnamed index) + if raw_idx_name == idx_col: + idx_name = _spatial_idx_name(table_name, idx_col) + if idx_name in reflected_names: + continue + indexes.append( + { + "name": idx_name, + "column_names": [idx_col], + "unique": 0, + "dialect_options": { + "_column_flag": True, + "info": {"geoalchemy2_spatial_index": True}, + }, + } + ) + reflected_names.add(idx_name) return indexes @@ -634,9 +646,39 @@ def alter_geo_column(context, revision, op): else: rewritten_ops = [] if col_type_is_spatial and getattr(col_type, "spatial_index", False): + idx_name = None + if getattr(col_type, "follow_naming_convention", False): + target_md = context.opts.get("target_metadata") + target_mds = ( + target_md + if isinstance(target_md, (list, tuple, set)) + else [target_md] + if target_md is not None + else [] + ) + table_key = f"{op.schema}.{op.table_name}" if op.schema else op.table_name + for md in target_mds: + model_table = md.tables.get(table_key) + if model_table is not None: + gen_indexes = model_table.info.get("_generated_spatial_indexes", {}) + if recreated_column_name in gen_indexes: + idx_name = gen_indexes[recreated_column_name].name + break + for model_idx in model_table.indexes: + cols = [c.name for c in model_idx.columns] + if recreated_column_name in cols and _is_generated_spatial_index( + model_idx + ): + idx_name = model_idx.name + break + if idx_name is not None: + break + if idx_name is None: + idx_name = _spatial_idx_name(op.table_name, recreated_column_name) + spatial_indexes.append( { - "name": _spatial_idx_name(op.table_name, recreated_column_name), + "name": idx_name, "dialect_options": {}, } ) @@ -1054,3 +1096,66 @@ def drop_geo_index(context, revision, op): ) return op + + +def _filter_sqlite_spatial_index_ops(sub_ops, default_table_name=None): + drop_ops = [op for op in sub_ops if isinstance(op, ops.DropIndexOp)] + create_ops = [op for op in sub_ops if isinstance(op, ops.CreateIndexOp)] + + ops_to_remove = set() + for d_op in drop_ops: + for c_op in create_ops: + if d_op in ops_to_remove or c_op in ops_to_remove: + continue + table_name = ( + default_table_name + or getattr(d_op, "table_name", None) + or getattr(c_op, "table_name", None) + ) + if ( + getattr(d_op, "table_name", None) + and getattr(c_op, "table_name", None) + and d_op.table_name != c_op.table_name + ): + continue + d_idx = d_op.to_index() + c_idx = c_op.to_index() + d_cols = list(d_idx.columns) + c_cols = list(c_idx.columns) + if len(d_cols) == 1 and len(c_cols) == 1 and d_cols[0].name == c_cols[0].name: + col_name = d_cols[0].name + expected_spat_name = _spatial_idx_name(table_name, col_name) if table_name else None + match = False + if expected_spat_name is not None and ( + ( + d_op.index_name == expected_spat_name + and getattr(c_cols[0].type, "follow_naming_convention", False) is True + ) + or ( + c_op.index_name == expected_spat_name + and getattr(d_cols[0].type, "follow_naming_convention", False) is True + ) + ): + match = True + + if match: + ops_to_remove.add(d_op) + ops_to_remove.add(c_op) + + if ops_to_remove: + return [op for op in sub_ops if op not in ops_to_remove] + return sub_ops + + +@writer.rewrites(ops.ModifyTableOps) +def filter_sqlite_spatial_index_diff_modify_table(context, revision, op): + """Remove false diff for SQLite/GeoPackage spatial indexes in ModifyTableOps.""" + dialect_name = getattr(context.bind, "dialect", None) + name = dialect_name.name if dialect_name is not None else None + if name not in ("sqlite", "geopackage"): + return op + new_ops = _filter_sqlite_spatial_index_ops(op.ops, default_table_name=op.table_name) + if not new_ops: + return [] + op.ops = new_ops + return op diff --git a/geoalchemy2/types/__init__.py b/geoalchemy2/types/__init__.py index 46bbe527..9fc5b4d3 100644 --- a/geoalchemy2/types/__init__.py +++ b/geoalchemy2/types/__init__.py @@ -95,6 +95,8 @@ class _GISType(UserDefinedType): column. To use check constraints instead set ``use_typmod`` to ``False``. By default this option is not included in the call to ``AddGeometryColumn``. Note that this option is only available for PostGIS 2.x. + follow_naming_convention: Indicate if the autogenerated spatial index should follow + the SQLAlchemy naming convention defined on the metadata. Default is ``False``. """ name: str | None = None @@ -128,6 +130,7 @@ def __init__( name: str | None = None, nullable: bool = True, _spatial_index_reflected=None, + follow_naming_convention: bool = False, ) -> None: geometry_type, srid, dimension = self.check_ctor_args( geometry_type, srid, dimension, use_typmod, nullable @@ -145,6 +148,7 @@ def __init__( self.extended: bool | None = self.as_binary == "ST_AsEWKB" self.nullable = nullable self._spatial_index_reflected = _spatial_index_reflected + self.follow_naming_convention = follow_naming_convention def get_col_spec(self): if not self.geometry_type: @@ -365,6 +369,8 @@ class Raster(_GISType): Args: spatial_index: Indicate if a spatial index should be created. Default is ``True``. + follow_naming_convention: Indicate if the autogenerated spatial index should follow + the SQLAlchemy naming convention defined on the metadata. Default is ``False``. """ comparator_factory = BaseComparator @@ -398,6 +404,7 @@ def __init__( name=None, nullable=True, _spatial_index_reflected=None, + follow_naming_convention: bool = False, ) -> None: # Enforce default values super().__init__( @@ -411,6 +418,7 @@ def __init__( name=name, nullable=nullable, _spatial_index_reflected=_spatial_index_reflected, + follow_naming_convention=follow_naming_convention, ) self.extended = None diff --git a/tests/test_admin_common.py b/tests/test_admin_common.py index e5f0f113..67b16fcb 100644 --- a/tests/test_admin_common.py +++ b/tests/test_admin_common.py @@ -69,3 +69,249 @@ def test_update_table_for_dispatch_uses_selected_column_collection(monkeypatch): assert table.info["_saved_columns"] is original_columns assert isinstance(table.columns, _WriteableColumnCollection) assert table.columns.columns == [regular_col] + + +def test_spatial_index_name_resolution_flag_off_default_convention(): + from geoalchemy2 import Geometry + + table = Table("lake", MetaData(), Column("geom", Geometry(spatial_index=True))) + idx = list(table.indexes)[0] + assert idx.name == "idx_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_flag_on_default_convention(): + from geoalchemy2 import Geometry + + table = Table( + "lake", + MetaData(), + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + idx = list(table.indexes)[0] + assert idx.name == "ix_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_flag_on_default_convention_schema(): + from geoalchemy2 import Geometry + + table = Table( + "lake", + MetaData(), + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + schema="gis", + ) + idx = list(table.indexes)[0] + assert idx.name == "ix_gis_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_flag_on_custom_ix_convention(): + from geoalchemy2 import Geometry + + metadata = MetaData(naming_convention={"ix": "my_idx_%(table_name)s_%(column_0_name)s"}) + table = Table( + "lake", + metadata, + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + idx = list(table.indexes)[0] + assert idx.name == "my_idx_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_flag_on_index_class_convention(): + from sqlalchemy import Index + + from geoalchemy2 import Geometry + + metadata = MetaData(naming_convention={Index: "my_idx_%(table_name)s_%(column_0_name)s"}) + table = Table( + "lake", + metadata, + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + idx = list(table.indexes)[0] + assert idx.name == "my_idx_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_flag_on_convention_without_ix_falls_back(): + from geoalchemy2 import Geometry + + metadata = MetaData(naming_convention={"uq": "uq_%(table_name)s_%(column_0_name)s"}) + table = Table( + "lake", + metadata, + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + idx = list(table.indexes)[0] + assert idx.name == "idx_lake_geom" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_spatial_index_name_resolution_raster_flag_on(): + from geoalchemy2 import Raster + + table = Table( + "ocean", + MetaData(), + Column("rast", Raster(spatial_index=True, follow_naming_convention=True)), + ) + idx = list(table.indexes)[0] + assert idx.name == "ix_ocean_rast" + assert idx._column_flag is True + assert idx.info.get("geoalchemy2_spatial_index") is True + + +def test_type_repr_includes_follow_naming_convention(): + from geoalchemy2 import Geography + from geoalchemy2 import Geometry + from geoalchemy2 import Raster + + geom = Geometry("POINT", 4326, follow_naming_convention=True) + assert "follow_naming_convention=True" in repr(geom) + + geog = Geography("POINT", 4326, follow_naming_convention=True) + assert "follow_naming_convention=True" in repr(geog) + + rast = Raster(follow_naming_convention=True) + assert "follow_naming_convention=True" in repr(rast) + + +def test_mysql_after_create_retains_generated_spatial_index_for_multiple_creates(monkeypatch): + from sqlalchemy.dialects import mysql + + from geoalchemy2 import Geometry + from geoalchemy2.admin.dialects import mysql as mysql_admin + + table = Table("lake", MetaData(), Column("geom", Geometry(spatial_index=True))) + executed = [] + bind = type( + "Bind", + (), + {"dialect": mysql.dialect(), "execute": lambda self, q, *a, **k: executed.append(str(q))}, + )() + + mysql_admin.before_create(table, bind) + mysql_admin.after_create(table, bind) + + # Generated index is kept in table.info, NOT table.indexes + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert "geom" in table.info.get("_generated_spatial_indexes", {}) + assert len(executed) == 1 + + # Second cycle preserves the index and executes again without error + mysql_admin.before_create(table, bind) + mysql_admin.after_create(table, bind) + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert len(executed) == 2 + + +def test_mysql_after_create_ddl_flag_off_vs_flag_on(monkeypatch): + from sqlalchemy.dialects import mysql + + from geoalchemy2 import Geometry + from geoalchemy2.admin.dialects import mysql as mysql_admin + + # Flag off: raw unnamed statement + table_off = Table("lake", MetaData(), Column("geom", Geometry(spatial_index=True))) + executed = [] + bind_off = type( + "Bind", + (), + {"dialect": mysql.dialect(), "execute": lambda self, q, *a, **k: executed.append(str(q))}, + )() + + mysql_admin.before_create(table_off, bind_off) + mysql_admin.after_create(table_off, bind_off) + assert executed == ["ALTER TABLE lake ADD SPATIAL INDEX(geom);"] + + # Flag on: formatted statement with format_table, format_index, quoted col + executed.clear() + table_on = Table( + "lake", + MetaData(), + Column("geom", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + bind_on = type( + "Bind", + (), + {"dialect": mysql.dialect(), "execute": lambda self, q, *a, **k: executed.append(str(q))}, + )() + + mysql_admin.before_create(table_on, bind_on) + mysql_admin.after_create(table_on, bind_on) + assert executed == ["ALTER TABLE lake ADD SPATIAL INDEX ix_lake_geom(geom);"] + + # Flag on with reserved words: identifier_preparer quotes them + executed.clear() + table_reserved = Table( + "order", + MetaData(), + Column("select", Geometry(spatial_index=True, follow_naming_convention=True)), + ) + mysql_admin.before_create(table_reserved, bind_on) + mysql_admin.after_create(table_reserved, bind_on) + assert executed == ["ALTER TABLE `order` ADD SPATIAL INDEX ix_order_select(`select`);"] + + +def test_sqlite_after_create_retains_generated_spatial_index_for_multiple_creates(monkeypatch): + from sqlalchemy.dialects import sqlite + + from geoalchemy2 import Geometry + from geoalchemy2.admin.dialects import sqlite as sqlite_admin + + table = Table("lake", MetaData(), Column("geom", Geometry(spatial_index=True))) + bind = type("Bind", (), {"dialect": sqlite.dialect(), "execute": lambda *a, **k: None})() + created = [] + monkeypatch.setattr(sqlite_admin, "create_spatial_index", lambda *a, **k: created.append(a)) + + sqlite_admin.before_create(table, bind) + sqlite_admin.after_create(table, bind) + + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert "geom" in table.info.get("_generated_spatial_indexes", {}) + assert len(created) == 1 + + # Second cycle + sqlite_admin.before_create(table, bind) + sqlite_admin.after_create(table, bind) + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert len(created) == 2 + + +def test_postgresql_managed_path_retains_generated_spatial_index(monkeypatch): + from sqlalchemy.dialects import postgresql + + from geoalchemy2 import Geometry + from geoalchemy2.admin.dialects import postgresql as postgresql_admin + + table = Table( + "lake", + MetaData(), + Column("geom", Geometry(spatial_index=True, use_typmod=False)), + ) + bind = type("Bind", (), {"dialect": postgresql.dialect(), "execute": lambda *a, **k: None})() + created = [] + monkeypatch.setattr(postgresql_admin, "create_spatial_index", lambda *a, **k: created.append(a)) + + postgresql_admin.before_create(table, bind) + postgresql_admin.after_create(table, bind) + + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert "geom" in table.info.get("_generated_spatial_indexes", {}) + assert len(created) == 1 + + # Second cycle + postgresql_admin.before_create(table, bind) + postgresql_admin.after_create(table, bind) + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert len(created) == 2 diff --git a/tests/test_alembic_migrations.py b/tests/test_alembic_migrations.py index 93023eb5..ae4ebced 100644 --- a/tests/test_alembic_migrations.py +++ b/tests/test_alembic_migrations.py @@ -66,6 +66,289 @@ def test_no_diff(self, conn, Lake, setup_tables, use_alembic_monkeypatch, dialec assert diff == [] + def test_no_diff_with_naming_convention( + self, + conn, + setup_tables, + use_alembic_monkeypatch, + dialect_name, + ): + """Check that autogeneration produces no diff with follow_naming_convention.""" + if dialect_name in ["sqlite", "geopackage"]: + pytest.skip( + "SQLite/GeoPackage compare_metadata reports R-tree pair; tested via rewriter" + ) + + naming_convention = {"ix": "ix_%(table_name)s_%(column_0_N_name)s"} + db_metadata = MetaData(naming_convention=naming_convention) + + Table( + "convention_lake", + db_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + nullable=dialect_name not in ["mysql", "mariadb"], + ), + ), + schema="gis" if dialect_name == "postgresql" else None, + ) + db_metadata.drop_all(conn, checkfirst=True) + db_metadata.create_all(conn) + + # Fresh model metadata representing the target schema without having run create_all() + target_metadata = MetaData(naming_convention=naming_convention) + Table( + "convention_lake", + target_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + nullable=dialect_name not in ["mysql", "mariadb"], + ), + ), + schema="gis" if dialect_name == "postgresql" else None, + ) + + try: + mc = MigrationContext.configure( + conn, + opts={ + "include_name": lambda name, type_, parent_names: ( + type_ != "table" or name in ["convention_lake"] + ), + "process_revision_directives": alembic_helpers.writer, + }, + ) + diff = compare_metadata(mc, target_metadata) + assert diff == [] + finally: + db_metadata.drop_all(conn, checkfirst=True) + + @test_only_with_dialects("sqlite-spatialite3", "sqlite-spatialite4") + def test_sqlite_rewriter_drops_false_spatial_index_diff( + self, + conn, + setup_tables, + use_alembic_monkeypatch, + ): + from alembic.autogenerate import produce_migrations + + naming_convention = {"ix": "ix_%(table_name)s_%(column_0_N_name)s"} + db_metadata = MetaData(naming_convention=naming_convention) + + Table( + "conv_lake", + db_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + db_metadata.drop_all(conn, checkfirst=True) + db_metadata.create_all(conn) + + target_metadata = MetaData(naming_convention=naming_convention) + Table( + "conv_lake", + target_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + + try: + mc = MigrationContext.configure( + conn, + opts={ + "include_name": lambda name, type_, parent_names: ( + type_ != "table" or name in ["conv_lake"] + ), + "process_revision_directives": alembic_helpers.writer, + }, + ) + # compare_metadata still reports the drop/create pair (documented limitation) + diff = compare_metadata(mc, target_metadata) + assert len(diff) == 2 + diff_ops = {d[0] for d in diff} + assert "remove_index" in diff_ops + assert "add_index" in diff_ops + + # produce_migrations followed by writer removes the false diff + migration_script = produce_migrations(mc, target_metadata) + assert not migration_script.upgrade_ops.is_empty() + + # Apply writer + alembic_helpers.writer(mc, None, [migration_script]) + + # Both upgrade and downgrade are now empty! + assert migration_script.upgrade_ops.is_empty() + assert migration_script.downgrade_ops.is_empty() + finally: + db_metadata.drop_all(conn, checkfirst=True) + + @test_only_with_dialects("sqlite-spatialite3", "sqlite-spatialite4") + def test_sqlite_rewriter_preserves_real_changes( + self, + conn, + setup_tables, + use_alembic_monkeypatch, + ): + from alembic.autogenerate import produce_migrations + + naming_convention = {"ix": "ix_%(table_name)s_%(column_0_N_name)s"} + db_metadata = MetaData(naming_convention=naming_convention) + + Table( + "conv_lake_change", + db_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + db_metadata.drop_all(conn, checkfirst=True) + db_metadata.create_all(conn) + + try: + # Model metadata has extra_val column + target_metadata = MetaData(naming_convention=naming_convention) + Table( + "conv_lake_change", + target_metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + Column("extra_val", Integer), + ) + + mc = MigrationContext.configure( + conn, + opts={ + "include_name": lambda name, type_, parent_names: ( + type_ != "table" or name in ["conv_lake_change"] + ), + "process_revision_directives": alembic_helpers.writer, + }, + ) + migration_script = produce_migrations(mc, target_metadata) + alembic_helpers.writer(mc, None, [migration_script]) + + # The index false diff was removed, but the extra_val column addition is kept! + assert not migration_script.upgrade_ops.is_empty() + upgrade_ops = migration_script.upgrade_ops.ops + + def find_add_column(ops_list): + for op in ops_list: + if isinstance(op, ops.AddColumnOp) and op.column.name == "extra_val": + return True + if hasattr(op, "ops") and find_add_column(op.ops): + return True + return False + + assert find_add_column(upgrade_ops) + finally: + db_metadata.drop_all(conn, checkfirst=True) + + @test_only_with_dialects("postgresql") + def test_postgresql_flag_switched_on_proposes_rebuild( + self, + conn, + setup_tables, + use_alembic_monkeypatch, + ): + """When switching follow_naming_convention on existing DB, Alembic proposes rebuild.""" + metadata_old = MetaData() + Table( + "lake_rebuild", + metadata_old, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=False, + ), + ), + schema="gis", + ) + metadata_old.drop_all(conn, checkfirst=True) + metadata_old.create_all(conn) + + try: + # DB has idx_lake_rebuild_geom + metadata_new = MetaData( + naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"} + ) + Table( + "lake_rebuild", + metadata_new, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + schema="gis", + ) + mc = MigrationContext.configure( + conn, + opts={ + "include_name": lambda name, type_, parent_names: ( + type_ != "table" or name in ["lake_rebuild"] + ), + "process_revision_directives": alembic_helpers.writer, + }, + ) + diff = compare_metadata(mc, metadata_new) + diff_ops = {d[0] for d in diff} + assert "remove_index" in diff_ops + assert "add_index" in diff_ops + finally: + metadata_old.drop_all(conn, checkfirst=True) + def test_diff(self, conn, Lake, setup_tables, use_alembic_monkeypatch): """Check that the autogeneration detects spatial types properly.""" metadata = MetaData() diff --git a/tests/test_functional_geopackage.py b/tests/test_functional_geopackage.py index 2c85060e..aef1e300 100644 --- a/tests/test_functional_geopackage.py +++ b/tests/test_functional_geopackage.py @@ -202,6 +202,35 @@ class LocalPoint(base): == 1 ) + def test_spatial_index_follow_naming_convention(self, conn, base, metadata): + class ConvLake(base): + __tablename__ = "conv_lake" + id = Column(Integer, primary_key=True) + geom = Column( + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ) + ) + + metadata.naming_convention = {"ix": "ix_%(table_name)s_%(column_0_N_name)s"} + metadata.drop_all(conn, checkfirst=True) + metadata.create_all(conn) + assert ( + conn.execute( + text( + """SELECT COUNT(*) FROM gpkg_extensions + WHERE table_name = 'conv_lake' + AND column_name = 'geom' + AND extension_name = 'gpkg_rtree_index';""" + ) + ).scalar() + == 1 + ) + metadata.drop_all(conn, checkfirst=True) + class TestMiscellaneous: def test_load_spatialite_gpkg(self, tmpdir, _engine_echo, check_spatialite): diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index fc5b9a15..690c1524 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -140,6 +140,88 @@ def test_custom_spatial_index_options_are_reflected_without_duplicates(self, con metadata.drop_all(conn, checkfirst=True) + def test_spatial_index_follow_naming_convention(self, conn): + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Table( + "auto_indexed_conv_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry(geometry_type="LINESTRING", srid=4326, follow_naming_convention=True), + ), + ) + metadata.drop_all(conn, checkfirst=True) + metadata.create_all(conn) + + indexes = conn.execute( + text( + """SELECT i.name, c.name + FROM sys.indexes AS i + JOIN sys.index_columns AS ic + ON i.object_id = ic.object_id + AND i.index_id = ic.index_id + JOIN sys.columns AS c + ON ic.object_id = c.object_id + AND ic.column_id = c.column_id + WHERE + i.object_id = OBJECT_ID('auto_indexed_conv_lake') + AND i.type_desc = 'SPATIAL' + ORDER BY i.name""" + ) + ).fetchall() + assert indexes == [("ix_auto_indexed_conv_lake_geom", "geom")] + + reflected = Table("auto_indexed_conv_lake", MetaData(), autoload_with=conn) + assert reflected.c.geom.type.spatial_index is True + reflected_indexes = inspect(conn).get_indexes("auto_indexed_conv_lake") + assert any(idx["name"] == "ix_auto_indexed_conv_lake_geom" for idx in reflected_indexes) + + metadata.drop_all(conn, checkfirst=True) + + def test_alter_geo_column_recreates_with_convention_name(self, conn): + from alembic.migration import MigrationContext + from alembic.operations import ops + + from geoalchemy2 import alembic_helpers + + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Table( + "alter_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + + empty_meta = MetaData() + ctx = MigrationContext.configure( + conn, + opts={"target_metadata": [empty_meta, metadata]}, + ) + op = ops.AlterColumnOp( + "alter_lake", + "geom", + modify_type=Geometry( + geometry_type="LINESTRING", + srid=4326, + spatial_index=True, + follow_naming_convention=True, + ), + existing_type=Integer(), + ) + rewritten = alembic_helpers.alter_geo_column(ctx, None, op) + index_ops = [o for o in rewritten if isinstance(o, alembic_helpers.CreateGeospatialIndexOp)] + assert len(index_ops) == 1 + assert index_ops[0].index_name == "ix_alter_lake_geom" + @test_only_with_dialects("mssql") class TestInsertionCore: diff --git a/tests/test_functional_mysql.py b/tests/test_functional_mysql.py index 627518ab..351b33bd 100644 --- a/tests/test_functional_mysql.py +++ b/tests/test_functional_mysql.py @@ -11,6 +11,7 @@ from sqlalchemy import and_ from sqlalchemy import bindparam from sqlalchemy import create_engine +from sqlalchemy import inspect from sqlalchemy import text from sqlalchemy.engine import URL from sqlalchemy.exc import OperationalError @@ -43,6 +44,101 @@ def test_create_drop_tables( metadata.create_all(conn) metadata.drop_all(conn, checkfirst=True) + def test_spatial_index_follow_naming_convention(self, conn, use_alembic_monkeypatch): + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Table( + "convention_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry(geometry_type="LINESTRING", srid=4326, follow_naming_convention=True), + nullable=False, + ), + ) + metadata.drop_all(conn, checkfirst=True) + metadata.create_all(conn) + + indexes = conn.execute( + text( + """SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'convention_lake' + AND INDEX_TYPE = 'SPATIAL';""" + ) + ).fetchall() + assert indexes == [("ix_convention_lake_geom", "geom", "SPATIAL")] + + # Reflection checks: reflected under real name, NOT synthesized + reflected = Table("convention_lake", MetaData(), autoload_with=conn) + assert reflected.c.geom.type.spatial_index is True + reflected_indexes = inspect(conn).get_indexes("convention_lake") + assert any(idx["name"] == "ix_convention_lake_geom" for idx in reflected_indexes) + assert not any(idx["name"] == "idx_convention_lake_geom" for idx in reflected_indexes) + + metadata.drop_all(conn, checkfirst=True) + + def test_spatial_index_follow_naming_convention_no_ix_key(self, conn): + metadata = MetaData(naming_convention={"fk": "fk_%(table_name)s_%(column_0_N_name)s"}) + Table( + "no_ix_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry(geometry_type="LINESTRING", srid=4326, follow_naming_convention=True), + nullable=False, + ), + ) + metadata.drop_all(conn, checkfirst=True) + metadata.create_all(conn) + + indexes = conn.execute( + text( + """SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'no_ix_lake' + AND INDEX_TYPE = 'SPATIAL';""" + ) + ).fetchall() + assert indexes == [("idx_no_ix_lake_geom", "geom", "SPATIAL")] + + metadata.drop_all(conn, checkfirst=True) + + def test_spatial_index_flag_off_mysql_unnamed(self, conn, use_alembic_monkeypatch): + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + Table( + "flag_off_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry(geometry_type="LINESTRING", srid=4326, follow_naming_convention=False), + nullable=False, + ), + ) + metadata.drop_all(conn, checkfirst=True) + metadata.create_all(conn) + + indexes = conn.execute( + text( + """SELECT INDEX_NAME, COLUMN_NAME, INDEX_TYPE + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'flag_off_lake' + AND INDEX_TYPE = 'SPATIAL';""" + ) + ).fetchall() + assert indexes == [("geom", "geom", "SPATIAL")] + + # Reflection checks: synthesized idx_t_c is present for Alembic compatibility + reflected_indexes = inspect(conn).get_indexes("flag_off_lake") + assert any(idx["name"] == "idx_flag_off_lake_geom" for idx in reflected_indexes) + + metadata.drop_all(conn, checkfirst=True) + class TestInsertionCore: @pytest.mark.parametrize("use_executemany", [True, False]) diff --git a/tests/test_functional_postgresql.py b/tests/test_functional_postgresql.py index 955c7bf9..e9d98e8d 100644 --- a/tests/test_functional_postgresql.py +++ b/tests/test_functional_postgresql.py @@ -28,6 +28,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import InternalError from sqlalchemy.exc import NoSuchModuleError +from sqlalchemy.exc import ProgrammingError from sqlalchemy.orm import declarative_base from sqlalchemy.sql import func from sqlalchemy.sql.expression import type_coerce @@ -287,6 +288,214 @@ class TableWithIndexes(BaseArgTest): assert idx[0] == expected_idx[0] assert idx[1] == re.sub("\n *", " ", expected_idx[1]) + def test_spatial_index_follow_naming_convention(self, conn): + metadata = MetaData( + schema="gis", + naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}, + ) + table = Table( + "convention_test", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="POINT", + spatial_index=True, + follow_naming_convention=True, + ), + ), + Column( + "rast", + Raster( + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + """SELECT indexname + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = 'convention_test' + ORDER BY indexname;""" + ) + indexes = [row[0] for row in conn.execute(index_query).fetchall()] + assert "ix_convention_test_geom" in indexes + assert "ix_convention_test_rast" in indexes + assert len(indexes) == 3 # pkey + geom + rast + finally: + table.drop(conn, checkfirst=True) + + def test_spatial_index_follow_naming_convention_managed_path(self, conn): + metadata = MetaData( + schema="gis", + naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}, + ) + table = Table( + "managed_convention_test", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="POINT", + spatial_index=True, + use_typmod=False, + follow_naming_convention=True, + ), + ), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + """SELECT indexname + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = 'managed_convention_test' + ORDER BY indexname;""" + ) + indexes = [row[0] for row in conn.execute(index_query).fetchall()] + assert "ix_managed_convention_test_geom" in indexes + assert len(indexes) == 2 # pkey + geom + finally: + table.drop(conn, checkfirst=True) + + def test_spatial_index_user_index_with_postgresql_where(self, conn): + metadata = MetaData(schema="gis") + table = Table( + "user_idx_where_test", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="POINT", spatial_index=False)), + ) + Index( + "custom_where_idx", + table.c.geom, + postgresql_using="gist", + postgresql_where=table.c.id > 10, + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + """SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = 'user_idx_where_test' + AND indexname = 'custom_where_idx';""" + ) + row = conn.execute(index_query).fetchone() + assert row is not None + assert "WHERE (id > 10)" in row[1] + finally: + table.drop(conn, checkfirst=True) + + def test_spatial_index_managed_user_index_named_idx_t_c_reports_collision(self, conn): + metadata = MetaData(schema="gis") + table = Table( + "managed_collision_test", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="POINT", spatial_index=True, use_typmod=False)), + ) + Index( + "idx_managed_collision_test_geom", + table.c.geom, + postgresql_using="gist", + postgresql_where=table.c.id > 10, + ) + table.drop(conn, checkfirst=True) + trans = conn.begin_nested() + try: + with pytest.raises( + ProgrammingError, match='relation "idx_managed_collision_test_geom" already exists' + ): + table.create(conn) + finally: + trans.rollback() + table.drop(conn, checkfirst=True) + + def test_spatial_index_user_index_named_idx_t_c_gives_two_indexes(self, conn): + metadata = MetaData(schema="gis") + table = Table( + "two_idx_test", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="POINT", spatial_index=True)), + ) + Index("idx_gis_two_idx_test_geom", table.c.geom, postgresql_using="gist") + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + """SELECT indexname + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = 'two_idx_test' + ORDER BY indexname;""" + ) + indexes = [row[0] for row in conn.execute(index_query).fetchall()] + assert "idx_two_idx_test_geom" in indexes + assert "idx_gis_two_idx_test_geom" in indexes + finally: + table.drop(conn, checkfirst=True) + + def test_spatial_index_column_index_true_is_user_index(self, conn): + metadata = MetaData(schema="gis") + table = Table( + "col_idx_true_test", + metadata, + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="POINT", spatial_index=True), index=True), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + """SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = 'col_idx_true_test' + ORDER BY indexname;""" + ) + indexes = conn.execute(index_query).fetchall() + names = [row[0] for row in indexes] + assert "idx_col_idx_true_test_geom" in names # Spatial GiST index + assert "ix_gis_col_idx_true_test_geom" in names # User btree index + finally: + table.drop(conn, checkfirst=True) + + def test_spatial_index_long_table_name(self, conn): + metadata = MetaData( + schema="gis", + naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}, + ) + long_name = "lake_with_a_long_name_that_tests_index_truncation_rules_in_pg" + table = Table( + long_name, + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry( + geometry_type="POINT", + spatial_index=True, + follow_naming_convention=True, + ), + ), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + index_query = text( + f"""SELECT count(*) + FROM pg_indexes + WHERE schemaname = 'gis' AND tablename = '{long_name}';""" + ) + assert conn.execute(index_query).scalar() == 2 # pkey + geom + finally: + table.drop(conn, checkfirst=True) + class TestInsertionCore: def test_insert_geog_poi(self, conn, Poi, setup_tables): diff --git a/tests/test_functional_sqlite.py b/tests/test_functional_sqlite.py index fb5a1b4b..8e058393 100644 --- a/tests/test_functional_sqlite.py +++ b/tests/test_functional_sqlite.py @@ -205,6 +205,54 @@ def test_all_indexes(self, conn, TableWithIndexes, setup_tables): assert indexes_after_drop == [] assert [table for table in tables_after_drop if "table_with_indexes" in table.name] == [] + @test_only_with_dialects("sqlite-spatialite3", "sqlite-spatialite4") + def test_spatial_index_follow_naming_convention(self, conn): + metadata = MetaData(naming_convention={"ix": "ix_%(table_name)s_%(column_0_N_name)s"}) + table = Table( + "conv_lake", + metadata, + Column("id", Integer, primary_key=True), + Column( + "geom", + Geometry(geometry_type="LINESTRING", srid=4326, follow_naming_convention=True), + ), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + # SpatiaLite R-tree virtual table is created under hardcoded name + assert self.check_spatial_idx(conn, "idx_conv_lake_geom") + # No standard SQLite btree index is created + btree_indexes = conn.execute( + text( + "SELECT name FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = 'conv_lake';" + ) + ).fetchall() + assert not any("ix_conv_lake_geom" in i[0] for i in btree_indexes) + finally: + table.drop(conn, checkfirst=True) + assert not self.check_spatial_idx(conn, "idx_conv_lake_geom") + + @test_only_with_dialects("sqlite-spatialite3", "sqlite-spatialite4") + def test_recreate_reflected_table_with_spatial_index(self, conn, use_alembic_monkeypatch): + table = Table( + "recreate_lake", + MetaData(), + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="LINESTRING", srid=4326, spatial_index=True)), + ) + table.drop(conn, checkfirst=True) + table.create(conn) + try: + reflected_md = MetaData() + reflected_table = Table("recreate_lake", reflected_md, autoload_with=conn) + reflected_table.drop(conn) + reflected_table.create(conn) + assert self.check_spatial_idx(conn, "idx_recreate_lake_geom") + finally: + table.drop(conn, checkfirst=True) + class TestMiscellaneous: @test_only_with_dialects("sqlite-spatialite3", "sqlite-spatialite4") diff --git a/tests/test_types_mssql.py b/tests/test_types_mssql.py index 28b85740..3d560ae8 100644 --- a/tests/test_types_mssql.py +++ b/tests/test_types_mssql.py @@ -719,7 +719,46 @@ def create_index(index, bind=None, **kw): assert idx in table.indexes assert created_indexes == [(idx, bind, {})] - assert spatial_indexes == [] + # Multi-column user index must not suppress the generated spatial index (D2/D9) + assert len(spatial_indexes) == 1 + assert spatial_indexes[0][0][1] == "lake" + assert spatial_indexes[0][0][2] == "geom" + + def test_mssql_after_create_retains_generated_spatial_index_for_multiple_creates( + self, + monkeypatch, + ): + table = Table( + "lake", + MetaData(), + Column("id", Integer, primary_key=True), + Column("geom", Geometry(geometry_type="POINT", srid=4326)), + ) + bind = type( + "Bind", + (), + {"dialect": self.dialect, "exec_driver_sql": lambda *a, **k: None}, + )() + spatial_indexes = [] + monkeypatch.setattr(mssql_admin, "create_spatial_constraints", lambda *args, **kw: None) + monkeypatch.setattr( + mssql_admin, + "create_spatial_index", + lambda *args, **kw: spatial_indexes.append((args, kw)), + ) + + mssql_admin.before_create(table, bind) + mssql_admin.after_create(table, bind) + + # The generated index is kept in table.info, NOT table.indexes + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert "geom" in table.info.get("_generated_spatial_indexes", {}) + + # A second create cycle still uses the preserved index + mssql_admin.before_create(table, bind) + mssql_admin.after_create(table, bind) + assert len([i for i in table.indexes if table.c.geom in i.columns.values()]) == 0 + assert len(spatial_indexes) == 2 def test_mssql_after_create_uses_spatial_ddl_for_custom_single_spatial_index( self,