使用SQLAlchemy定制PostgreSQL实体化视图的迁移

时间:2018-01-25 13:01:35

标签: python sqlalchemy alembic

我正在尝试在SQLAlchemy中为物化视图自动生成迁移,唯一不起作用的是检测修订之间的更改。更具体地说,即使模型没有改变,总是也会产生变化。

我认为问题是,我可能只是在这里使用错误的方法,是schema.compile(compile_kwargs={'literal_binds': True})生成的SQL与inspector.get_view_definition(name)返回的SQL之间的区别。

备份,这是模型定义的样子,只是一个简单的邻接列表树。使用物化视图的目的是自动生成节点的深度和路径,以便可以查询它:

from app.extensions import db  # Flask-SQLAlchemy

@db.slugify('name')
class Node(db.Model):
    id = db.Column(db.BigInteger, primary_key=True)
    name = db.Column(db.String, index=True)
    slug = db.Column(db.String, index=True)

    parent_id = db.foreign_key('Node', nullable=True)
    parent = db.relationship('Node', back_populates='children', remote_side=id)
    children = db.relationship('Node', back_populates='parent')

    mv = db.relationship('NodeMV', uselist=False, foreign_keys='NodeMV.id',
                         primaryjoin='Node.id == NodeMV.id')
    depth = db.association_proxy('mv', 'depth')
    path = db.association_proxy('mv', 'path')

_cte = (db.select([Node.id.label('id'),
                   literal(0).label('depth'),
                   literal('/').label('path')])
        .where(Node.parent_id is None)
        .cte(name='nodes_cte', recursive=True))
_union = _cte.union_all(
    db.select([
        Node.id.label('id'),
        label('depth', _cte.c.depth + 1),
        label('path', case([(_cte.c.depth == 0, _cte.c.path + Node.slug)],
                           else_=_cte.c.path + '/' + Node.slug))])
    .select_from(db.join(_cte, Node, _cte.c.id == Node.parent_id))
)

class NodeMV(db.Model):
    __tablename__ = 'node_mv'
    __table__ = create_materialized_view('node_mv', db.select([_union]))

db.Index('ix_node_mv_id', NodeMV.id, unique=True)

create_materialized_view工厂方法的位置如下:

def create_materialized_view(name, selectable, metadata=db.metadata):
    # must use a temporary metadata here so that SQLAlchemy doesn't detect the
    # table as "standalone". (it will still use the correct metadata once
    # attached to the __table__ attribute of the declarative base model)
    table = db.Table(name, db.MetaData())
    for col in selectable.c:
        table.append_column(
            db.Column(col.name, col.type, primary_key=col.primary_key))

    if not any([col.primary_key for col in selectable.c]):
        table.append_constraint(
            db.PrimaryKeyConstraint(*[col.name for col in selectable.c]))

    # to support using db.drop_all()
    db.event.listen(metadata, 'before_drop',
                    db.DDL(f'DROP MATERIALIZED VIEW IF EXISTS {name}'))

    # to support auto-generated migrations
    metadata.info.setdefault('materialized_views', set()).add(
        (name, selectable))
    metadata.info.setdefault('materialized_tables', {})[name] = table

    return table

检测更改的迁移比较器

class ReplaceableSql:
    def __init__(self, name, create_sql, drop_sql, indexes, **kwargs):
        self.name = name
        self.create_sql = create_sql
        self.drop_sql = drop_sql
        self.indexes = indexes
        self.kwargs = kwargs

    def __eq__(self, other):
        if not isinstance(other, ReplaceableSql):
            return False
        return self.__hash__() == other.__hash__()

    def __hash__(self):
        key = (self.name, re.sub(r'\s+', ' ', self.create_sql), self.indexes)
        return hash(key)

def create_replaceable_sql(name, schema, metadata):
    query = (schema if isinstance(schema, str)
                    else schema.compile(compile_kwargs={'literal_binds': True}))
    indexes = [(index.name, [c.name for c in index.columns], index.unique)
               for index in metadata.info['materialized_tables'][name].indexes]
    return ReplaceableSql(name,
                          f'CREATE MATERIALIZED VIEW {name} AS {query}',
                          f'DROP MATERIALIZED VIEW IF EXISTS {name}',
                          indexes)

@comparators.dispatch_for('schema')
def compare_views(autogen_context, upgrade_ops, schemas):
    inspector = autogen_context.inspector
    metadata = autogen_context.metadata
    prev_revision = autogen_context.migration_context.get_current_revision()

    # existing views
    views = set()
    view_names = set()
    for schema in schemas:
        for name in inspector.get_view_names(include='materialized', schema=schema):
            views.add(create_replaceable_sql(
                name, inspector.get_view_definition(name, schema=schema), metadata))
            view_names.add(name)

    # current metadata views
    metadata_views = set()
    metadata_view_names = set()
    for name, selectable in metadata.info.setdefault('materialized_views', set()):
        metadata_views.add(create_replaceable_sql(name, selectable, metadata))
        metadata_view_names.add(name)

    creates = metadata_views.difference(views)
    drops = views.difference(metadata_views)
    upgrades = view_names.intersection(metadata_view_names)

    for replaceable_sql in creates:
        name = replaceable_sql.name
        if name in upgrades:
            replaceable_sql.kwargs['prev'] = f'{prev_revision}.{name}'
        upgrade_ops.ops.append(CreateOp(replaceable_sql))

    for replaceable_sql in drops:
        if replaceable_sql.name in upgrades:
            continue
        upgrade_ops.ops.append(DropOp(replaceable_sql))

检查与生成的SQL之间的差异很小(忽略空格),但它们足以甩掉字符串比较:

inspector.get_view_definition(表名)

WITH RECURSIVE nodes_cte(id, depth, path) AS (
    SELECT
      node.id,
      0 AS depth,
      '/'::text AS path
    FROM node
    WHERE false
  UNION ALL
    SELECT
      node.id,
      (nodes_cte_1.depth + 1) AS depth,
      CASE
        WHEN (nodes_cte_1.depth = 0) THEN (nodes_cte_1.path || (node.slug)::text)
        ELSE ((nodes_cte_1.path || '/'::text) || (node.slug)::text)
      END AS path
    FROM (nodes_cte nodes_cte_1
    JOIN node ON ((nodes_cte_1.id = node.parent_id)))
)
SELECT nodes_cte.id, nodes_cte.depth, nodes_cte.path FROM nodes_cte;

schema.compile(compile_kwargs = {'literal_binds':True})

WITH RECURSIVE nodes_cte(id, depth, path) AS (
    SELECT
      node.id AS id,
      0 AS depth,
      '/' AS path
    FROM node
    WHERE false
  UNION ALL
    SELECT
      node.id AS id,
      nodes_cte.depth + 1 AS depth,
      CASE
        WHEN (nodes_cte.depth = 0) THEN nodes_cte.path || node.slug
        ELSE nodes_cte.path || '/' || node.slug
      END AS path
    FROM nodes_cte
    JOIN node ON nodes_cte.id = node.parent_id
)
SELECT nodes_cte.id, nodes_cte.depth, nodes_cte.path FROM nodes_cte

所以我想问题是,我应该采取什么样的正确方法? (也可能使用物化视图是解决我问题的错误方法 - 我确实尝试使用@hybrid_property / @prop_name.expression的东西,但我无法弄清楚如何使其与递归一起工作数据的性质。)

提前致谢!

0 个答案:

没有答案