synapse/storage/prepare_database.py
def _upgrade_existing_database(cur, current_version, applied_delta_files,
                               upgraded, database_engine, config, is_empty=False):
    """Upgrades an existing database.

    Delta files can either be SQL stored in *.sql files, or python modules
    in *.py.

    There can be multiple delta files per version. Synapse will keep track of
    which delta files have been applied, and will apply any that haven't been
    even if there has been no version bump. This is useful for development
    where orthogonal schema changes may happen on separate branches.

    Different delta files for the same version *must* be orthogonal and give
    the same result when applied in any order. No guarantees are made on the
    order of execution of these scripts.

    This is a no-op of current_version == SCHEMA_VERSION.

    Example directory structure:

        schema/
            delta/
                11/
                    foo.sql
                    ...
                12/
                    foo.sql
                    bar.py
                ...
            full_schemas/
                ...

    In the example, if current_version is 11, then foo.sql will be run if and
    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
    some arbitrary order.

    Args:
        cur (Cursor)
        current_version (int): The current version of the schema.
        applied_delta_files (list): A list of deltas that have already been
            applied.
        upgraded (bool): Whether the current version was generated by having
            applied deltas or from full schema file. If `True` the function
            will never apply delta files for the given `current_version`, since
            the current_version wasn't generated by applying those delta files.
    """

    if current_version > SCHEMA_VERSION:
        raise ValueError(
            "Cannot use this database as it is too " +
            "new for the server to understand"
        )

    start_ver = current_version
    if not upgraded:
        start_ver += 1

    logger.debug("applied_delta_files: %s", applied_delta_files)

    for v in range(start_ver, SCHEMA_VERSION + 1):
        logger.info("Upgrading schema to v%d", v)

        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))

        try:
            directory_entries = os.listdir(delta_dir)
        except OSError:
            logger.exception("Could not open delta dir for version %d", v)
            raise UpgradeDatabaseException(
                "Could not open delta dir for version %d" % (v,)
            )

        directory_entries.sort()
        for file_name in directory_entries:
            relative_path = os.path.join(str(v), file_name)
            logger.debug("Found file: %s", relative_path)
            if relative_path in applied_delta_files:
                continue

            absolute_path = os.path.join(
                dir_path, "schema", "delta", relative_path,
            )
            root_name, ext = os.path.splitext(file_name)
            if ext == ".py":
                # This is a python upgrade module. We need to import into some
                # package and then execute its `run_upgrade` function.
                module_name = "synapse.storage.v%d_%s" % (
                    v, root_name
                )
                with open(absolute_path) as python_file:
                    module = imp.load_source(
                        module_name, absolute_path, python_file
                    )
                logger.info("Running script %s", relative_path)
                module.run_create(cur, database_engine)
                if not is_empty:
                    module.run_upgrade(cur, database_engine, config=config)
            elif ext == ".pyc":
                # Sometimes .pyc files turn up anyway even though we've
                # disabled their generation; e.g. from distribution package
                # installers. Silently skip it
                pass
            elif ext == ".sql":
                # A plain old .sql file, just read and execute it
                logger.info("Applying schema %s", relative_path)
                executescript(cur, absolute_path)
            else:
                # Not a valid delta file.
                logger.warn(
                    "Found directory entry that did not end in .py or"
                    " .sql: %s",
                    relative_path,
                )
                continue

            # Mark as done.
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO applied_schema_deltas (version, file)"
                    " VALUES (?,?)",
                ),
                (v, relative_path)
            )

            cur.execute("DELETE FROM schema_version")
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO schema_version (version, upgraded)"
                    " VALUES (?,?)",
                ),
                (v, True)
            )
Upgrades an existing database.

    Delta files can either be SQL stored in *.sql files, or python modules
    in *.py.

    There can be multiple delta files per version. Synapse will keep track of
    which delta files have been applied, and will apply any that haven't been
    even if there has been no version bump. This is useful for development
    where orthogonal schema changes may happen on separate branches.

    Different delta files for the same version *must* be orthogonal and give
    the same result when applied in any order. No guarantees are made on the
    order of execution of these scripts.

    This is a no-op of current_version == SCHEMA_VERSION.

    Example directory structure:

        schema/
            delta/
                11/
                    foo.sql
                    ...
                12/
                    foo.sql
                    bar.py
                ...
            full_schemas/
                ...

    In the example, if current_version is 11, then foo.sql will be run if and
    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
    some arbitrary order.

    Args:
        cur (Cursor)
        current_version (int): The current version of the schema.
        applied_delta_files (list): A list of deltas that have already been
            applied.
        upgraded (bool): Whether the current version was generated by having
            applied deltas or from full schema file. If `True` the function
            will never apply delta files for the given `current_version`, since
            the current_version wasn't generated by applying those delta files.
    
Upgrades an existing database.
Delta files can either be SQL stored in *.sql files, or python modules
in *.py.

There can be multiple delta files per version. Synapse will keep track of
which delta files have been applied, and will apply any that haven't been
even if there has been no version bump. This is useful for development
where orthogonal schema changes may happen on separate branches.

Different delta files for the same version *must* be orthogonal and give
the same result when applied in any order. No guarantees are made on the
order of execution of these scripts.

This is a no-op of current_version == SCHEMA_VERSION.

Example directory structure.



In the example, if current_version is 11, then foo.sql will be run if and
only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
some arbitrary order.

cur (Cursor)
current_version (int): The current version of the schema.
applied_delta_files (list): A list of deltas that have already been
applied.
upgraded (bool): Whether the current version was generated by having
applied deltas or from full schema file. If `True` the function
will never apply delta files for the given `current_version`, since
the current_version wasn't generated by applying those delta files.
Upgrades an existing database . Delta files can either be SQL stored in * . sql files or python modules in * . py . There can be multiple delta files per version . Synapse will keep track of which delta files have been applied and will apply any that haven ' t been even if there has been no version bump . This is useful for development where orthogonal schema changes may happen on separate branches . Different delta files for the same version * must * be orthogonal and give the same result when applied in any order . No guarantees are made on the order of execution of these scripts . This is a no - op of current_version == SCHEMA_VERSION . Example directory structure . In the example if current_version is 11 then foo . sql will be run if and only if ` upgraded ` is True . Then ` foo . sql ` and ` bar . py ` would be run in some arbitrary order . cur ( Cursor ) current_version ( int ) : The current version of the schema . applied_delta_files ( list ) : A list of deltas that have already been applied . upgraded ( bool ) : Whether the current version was generated by having applied deltas or from full schema file . If ` True ` the function will never apply delta files for the given ` current_version ` since the current_version wasn ' t generated by applying those delta files .
def _upgrade_existing_database(cur, current_version, applied_delta_files,
                               upgraded, database_engine, config, is_empty=False):
    if current_version > SCHEMA_VERSION:
        raise ValueError(
            "Cannot use this database as it is too " +
            "new for the server to understand"
        )
    start_ver = current_version
    if not upgraded:
        start_ver += 1
    logger.debug("applied_delta_files: %s", applied_delta_files)
    for v in range(start_ver, SCHEMA_VERSION + 1):
        logger.info("Upgrading schema to v%d", v)
        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
        try:
            directory_entries = os.listdir(delta_dir)
        except OSError:
            logger.exception("Could not open delta dir for version %d", v)
            raise UpgradeDatabaseException(
                "Could not open delta dir for version %d" % (v,)
            )
        directory_entries.sort()
        for file_name in directory_entries:
            relative_path = os.path.join(str(v), file_name)
            logger.debug("Found file: %s", relative_path)
            if relative_path in applied_delta_files:
                continue
            absolute_path = os.path.join(
                dir_path, "schema", "delta", relative_path,
            )
            root_name, ext = os.path.splitext(file_name)
            if ext == ".py":
                module_name = "synapse.storage.v%d_%s" % (
                    v, root_name
                )
                with open(absolute_path) as python_file:
                    module = imp.load_source(
                        module_name, absolute_path, python_file
                    )
                logger.info("Running script %s", relative_path)
                module.run_create(cur, database_engine)
                if not is_empty:
                    module.run_upgrade(cur, database_engine, config=config)
            elif ext == ".pyc":
                pass
            elif ext == ".sql":
                logger.info("Applying schema %s", relative_path)
                executescript(cur, absolute_path)
            else:
                logger.warn(
                    "Found directory entry that did not end in .py or"
                    " .sql: %s",
                    relative_path,
                )
                continue
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO applied_schema_deltas (version, file)"
                    " VALUES (?,?)",
                ),
                (v, relative_path)
            )
            cur.execute("DELETE FROM schema_version")
            cur.execute(
                database_engine.convert_param_style(
                    "INSERT INTO schema_version (version, upgraded)"
                    " VALUES (?,?)",
                ),
                (v, True)
            )
def _upgrade_existing_database ( cur , current_version , applied_delta_files , upgraded , database_engine , config , is_empty = False ) : if current_version > SCHEMA_VERSION : raise ValueError ( "Cannot use this database as it is too " + "new for the server to understand" ) start_ver = current_version if not upgraded : start_ver += 1 logger . debug ( "applied_delta_files: %s" , applied_delta_files ) for v in range ( start_ver , SCHEMA_VERSION + 1 ) : logger . info ( "Upgrading schema to v%d" , v ) delta_dir = os . path . join ( dir_path , "schema" , "delta" , str ( v ) ) try : directory_entries = os . listdir ( delta_dir ) except OSError : logger . exception ( "Could not open delta dir for version %d" , v ) raise UpgradeDatabaseException ( "Could not open delta dir for version %d" % ( v , ) ) directory_entries . sort ( ) for file_name in directory_entries : relative_path = os . path . join ( str ( v ) , file_name ) logger . debug ( "Found file: %s" , relative_path ) if relative_path in applied_delta_files : continue absolute_path = os . path . join ( dir_path , "schema" , "delta" , relative_path , ) root_name , ext = os . path . splitext ( file_name ) if ext == ".py" : module_name = "synapse.storage.v%d_%s" % ( v , root_name ) with open ( absolute_path ) as python_file : module = imp . load_source ( module_name , absolute_path , python_file ) logger . info ( "Running script %s" , relative_path ) module . run_create ( cur , database_engine ) if not is_empty : module . run_upgrade ( cur , database_engine , config = config ) elif ext == ".pyc" : pass elif ext == ".sql" : logger . info ( "Applying schema %s" , relative_path ) executescript ( cur , absolute_path ) else : logger . warn ( "Found directory entry that did not end in .py or" " .sql: %s" , relative_path , ) continue cur . execute ( database_engine . convert_param_style ( "INSERT INTO applied_schema_deltas (version, file)" " VALUES (?,?)" , ) , ( v , relative_path ) ) cur . execute ( "DELETE FROM schema_version" ) cur . execute ( database_engine . convert_param_style ( "INSERT INTO schema_version (version, upgraded)" " VALUES (?,?)" , ) , ( v , True ) )
Upgrades an existing database.
Upgrades an existing database .
"""Upgrades an existing database.

    Delta files can either be SQL stored in *.sql files, or python modules
    in *.py.

    There can be multiple delta files per version. Synapse will keep track of
    which delta files have been applied, and will apply any that haven't been
    even if there has been no version bump. This is useful for development
    where orthogonal schema changes may happen on separate branches.

    Different delta files for the same version *must* be orthogonal and give
    the same result when applied in any order. No guarantees are made on the
    order of execution of these scripts.

    This is a no-op of current_version == SCHEMA_VERSION.

    Example directory structure:

        schema/
            delta/
                11/
                    foo.sql
                    ...
                12/
                    foo.sql
                    bar.py
                ...
            full_schemas/
                ...

    In the example, if current_version is 11, then foo.sql will be run if and
    only if `upgraded` is True. Then `foo.sql` and `bar.py` would be run in
    some arbitrary order.

    Args:
        cur (Cursor)
        current_version (int): The current version of the schema.
        applied_delta_files (list): A list of deltas that have already been
            applied.
        upgraded (bool): Whether the current version was generated by having
            applied deltas or from full schema file. If `True` the function
            will never apply delta files for the given `current_version`, since
            the current_version wasn't generated by applying those delta files.
    """ # This is a python upgrade module. We need to import into some # package and then execute its `run_upgrade` function. # Sometimes .pyc files turn up anyway even though we've # disabled their generation; e.g. from distribution package # installers. Silently skip it # A plain old .sql file, just read and execute it # Not a valid delta file. # Mark as done.
cur
current_version
applied_delta_files
upgraded
database_engine
config
is_empty