On Mon, 29 May 2017, Raphael Hertzog wrote:
> Updated patches attached, I missed to update some tests to account
> for the move of the detect_soft_applied() method.
Third set of patches, this time the package builds fine at least.
Which means you can just test this package and let me know if it fixes
your issue:
$ dget
https://people.debian.org/~hertzog/packages/python-django_1.10.7-2~test1_amd64.changes
Cheers,
--
Raphaël Hertzog ◈ Debian Developer
Support Debian LTS: https://www.freexian.com/services/debian-lts.html
Learn to master Debian: https://debian-handbook.info/get/
>From c6d66195d7f816aeb47a77570bdd3836a99d4183 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Hertzog?= <hert...@debian.org>
Date: Mon, 29 May 2017 15:44:39 +0200
Subject: [PATCH 1/2] Move detect_soft_applied() from
django.db.migrations.executor to .loader
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We want to be able to use that method in
loader.check_consistent_history() to accept an history where the initial
migration is going to be fake-applied. Since the executor has the
knowledge of the loader (but not the opposite), it makes sens to move
the code around.
Signed-off-by: Raphaël Hertzog <hert...@debian.org>
---
django/db/migrations/executor.py | 83 +--------------------------------------
django/db/migrations/loader.py | 81 ++++++++++++++++++++++++++++++++++++++
tests/migrations/test_executor.py | 12 +++---
3 files changed, 88 insertions(+), 88 deletions(-)
diff --git a/django/db/migrations/executor.py b/django/db/migrations/executor.py
index 1a0b6f6322..2ac787b0b2 100644
--- a/django/db/migrations/executor.py
+++ b/django/db/migrations/executor.py
@@ -1,8 +1,5 @@
from __future__ import unicode_literals
-from django.apps.registry import apps as global_apps
-from django.db import migrations, router
-
from .exceptions import InvalidMigrationPlan
from .loader import MigrationLoader
from .recorder import MigrationRecorder
@@ -235,7 +232,7 @@ class MigrationExecutor(object):
if not fake:
if fake_initial:
# Test to see if this is an already-applied initial migration
- applied, state = self.detect_soft_applied(state, migration)
+ applied, state = self.loader.detect_soft_applied(state, migration)
if applied:
fake = True
if not fake:
@@ -290,81 +287,3 @@ class MigrationExecutor(object):
if all_applied and key not in applied:
self.recorder.record_applied(*key)
- def detect_soft_applied(self, project_state, migration):
- """
- Tests whether a migration has been implicitly applied - that the
- tables or columns it would create exist. This is intended only for use
- on initial migrations (as it only looks for CreateModel and AddField).
- """
- def should_skip_detecting_model(migration, model):
- """
- No need to detect tables for proxy models, unmanaged models, or
- models that can't be migrated on the current database.
- """
- return (
- model._meta.proxy or not model._meta.managed or not
- router.allow_migrate(
- self.connection.alias, migration.app_label,
- model_name=model._meta.model_name,
- )
- )
-
- if migration.initial is None:
- # Bail if the migration isn't the first one in its app
- if any(app == migration.app_label for app, name in migration.dependencies):
- return False, project_state
- elif migration.initial is False:
- # Bail if it's NOT an initial migration
- return False, project_state
-
- if project_state is None:
- after_state = self.loader.project_state((migration.app_label, migration.name), at_end=True)
- else:
- after_state = migration.mutate_state(project_state)
- apps = after_state.apps
- found_create_model_migration = False
- found_add_field_migration = False
- existing_table_names = self.connection.introspection.table_names(self.connection.cursor())
- # Make sure all create model and add field operations are done
- for operation in migration.operations:
- if isinstance(operation, migrations.CreateModel):
- model = apps.get_model(migration.app_label, operation.name)
- if model._meta.swapped:
- # We have to fetch the model to test with from the
- # main app cache, as it's not a direct dependency.
- model = global_apps.get_model(model._meta.swapped)
- if should_skip_detecting_model(migration, model):
- continue
- if model._meta.db_table not in existing_table_names:
- return False, project_state
- found_create_model_migration = True
- elif isinstance(operation, migrations.AddField):
- model = apps.get_model(migration.app_label, operation.model_name)
- if model._meta.swapped:
- # We have to fetch the model to test with from the
- # main app cache, as it's not a direct dependency.
- model = global_apps.get_model(model._meta.swapped)
- if should_skip_detecting_model(migration, model):
- continue
-
- table = model._meta.db_table
- field = model._meta.get_field(operation.name)
-
- # Handle implicit many-to-many tables created by AddField.
- if field.many_to_many:
- if field.remote_field.through._meta.db_table not in existing_table_names:
- return False, project_state
- else:
- found_add_field_migration = True
- continue
-
- column_names = [
- column.name for column in
- self.connection.introspection.get_table_description(self.connection.cursor(), table)
- ]
- if field.column not in column_names:
- return False, project_state
- found_add_field_migration = True
- # If we get this far and we found at least one CreateModel or AddField migration,
- # the migration is considered implicitly applied.
- return (found_create_model_migration or found_add_field_migration), after_state
diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py
index 3a34e6da33..eb75448c2c 100644
--- a/django/db/migrations/loader.py
+++ b/django/db/migrations/loader.py
@@ -5,7 +5,9 @@ import sys
from importlib import import_module
from django.apps import apps
+from django.apps.registry import apps as global_apps
from django.conf import settings
+from django.db import migrations, router
from django.db.migrations.graph import MigrationGraph
from django.db.migrations.recorder import MigrationRecorder
from django.utils import six
@@ -315,3 +317,82 @@ class MigrationLoader(object):
See graph.make_state for the meaning of "nodes" and "at_end"
"""
return self.graph.make_state(nodes=nodes, at_end=at_end, real_apps=list(self.unmigrated_apps))
+
+ def detect_soft_applied(self, project_state, migration):
+ """
+ Tests whether a migration has been implicitly applied - that the
+ tables or columns it would create exist. This is intended only for use
+ on initial migrations (as it only looks for CreateModel and AddField).
+ """
+ def should_skip_detecting_model(migration, model):
+ """
+ No need to detect tables for proxy models, unmanaged models, or
+ models that can't be migrated on the current database.
+ """
+ return (
+ model._meta.proxy or not model._meta.managed or not
+ router.allow_migrate(
+ self.connection.alias, migration.app_label,
+ model_name=model._meta.model_name,
+ )
+ )
+
+ if migration.initial is None:
+ # Bail if the migration isn't the first one in its app
+ if any(app == migration.app_label for app, name in migration.dependencies):
+ return False, project_state
+ elif migration.initial is False:
+ # Bail if it's NOT an initial migration
+ return False, project_state
+
+ if project_state is None:
+ after_state = self.project_state((migration.app_label, migration.name), at_end=True)
+ else:
+ after_state = migration.mutate_state(project_state)
+ apps = after_state.apps
+ found_create_model_migration = False
+ found_add_field_migration = False
+ existing_table_names = self.connection.introspection.table_names(self.connection.cursor())
+ # Make sure all create model and add field operations are done
+ for operation in migration.operations:
+ if isinstance(operation, migrations.CreateModel):
+ model = apps.get_model(migration.app_label, operation.name)
+ if model._meta.swapped:
+ # We have to fetch the model to test with from the
+ # main app cache, as it's not a direct dependency.
+ model = global_apps.get_model(model._meta.swapped)
+ if should_skip_detecting_model(migration, model):
+ continue
+ if model._meta.db_table not in existing_table_names:
+ return False, project_state
+ found_create_model_migration = True
+ elif isinstance(operation, migrations.AddField):
+ model = apps.get_model(migration.app_label, operation.model_name)
+ if model._meta.swapped:
+ # We have to fetch the model to test with from the
+ # main app cache, as it's not a direct dependency.
+ model = global_apps.get_model(model._meta.swapped)
+ if should_skip_detecting_model(migration, model):
+ continue
+
+ table = model._meta.db_table
+ field = model._meta.get_field(operation.name)
+
+ # Handle implicit many-to-many tables created by AddField.
+ if field.many_to_many:
+ if field.remote_field.through._meta.db_table not in existing_table_names:
+ return False, project_state
+ else:
+ found_add_field_migration = True
+ continue
+
+ column_names = [
+ column.name for column in
+ self.connection.introspection.get_table_description(self.connection.cursor(), table)
+ ]
+ if field.column not in column_names:
+ return False, project_state
+ found_add_field_migration = True
+ # If we get this far and we found at least one CreateModel or AddField migration,
+ # the migration is considered implicitly applied.
+ return (found_create_model_migration or found_add_field_migration), after_state
diff --git a/tests/migrations/test_executor.py b/tests/migrations/test_executor.py
index 847b11c088..d0f5228d96 100644
--- a/tests/migrations/test_executor.py
+++ b/tests/migrations/test_executor.py
@@ -326,7 +326,7 @@ class ExecutorTests(MigrationTestBase):
global_apps.get_app_config("migrations").models["author"] = migrations_apps.get_model("migrations", "author")
try:
migration = executor.loader.get_migration("auth", "0001_initial")
- self.assertIs(executor.detect_soft_applied(None, migration)[0], True)
+ self.assertIs(executor.loader.detect_soft_applied(None, migration)[0], True)
finally:
connection.introspection.table_names = old_table_names
del global_apps.get_app_config("migrations").models["author"]
@@ -343,7 +343,7 @@ class ExecutorTests(MigrationTestBase):
)
def test_detect_soft_applied_add_field_manytomanyfield(self):
"""
- executor.detect_soft_applied() detects ManyToManyField tables from an
+ loader.detect_soft_applied() detects ManyToManyField tables from an
AddField operation. This checks the case of AddField in a migration
with other operations (0001) and the case of AddField in its own
migration (0002).
@@ -365,9 +365,9 @@ class ExecutorTests(MigrationTestBase):
self.assertTableExists(table)
# Table detection sees 0001 is applied but not 0002.
migration = executor.loader.get_migration("migrations", "0001_initial")
- self.assertIs(executor.detect_soft_applied(None, migration)[0], True)
+ self.assertIs(executor.loader.detect_soft_applied(None, migration)[0], True)
migration = executor.loader.get_migration("migrations", "0002_initial")
- self.assertIs(executor.detect_soft_applied(None, migration)[0], False)
+ self.assertIs(executor.loader.detect_soft_applied(None, migration)[0], False)
# Create the tables for both migrations but make it look like neither
# has been applied.
@@ -378,7 +378,7 @@ class ExecutorTests(MigrationTestBase):
executor.migrate([("migrations", None)], fake=True)
# Table detection sees 0002 is applied.
migration = executor.loader.get_migration("migrations", "0002_initial")
- self.assertIs(executor.detect_soft_applied(None, migration)[0], True)
+ self.assertIs(executor.loader.detect_soft_applied(None, migration)[0], True)
# Leave the tables for 0001 except the many-to-many table. That missing
# table should cause detect_soft_applied() to return False.
@@ -386,7 +386,7 @@ class ExecutorTests(MigrationTestBase):
for table in tables[2:]:
editor.execute(editor.sql_delete_table % {"table": table})
migration = executor.loader.get_migration("migrations", "0001_initial")
- self.assertIs(executor.detect_soft_applied(None, migration)[0], False)
+ self.assertIs(executor.loader.detect_soft_applied(None, migration)[0], False)
# Cleanup by removing the remaining tables.
with connection.schema_editor() as editor:
--
2.11.0
>From a9746e85b9adbf8d4f29d820ff18b41c0cdf1035 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Hertzog?= <hert...@debian.org>
Date: Mon, 29 May 2017 16:20:49 +0200
Subject: [PATCH 2/2] Fixed #25850 -- improve migration history consistency
check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
With this change, we are now accepting unsatisfied dependencies on initial
migrations that are going to be fake-applied in the next migrate run.
This is a regression compared to 1.8 when we have to deal with
migration dependencies on applications who started without migrations.
Signed-off-by: Raphaël Hertzog <hert...@debian.org>
---
django/core/management/commands/migrate.py | 3 ++-
django/db/migrations/loader.py | 5 ++++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/django/core/management/commands/migrate.py b/django/core/management/commands/migrate.py
index 6846b33d2a..e589b0b0ae 100644
--- a/django/core/management/commands/migrate.py
+++ b/django/core/management/commands/migrate.py
@@ -83,7 +83,8 @@ class Command(BaseCommand):
executor = MigrationExecutor(connection, self.migration_progress_callback)
# Raise an error if any migrations are applied before their dependencies.
- executor.loader.check_consistent_history(connection)
+ executor.loader.check_consistent_history(
+ connection, fake_initial=options['fake_initial'])
# Before anything else, see if there's conflicting apps and drop out
# hard if there are any
diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py
index eb75448c2c..71ec2eead6 100644
--- a/django/db/migrations/loader.py
+++ b/django/db/migrations/loader.py
@@ -269,7 +269,7 @@ class MigrationLoader(object):
six.reraise(NodeNotFoundError, exc_value, sys.exc_info()[2])
raise exc
- def check_consistent_history(self, connection):
+ def check_consistent_history(self, connection, fake_initial=False):
"""
Raise InconsistentMigrationHistory if any applied migrations have
unapplied dependencies.
@@ -287,6 +287,9 @@ class MigrationLoader(object):
if parent in self.replacements:
if all(m in applied for m in self.replacements[parent].replaces):
continue
+ # Skip initial migration that is going to be fake-applied
+ if fake_initial and self.detect_soft_applied(None, parent):
+ continue
raise InconsistentMigrationHistory(
"Migration {}.{} is applied before its dependency "
"{}.{} on database '{}'.".format(
--
2.11.0
