# -*- coding: utf-8 -*-
# Generated by Django 1.9.13 on 2019-12-30 20:52
from __future__ import unicode_literals, print_function

import os
import types
from datetime import timedelta, datetime

from django.db import migrations
from django.utils import timezone

from sentry import options
from sentry.eventstore.models import Event as NewEvent


def backfill_eventstream(apps, schema_editor):
    """
    Inserts Postgres events into the eventstream if there are recent events in Postgres.

    This is for open source users migrating from 9.x who want to keep their events.
    If there are no recent events in Postgres, skip the backfill.
    """
    from sentry import eventstore, eventstream
    from sentry.utils.query import RangeQuerySetWrapper

    Event = apps.get_model("sentry", "Event")
    Group = apps.get_model("sentry", "Group")
    Project = apps.get_model("sentry", "Project")

    # Kill switch to skip this migration
    skip_backfill = os.environ.get("SENTRY_SKIP_EVENTS_BACKFILL_FOR_10", False)

    # Use 90 day retention if the option has not been set or set to 0
    DEFAULT_RETENTION = 90
    retention_days = options.get("system.event-retention-days") or DEFAULT_RETENTION

    def get_events(last_days):
        to_date = timezone.now()
        from_date = to_date - timedelta(days=last_days)
        return Event.objects.filter(
            datetime__gte=from_date, datetime__lte=to_date, group_id__isnull=False
        )

    def _attach_related(_events):
        project_ids = set()
        group_ids = set()
        for event in _events:
            project_ids.add(event.project_id)
            group_ids.add(event.group_id)
        projects = {p.id: p for p in Project.objects.filter(id__in=project_ids)}
        groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)}

        for event in _events:
            event.project = projects.get(event.project_id)
            event.group = groups.get(event.group_id)
        eventstore.bind_nodes(_events, "data")

    if skip_backfill:
        print("Skipping backfill.\n")
        return

    events = get_events(retention_days)
    count = events.count()

    if count == 0:
        print("Nothing to do, skipping migration.\n")
        return

    print("Events to process: {}\n".format(count))

    processed = 0
    for e in RangeQuerySetWrapper(events, step=100, callbacks=(_attach_related,)):
        event = NewEvent(
            project_id=e.project_id, event_id=e.event_id, group_id=e.group_id, data=e.data.data
        )
        primary_hash = event.get_primary_hash()
        if event.project is None or event.group is None:
            print("Skipped {} as group or project information is invalid.\n".format(event))
            continue

        try:
            eventstream.insert(
                group=event.group,
                event=event,
                is_new=False,
                is_regression=False,
                is_new_group_environment=False,
                primary_hash=primary_hash,
                skip_consume=True,
            )
            processed += 1
        except Exception as error:
            print(
                "An error occured while trying to instert the following event: {}\n.----\n{}".format(
                    event, error
                )
            )

    print("Event migration done. Migrated {} of {} events.\n".format(processed, count))


class Migration(migrations.Migration):
    # This flag is used to mark that a migration shouldn't be automatically run in
    # production. We set this to True for operations that we think are risky and want
    # someone from ops to run manually and monitor.
    # General advice is that if in doubt, mark your migration as `is_dangerous`.
    # Some things you should always mark as dangerous:
    # - Adding indexes to large tables. These indexes should be created concurrently,
    #   unfortunately we can't run migrations outside of a transaction until Django
    #   1.10. So until then these should be run manually.
    # - Large data migrations. Typically we want these to be run manually by ops so that
    #   they can be monitored. Since data migrations will now hold a transaction open
    #   this is even more important.
    # - Adding columns to highly active tables, even ones that are NULL.
    is_dangerous = True

    dependencies = [
        ("sentry", "0023_hide_environment_none_20191126"),
    ]

    operations = [
        migrations.RunPython(backfill_eventstream, reverse_code=migrations.RunPython.noop),
    ]

src/sentry/migrations/0024_auto_20191230_2052.py
# -*- coding: utf-8 -*-
# Generated by Django 1.9.13 on 2019-12-30 20:52
from __future__ import unicode_literals, print_function

import os
import types
from datetime import timedelta, datetime

from django.db import migrations
from django.utils import timezone

from sentry import options
from sentry.eventstore.models import Event as NewEvent


def backfill_eventstream(apps, schema_editor):
    """
    Inserts Postgres events into the eventstream if there are recent events in Postgres.

    This is for open source users migrating from 9.x who want to keep their events.
    If there are no recent events in Postgres, skip the backfill.
    """
    from sentry import eventstore, eventstream
    from sentry.utils.query import RangeQuerySetWrapper

    Event = apps.get_model("sentry", "Event")
    Group = apps.get_model("sentry", "Group")
    Project = apps.get_model("sentry", "Project")

    # Kill switch to skip this migration
    skip_backfill = os.environ.get("SENTRY_SKIP_EVENTS_BACKFILL_FOR_10", False)

    # Use 90 day retention if the option has not been set or set to 0
    DEFAULT_RETENTION = 90
    retention_days = options.get("system.event-retention-days") or DEFAULT_RETENTION

    def get_events(last_days):
        to_date = timezone.now()
        from_date = to_date - timedelta(days=last_days)
        return Event.objects.filter(
            datetime__gte=from_date, datetime__lte=to_date, group_id__isnull=False
        )

    def _attach_related(_events):
        project_ids = set()
        group_ids = set()
        for event in _events:
            project_ids.add(event.project_id)
            group_ids.add(event.group_id)
        projects = {p.id: p for p in Project.objects.filter(id__in=project_ids)}
        groups = {g.id: g for g in Group.objects.filter(id__in=group_ids)}

        for event in _events:
            event.project = projects.get(event.project_id)
            event.group = groups.get(event.group_id)
        eventstore.bind_nodes(_events, "data")

    if skip_backfill:
        print("Skipping backfill.\n")
        return

    events = get_events(retention_days)
    count = events.count()

    if count == 0:
        print("Nothing to do, skipping migration.\n")
        return

    print("Events to process: {}\n".format(count))

    processed = 0
    for e in RangeQuerySetWrapper(events, step=100, callbacks=(_attach_related,)):
        event = NewEvent(
            project_id=e.project_id, event_id=e.event_id, group_id=e.group_id, data=e.data.data
        )
        primary_hash = event.get_primary_hash()
        if event.project is None or event.group is None:
            print("Skipped {} as group or project information is invalid.\n".format(event))
            continue

        try:
            eventstream.insert(
                group=event.group,
                event=event,
                is_new=False,
                is_regression=False,
                is_new_group_environment=False,
                primary_hash=primary_hash,
                received_timestamp=event.data.get("received")
                or float(event.datetime.strftime("%s")),
                skip_consume=True,
            )
            processed += 1
        except Exception as error:
            print(
                "An error occured while trying to instert the following event: {}\n.----\n{}".format(
                    event, error
                )
            )

    print("Event migration done. Migrated {} of {} events.\n".format(processed, count))


class Migration(migrations.Migration):
    # This flag is used to mark that a migration shouldn't be automatically run in
    # production. We set this to True for operations that we think are risky and want
    # someone from ops to run manually and monitor.
    # General advice is that if in doubt, mark your migration as `is_dangerous`.
    # Some things you should always mark as dangerous:
    # - Adding indexes to large tables. These indexes should be created concurrently,
    #   unfortunately we can't run migrations outside of a transaction until Django
    #   1.10. So until then these should be run manually.
    # - Large data migrations. Typically we want these to be run manually by ops so that
    #   they can be monitored. Since data migrations will now hold a transaction open
    #   this is even more important.
    # - Adding columns to highly active tables, even ones that are NULL.
    is_dangerous = True

    dependencies = [
        ("sentry", "0023_hide_environment_none_20191126"),
    ]

    operations = [
        migrations.RunPython(backfill_eventstream, reverse_code=migrations.RunPython.noop),
    ]

src/sentry/migrations/0024_auto_20191230_2052.py
diff --git a/src/sentry/migrations/0024_auto_20191230_2052.py b/src/sentry/migrations/0024_auto_20191230_2052.py
index 8971814d2643ea..4544195bba4d8c 100644
--- a/src/sentry/migrations/0024_auto_20191230_2052.py
+++ b/src/sentry/migrations/0024_auto_20191230_2052.py
@@ -86,6 +86,8 @@ def _attach_related(_events):
                 is_regression=False,
                 is_new_group_environment=False,
                 primary_hash=primary_hash,
+                received_timestamp=event.data.get("received")
+                or float(event.datetime.strftime("%s")),
                 skip_consume=True,
             )
             processed += 1

Event migration 9.1.2 -> 10
<!--

Do you want to ask a question? Are you looking for support? The Sentry message
board is the best place for getting support: https://forum.sentry.io
-->

## Important Details

How are you running Sentry?

* [X] On-Premise docker [Version 9.1.2]
* [ ] Saas (sentry.io)
* [ ] Other [briefly describe your environment]

## Description

I followed the migration guide, alongside all fixes and workaround and managed to get to the actual migration routine. Sentry tries to process all existing postgres events but fails to (for every event):

```
An error occured while trying to instert the following event: <sentry.eventstore.models.Event object at 0x7f2f08e552d0>
.----
insert() takes at least 8 arguments (8 given)
[...]
Event migration done. Migrated 0 of 197988 events.
```

## Steps to Reproduce

1. Have a 9.1.2 onpremise setup and have event data
2. Upgrade to 10 (dev-master), run `install.sh` etc.

### What you expected to happen

Migration scripts succeeds and I have all event data in the new version.

### Possible Solution

Error message suggests a syntax error?

