#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2008-2013 Zuza Software Foundation
# Copyright 2014 Evernote Corporation
#
# This file is part of Pootle.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import glob
import os
import re
import sys

from distutils import log
from distutils.command.build import build as DistutilsBuild
from distutils.errors import DistutilsOptionError

from setuptools import find_packages, setup
from setuptools.command.test import test as TestCommand

from pootle.__version__ import sver as pootle_version


def parse_requirements(file_name):
    """Parses a pip requirements file and returns a list of packages.

    Use the result of this function in the ``install_requires`` field.
    Copied from cburgmer/pdfserver.
    """
    requirements = []
    for line in open(file_name, 'r').read().split('\n'):
        # Ignore comments, blank lines and included requirements files
        if re.match(r'(\s*#)|(\s*$)|(-r .*$)', line):
            continue

        if re.match(r'\s*-e\s+', line):
            requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', line))
        elif re.match(r'\s*-f\s+', line):
            pass
        else:
            requirements.append(line)

    return requirements


class PyTest(TestCommand):

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = ['--tb=short', 'tests/']
        self.test_suite = True

    def run_tests(self):
        #import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main(self.test_args)
        sys.exit(errno)


class PootleBuildMo(DistutilsBuild):

    description = "compile Gettext PO files into MO"
    user_options = [
        ('all', None,
         "compile all language (don't use LINGUAS file)"),
        ('lang=', 'l',
         "specify a language to compile"),
    ]
    boolean_options = ['all']

    po_path_base = os.path.join('pootle', 'locale')
    _langs = []

    def initialize_options(self):
        self.all = False
        self.lang = None

    def finalize_options(self):
        if self.all and self.lang is not None:
            raise DistutilsOptionError(
                "Can't use --all and --lang together"
            )
        if self.lang is not None:
            self._langs = [self.lang]
        elif self.all:
            for lang in os.listdir(self.po_path_base):
                if (os.path.isdir(os.path.join(self.po_path_base, lang)) and
                    lang != "templates"):
                    self._langs.append(lang)
        else:
            for lang in open(os.path.join('pootle', 'locale', 'LINGUAS')):
                self._langs.append(lang.rstrip())

    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        for lang in self._langs:
            lang = lang.rstrip()

            po_path = os.path.join('pootle', 'locale', lang)
            mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')

            if not os.path.exists(mo_path):
                os.makedirs(mo_path)

            for po, mo in (('pootle.po', 'django.mo'),
                           ('pootle_js.po', 'djangojs.mo')):
                po_filename = os.path.join(po_path, po)
                mo_filename = os.path.join(mo_path, mo)

                if not os.path.exists(po_filename):
                    log.warn("%s: missing file %s", lang, po_filename)
                    continue

                if not os.path.exists(mo_path):
                    os.makedirs(mo_path)

                log.info("compiling %s", lang)
                try:
                    subprocess.call([
                        'msgfmt', '--strict', '-o', mo_filename, po_filename],
                        stderr=subprocess.STDOUT)
                except Exception as e:
                    log.warn("%s: skipping, running msgfmt failed: %s",
                             lang, e)

                try:
                    store = factory.getobject(po_filename)
                    gettext.c2py(store.getheaderplural()[1])
                except Exception:
                    log.warn("%s: invalid plural header in %s",
                             lang, po_filename)

    def run(self):
        self.build_mo()


setup(
    name="Pootle",
    version=pootle_version,

    description="An online collaborative localization tool.",
    long_description=open(
        os.path.join(os.path.dirname(__file__), 'README.rst')
    ).read(),

    author="Translate",
    author_email="dev@translate.org.za",
    license="GNU General Public License (GPL)",
    url="http://pootle.translatehouse.org",
    download_url="http://sourceforge.net/projects/translate/files/Pootle/" + pootle_version,

    install_requires=parse_requirements('requirements/base.txt'),
    tests_require=parse_requirements('requirements/tests.txt'),

    platforms=["any"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Framework :: Django",
        "Intended Audience :: Developers",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: Information Technology",
        "License :: OSI Approved :: GNU General Public License (GPL)",
        "Operating System :: OS Independent",
        "Operating System :: Microsoft :: Windows",
        "Operating System :: Unix",
        "Programming Language :: JavaScript",
        "Programming Language :: Python",
        "Topic :: Software Development :: Localization",
        "Topic :: Text Processing :: Linguistic"
    ],
    zip_safe=False,
    packages=find_packages(exclude=['deploy*']),
    include_package_data=True,

    entry_points={
        'console_scripts': [
            'pootle = pootle.runner:main',
        ],
    },
    cmdclass={
        'build_mo': PootleBuildMo,
        'test': PyTest,
    },
)

setup.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2008-2013 Zuza Software Foundation
# Copyright 2014 Evernote Corporation
#
# This file is part of Pootle.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

import glob
import os
import re
import sys

from distutils import log
from distutils.command.build import build as DistutilsBuild
from distutils.errors import DistutilsOptionError

from setuptools import find_packages, setup
from setuptools.command.test import test as TestCommand

from pootle.__version__ import sver as pootle_version


def parse_requirements(file_name):
    """Parses a pip requirements file and returns a list of packages.

    Use the result of this function in the ``install_requires`` field.
    Copied from cburgmer/pdfserver.
    """
    requirements = []
    for line in open(file_name, 'r').read().split('\n'):
        # Ignore comments, blank lines and included requirements files
        if re.match(r'(\s*#)|(\s*$)|((-r|--allow-external|--allow-unverified) .*$)', line):
            continue

        if re.match(r'\s*-e\s+', line):
            requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', line))
        elif re.match(r'\s*-f\s+', line):
            pass
        else:
            requirements.append(line)

    return requirements


class PyTest(TestCommand):

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = ['--tb=short', 'tests/']
        self.test_suite = True

    def run_tests(self):
        #import here, cause outside the eggs aren't loaded
        import pytest
        errno = pytest.main(self.test_args)
        sys.exit(errno)


class PootleBuildMo(DistutilsBuild):

    description = "compile Gettext PO files into MO"
    user_options = [
        ('all', None,
         "compile all language (don't use LINGUAS file)"),
        ('lang=', 'l',
         "specify a language to compile"),
    ]
    boolean_options = ['all']

    po_path_base = os.path.join('pootle', 'locale')
    _langs = []

    def initialize_options(self):
        self.all = False
        self.lang = None

    def finalize_options(self):
        if self.all and self.lang is not None:
            raise DistutilsOptionError(
                "Can't use --all and --lang together"
            )
        if self.lang is not None:
            self._langs = [self.lang]
        elif self.all:
            for lang in os.listdir(self.po_path_base):
                if (os.path.isdir(os.path.join(self.po_path_base, lang)) and
                    lang != "templates"):
                    self._langs.append(lang)
        else:
            for lang in open(os.path.join('pootle', 'locale', 'LINGUAS')):
                self._langs.append(lang.rstrip())

    def build_mo(self):
        """Compile .mo files from available .po files"""
        import subprocess
        import gettext
        from translate.storage import factory

        for lang in self._langs:
            lang = lang.rstrip()

            po_path = os.path.join('pootle', 'locale', lang)
            mo_path = os.path.join('pootle', 'locale', lang, 'LC_MESSAGES')

            if not os.path.exists(mo_path):
                os.makedirs(mo_path)

            for po, mo in (('pootle.po', 'django.mo'),
                           ('pootle_js.po', 'djangojs.mo')):
                po_filename = os.path.join(po_path, po)
                mo_filename = os.path.join(mo_path, mo)

                if not os.path.exists(po_filename):
                    log.warn("%s: missing file %s", lang, po_filename)
                    continue

                if not os.path.exists(mo_path):
                    os.makedirs(mo_path)

                log.info("compiling %s", lang)
                try:
                    subprocess.call([
                        'msgfmt', '--strict', '-o', mo_filename, po_filename],
                        stderr=subprocess.STDOUT)
                except Exception as e:
                    log.warn("%s: skipping, running msgfmt failed: %s",
                             lang, e)

                try:
                    store = factory.getobject(po_filename)
                    gettext.c2py(store.getheaderplural()[1])
                except Exception:
                    log.warn("%s: invalid plural header in %s",
                             lang, po_filename)

    def run(self):
        self.build_mo()


setup(
    name="Pootle",
    version=pootle_version,

    description="An online collaborative localization tool.",
    long_description=open(
        os.path.join(os.path.dirname(__file__), 'README.rst')
    ).read(),

    author="Translate",
    author_email="dev@translate.org.za",
    license="GNU General Public License (GPL)",
    url="http://pootle.translatehouse.org",
    download_url="http://sourceforge.net/projects/translate/files/Pootle/" + pootle_version,

    install_requires=parse_requirements('requirements/base.txt'),
    tests_require=parse_requirements('requirements/tests.txt'),

    platforms=["any"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Web Environment",
        "Framework :: Django",
        "Intended Audience :: Developers",
        "Intended Audience :: End Users/Desktop",
        "Intended Audience :: Information Technology",
        "License :: OSI Approved :: GNU General Public License (GPL)",
        "Operating System :: OS Independent",
        "Operating System :: Microsoft :: Windows",
        "Operating System :: Unix",
        "Programming Language :: JavaScript",
        "Programming Language :: Python",
        "Topic :: Software Development :: Localization",
        "Topic :: Text Processing :: Linguistic"
    ],
    zip_safe=False,
    packages=find_packages(exclude=['deploy*']),
    include_package_data=True,

    entry_points={
        'console_scripts': [
            'pootle = pootle.runner:main',
        ],
    },
    cmdclass={
        'build_mo': PootleBuildMo,
        'test': PyTest,
    },
)

setup.py
diff --git a/.travis.yml b/.travis.yml
index 105e076d692..baea2136dac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,24 +4,71 @@ python:
   - 2.6
   - 2.7
 env:
-  - DJANGO_VERSION=1.4.5
-# When we do finally support Django 1.5 - it requires Python 2.6.5, so exclude
-# earlier versions (see https://docs.djangoproject.com/en/dev/faq/install/).
-#matrix:
-#  exclude:
-#    - python: 2.5
-#      env: DJANGO_VERSION=1.5
+  matrix:
+    - DJANGO_VERSION=1.6.5 DATABASE_BACKEND=sqlite
+    - DJANGO_VERSION=1.6.5 DATABASE_BACKEND=postgres
+    - DJANGO_VERSION=1.6.5 DATABASE_BACKEND=mysql
+    - DJANGO_VERSION=1.7.1 DATABASE_BACKEND=sqlite
+    - DJANGO_VERSION=1.7.1 DATABASE_BACKEND=postgres
+    - DJANGO_VERSION=1.7.1 DATABASE_BACKEND=mysql
+matrix:
+  exclude:
+    - python: 2.6
+      env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=sqlite
+    - python: 2.6
+      env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=mysql
+    - python: 2.6
+      env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=postgres
+  allow_failures:
+    # Allow failure on Django 1.7.1 until we've migrated away from South to
+    # Django migration
+    - env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=sqlite
+    - env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=mysql
+    - env: DJANGO_VERSION=1.7.1 DATABASE_BACKEND=postgres
+    # Disable alternate databases until we break away from being stuck on
+    # MyISAM
+    - env: DJANGO_VERSION=1.6.5 DATABASE_BACKEND=sqlite
+    - env: DJANGO_VERSION=1.6.5 DATABASE_BACKEND=postgres
+cache:
+  - apt
+before_install:
+  - travis_retry sudo apt-get install -qq bc
 install:
-  - pip install --use-mirrors Django==$DJANGO_VERSION --timeout=240
-  - pip install -r requirements/build.txt --use-mirrors --timeout=240
-  - python -V; pip freeze  # Print all installed versions for reference.
+  - travis_retry pip install -r requirements/travis.txt --timeout=240
+  - travis_retry pip install "Django>=$DJANGO_VERSION,<"$(echo $(echo $DJANGO_VERSION | cut -d"." -f1,2) + 0.1 | bc) --timeout=240
+  - pip freeze  # Print all installed versions for reference.
+before_script:
+  - if [[ $DATABASE_BACKEND == 'postgres' ]]; then psql -c 'create database pootle;' -U postgres; fi
+  - if [[ $DATABASE_BACKEND == 'mysql' ]]; then mysql -e 'create database pootle;'; fi
 script:
-  - make build SHELL=/bin/bash TAIL='|tail -40; exit $$PIPESTATUS'
-  - ./manage.py runserver &
-  - TESTPID=$!
-  - sleep 100
-  - kill -2 $TESTPID
+  - python -m compileall -q -f .
+  # FIXME revert breakdown of 'make build' after core merge
+  #- make docs
+  #- make build SHELL=/bin/bash TAIL='|tail -40; exit $$PIPESTATUS'
+  # FIXME Don't raise Warning as Errors till docs have landed and are cleaned
+  # up See https://github.com/translate/pootle/issues/3307
+  - (cd docs; make SPHINXOPTS="-q" html)
+  - make assets
+  - make mo
+  - coverage run --parallel-mode manage.py syncdb --noinput --traceback
+  # FIXME disable the migrate, initdb, runserver..kill block until merge or
+  # #3378 and #3379 are fixed
+  #- coverage run --parallel-mode manage.py migrate --traceback
+  #- coverage run --parallel-mode manage.py initdb --traceback
+  #- coverage run --parallel-mode ./manage.py runserver --traceback &
+  #- TESTPID=$!
+  #- sleep 20
+  #- kill -2 $TESTPID
+  # FIXME reenable migration testing
+  #- tests/version-migration/migrate.sh
+  - make pep8
+  - py.test
+after_success:
+  - coverage combine
+  - coveralls
 notifications:
   email:
     on_failure: always
     on_success: change
+services:
+  - redis-server
diff --git a/Makefile b/Makefile
index d37a4218fd2..3f959c9ecb0 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ SFUSERNAME=$(shell egrep -A5 sourceforge ~/.ssh/config | egrep -m1 User | cut -d
 FORMATS=--formats=bztar
 TEST_ENV_NAME = pootle_test_env
 
-.PHONY: all build clean sprite test pot mo mo-all help docs assets
+.PHONY: all build clean sprite test pot mo mo-all help docs assets pep8
 
 all: help
 
@@ -60,6 +60,9 @@ mo:
 mo-all:
 	python setup.py build_mo --all
 
+pep8:
+	@./pootle/tools/pep8.sh travis
+
 publish-pypi:
 	python setup.py sdist ${FORMATS} upload
 
@@ -85,6 +88,7 @@ help:
 	@echo "  sprite - create CSS sprite"
 	@echo "  clean - remove any temporal files"
 	@echo "  test - run test suite"
+	@echo "  pep8 - run pep8 checks"
 	@echo "  pot - update the POT translations templates"
 	@echo "  mo - build MO files for languages listed in 'pootle/locale/LINGUAS'"
 	@echo "  mo-all - build MO files for all languages (only use for testing)"
diff --git a/pootle/log/README b/pootle/log/README
new file mode 100644
index 00000000000..ddd242e88ba
--- /dev/null
+++ b/pootle/log/README
@@ -0,0 +1 @@
+This directory (logs) contains the Pootle activity logs.
diff --git a/pootle/settings/91-travis.conf b/pootle/settings/91-travis.conf
new file mode 100644
index 00000000000..f8f265b6187
--- /dev/null
+++ b/pootle/settings/91-travis.conf
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+"""
+Pootle configuration for Travis
+https://travis-ci.org
+"""
+
+import os
+
+
+if os.environ.get("TRAVIS"):
+    DEBUG = True
+    TEMPLATE_DEBUG = DEBUG
+
+    DATABASE_BACKEND = os.environ.get("DATABASE_BACKEND")
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': working_path('dbs/pootle_travis.db'),
+            'USER': '',
+            'PASSWORD': '',
+            'HOST': '',
+            'PORT': '',
+            'ATOMIC_REQUESTS': True,
+        }
+    }
+
+    CACHES = {
+        'default': {
+            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+        }
+    }
+
+
+    if DATABASE_BACKEND == "postgres":
+        DATABASES['default']['ENGINE'] = 'django.db.backends.postgresql_psycopg2'
+        DATABASES['default']['NAME'] = 'pootle'
+        DATABASES['default']['USER'] = 'postgres'
+    elif DATABASE_BACKEND == "mysql":
+        DATABASES['default']['ENGINE'] = 'django.db.backends.mysql'
+        DATABASES['default']['NAME'] = 'pootle'
+        DATABASES['default']['USER'] = 'travis'
+        # Remove this once we've closed #3363 and #3364
+        DATABASES['default']['OPTIONS'] = {
+            'init_command': 'SET storage_engine=MyISAM',
+        }
diff --git a/requirements/base.txt b/requirements/base.txt
index da54c9071ec..73d2f477d0b 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -9,12 +9,19 @@ django-redis-cache
 django-rq
 
 # Required by webassets, isn't shipped in Python < 2.7
+# FIXME remove once Django >= 1.7 is required
 argparse
+# Needed for Python < 2.7
+importlib
 
 # Libraries
 cssmin
 diff-match-patch>=20121119
 lxml>=2.2.0
+# FIXME drop once https://github.com/translate/pootle/issues/3343 and Evernote
+# allauth backend have landed.
+--allow-external pyDes
+--allow-unverified pyDes
 pyDes
 
 # Translate Toolkit
diff --git a/requirements/tests.txt b/requirements/tests.txt
index 96b586ece1c..6d1ade01559 100644
--- a/requirements/tests.txt
+++ b/requirements/tests.txt
@@ -1,5 +1,3 @@
--r deploy.txt
-
 factory_boy
 pytest
-pytest-django>=2.6.1
+pytest-django>=2.6.1,<2.7
diff --git a/requirements/travis.txt b/requirements/travis.txt
new file mode 100644
index 00000000000..a9e60ff03cb
--- /dev/null
+++ b/requirements/travis.txt
@@ -0,0 +1,7 @@
+-r build.txt
+-r tests.txt
+
+coveralls
+MySQL-python
+psycopg2
+pep8
diff --git a/setup.py b/setup.py
index 6f5b2df060d..a90be31ee96 100755
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@ def parse_requirements(file_name):
     requirements = []
     for line in open(file_name, 'r').read().split('\n'):
         # Ignore comments, blank lines and included requirements files
-        if re.match(r'(\s*#)|(\s*$)|(-r .*$)', line):
+        if re.match(r'(\s*#)|(\s*$)|((-r|--allow-external|--allow-unverified) .*$)', line):
             continue
 
         if re.match(r'\s*-e\s+', line):

Core: drop MySQL dependence on MyISAM
Core depends on MyISAM at the moment because of low level features used for changeid tracking.  We need to migrate that to a more general approach that works on InnoDB and other supported DB engines.
- [x] Make resources list work in all DB backends (#3539)
- [x] Switch revision counter to Redis (#3364)
- [x] Ensure tests run on InnoDB (#3777)

