import operator
import os
import io
import six

from datetime import datetime
from flask import (Blueprint, render_template, flash, redirect, url_for, g,
                   session, current_app, request, Markup, abort)
from flask_babel import gettext
from sqlalchemy.exc import IntegrityError

import store

from db import db
from models import Source, Submission, Reply, get_one_or_else
from source_app.decorators import login_required
from source_app.utils import (logged_in, generate_unique_codename,
                              async_genkey, normalize_timestamps,
                              valid_codename, get_entropy_estimate)
from source_app.forms import LoginForm


def make_blueprint(config):
    view = Blueprint('main', __name__)

    @view.route('/')
    def index():
        return render_template('index.html')

    @view.route('/generate', methods=('GET', 'POST'))
    def generate():
        if logged_in():
            flash(gettext(
                "You were redirected because you are already logged in. "
                "If you want to create a new account, you should log out "
                "first."),
                  "notification")
            return redirect(url_for('.lookup'))

        codename = generate_unique_codename(config)
        session['codename'] = codename
        session['new_user'] = True
        return render_template('generate.html', codename=codename)

    @view.route('/org-logo')
    def select_logo():
        if os.path.exists(os.path.join(current_app.static_folder, 'i',
                          'custom_logo.png')):
            return redirect(url_for('static', filename='i/custom_logo.png'))
        else:
            return redirect(url_for('static', filename='i/logo.png'))

    @view.route('/create', methods=['POST'])
    def create():
        filesystem_id = current_app.crypto_util.hash_codename(
            session['codename'])

        source = Source(filesystem_id, current_app.crypto_util.display_id())
        db.session.add(source)
        try:
            db.session.commit()
        except IntegrityError as e:
            db.session.rollback()
            current_app.logger.error(
                "Attempt to create a source with duplicate codename: %s" %
                (e,))

            # Issue 2386: don't log in on duplicates
            del session['codename']
            abort(500)
        else:
            os.mkdir(current_app.storage.path(filesystem_id))

        session['logged_in'] = True
        return redirect(url_for('.lookup'))

    @view.route('/lookup', methods=('GET',))
    @login_required
    def lookup():
        replies = []
        source_inbox = Reply.query.filter(Reply.source_id == g.source.id) \
                                  .filter(Reply.deleted_by_source == False).all()  # noqa

        for reply in source_inbox:
            reply_path = current_app.storage.path(
                g.filesystem_id,
                reply.filename,
            )
            try:
                with io.open(reply_path, "rb") as f:
                    contents = f.read()
                reply_obj = current_app.crypto_util.decrypt(g.codename, contents)
                if six.PY2:  # Python2
                    reply.decrypted = reply_obj.decode('utf-8')
                else:
                    reply.decrypted = reply_obj
            except UnicodeDecodeError:
                current_app.logger.error("Could not decode reply %s" %
                                         reply.filename)
            else:
                reply.date = datetime.utcfromtimestamp(
                    os.stat(reply_path).st_mtime)
                replies.append(reply)

        # Sort the replies by date
        replies.sort(key=operator.attrgetter('date'), reverse=True)

        # Generate a keypair to encrypt replies from the journalist
        # Only do this if the journalist has flagged the source as one
        # that they would like to reply to. (Issue #140.)
        if not current_app.crypto_util.getkey(g.filesystem_id) and \
                g.source.flagged:
            db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']
            async_genkey(current_app.crypto_util,
                         db_uri,
                         g.filesystem_id,
                         g.codename)

        return render_template(
            'lookup.html',
            codename=g.codename,
            replies=replies,
            flagged=g.source.flagged,
            new_user=session.get('new_user', None),
            haskey=current_app.crypto_util.getkey(
                g.filesystem_id))

    @view.route('/submit', methods=('POST',))
    @login_required
    def submit():
        msg = request.form['msg']
        fh = None
        if 'fh' in request.files:
            fh = request.files['fh']

        # Don't submit anything if it was an "empty" submission. #878
        if not (msg or fh):
            flash(gettext(
                "You must enter a message or choose a file to submit."),
                  "error")
            return redirect(url_for('main.lookup'))

        fnames = []
        journalist_filename = g.source.journalist_filename
        first_submission = g.source.interaction_count == 0

        if msg:
            g.source.interaction_count += 1
            fnames.append(
                current_app.storage.save_message_submission(
                    g.filesystem_id,
                    g.source.interaction_count,
                    journalist_filename,
                    msg))
        if fh:
            g.source.interaction_count += 1
            fnames.append(
                current_app.storage.save_file_submission(
                    g.filesystem_id,
                    g.source.interaction_count,
                    journalist_filename,
                    fh.filename,
                    fh.stream))

        if first_submission:
            msg = render_template('first_submission_flashed_message.html')
            flash(Markup(msg), "success")

        else:
            if msg and not fh:
                html_contents = gettext('Thanks! We received your message.')
            elif not msg and fh:
                html_contents = gettext('Thanks! We received your document.')
            else:
                html_contents = gettext('Thanks! We received your message and '
                                        'document.')

            msg = render_template('next_submission_flashed_message.html',
                                  html_contents=html_contents)
            flash(Markup(msg), "success")

        new_submissions = []
        for fname in fnames:
            submission = Submission(g.source, fname)
            db.session.add(submission)
            new_submissions.append(submission)

        if g.source.pending:
            g.source.pending = False

            # Generate a keypair now, if there's enough entropy (issue #303)
            # (gpg reads 300 bytes from /dev/random)
            entropy_avail = get_entropy_estimate()
            if entropy_avail >= 2400:
                db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']

                async_genkey(current_app.crypto_util,
                             db_uri,
                             g.filesystem_id,
                             g.codename)
                current_app.logger.info("generating key, entropy: {}".format(
                    entropy_avail))
            else:
                current_app.logger.warn(
                        "skipping key generation. entropy: {}".format(
                                entropy_avail))

        g.source.last_updated = datetime.utcnow()
        db.session.commit()

        for sub in new_submissions:
            store.async_add_checksum_for_file(sub)

        normalize_timestamps(g.filesystem_id)

        return redirect(url_for('main.lookup'))

    @view.route('/delete', methods=('POST',))
    @login_required
    def delete():
        """This deletes the reply from the source's inbox, but preserves
        the history for journalists such that they can view conversation
        history.
        """

        query = Reply.query.filter_by(
            filename=request.form['reply_filename'],
            source_id=g.source.id)
        reply = get_one_or_else(query, current_app.logger, abort)
        reply.deleted_by_source = True
        db.session.add(reply)
        db.session.commit()

        flash(gettext("Reply deleted"), "notification")
        return redirect(url_for('.lookup'))

    @view.route('/delete-all', methods=('POST',))
    @login_required
    def batch_delete():
        replies = Reply.query.filter(Reply.source_id == g.source.id) \
                             .filter(Reply.deleted_by_source == False).all()  # noqa
        if len(replies) == 0:
            current_app.logger.error("Found no replies when at least one was "
                                     "expected")
            return redirect(url_for('.lookup'))

        for reply in replies:
            reply.deleted_by_source = True
            db.session.add(reply)
        db.session.commit()

        flash(gettext("All replies have been deleted"), "notification")
        return redirect(url_for('.lookup'))

    @view.route('/login', methods=('GET', 'POST'))
    def login():
        form = LoginForm()
        if form.validate_on_submit():
            codename = request.form['codename'].strip()
            if valid_codename(codename):
                session.update(codename=codename, logged_in=True)
                return redirect(url_for('.lookup', from_login='1'))
            else:
                current_app.logger.info(
                        "Login failed for invalid codename")
                flash(gettext("Sorry, that is not a recognized codename."),
                      "error")
        return render_template('login.html', form=form)

    @view.route('/logout')
    def logout():
        if logged_in():
            msg = render_template('logout_flashed_message.html')

            # Clear the session after we render the message so it's localized
            # If a user specified a locale, save it and restore it
            user_locale = g.locale
            session.clear()
            session['locale'] = user_locale

            flash(Markup(msg), "important hide-if-not-tor-browser")
        return redirect(url_for('.index'))

    return view

securedrop/source_app/main.py
import operator
import os
import io
import six

from datetime import datetime
from flask import (Blueprint, render_template, flash, redirect, url_for, g,
                   session, current_app, request, Markup, abort)
from flask_babel import gettext
from sqlalchemy.exc import IntegrityError

import store

from db import db
from models import Source, Submission, Reply, get_one_or_else
from source_app.decorators import login_required
from source_app.utils import (logged_in, generate_unique_codename,
                              async_genkey, normalize_timestamps,
                              valid_codename, get_entropy_estimate)
from source_app.forms import LoginForm


def make_blueprint(config):
    view = Blueprint('main', __name__)

    @view.route('/')
    def index():
        return render_template('index.html')

    @view.route('/generate', methods=('GET', 'POST'))
    def generate():
        if logged_in():
            flash(gettext(
                "You were redirected because you are already logged in. "
                "If you want to create a new account, you should log out "
                "first."),
                  "notification")
            return redirect(url_for('.lookup'))

        codename = generate_unique_codename(config)
        session['codename'] = codename
        session['new_user'] = True
        return render_template('generate.html', codename=codename)

    @view.route('/org-logo')
    def select_logo():
        if os.path.exists(os.path.join(current_app.static_folder, 'i',
                          'custom_logo.png')):
            return redirect(url_for('static', filename='i/custom_logo.png'))
        else:
            return redirect(url_for('static', filename='i/logo.png'))

    @view.route('/create', methods=['POST'])
    def create():
        filesystem_id = current_app.crypto_util.hash_codename(
            session['codename'])

        source = Source(filesystem_id, current_app.crypto_util.display_id())
        db.session.add(source)
        try:
            db.session.commit()
        except IntegrityError as e:
            db.session.rollback()
            current_app.logger.error(
                "Attempt to create a source with duplicate codename: %s" %
                (e,))

            # Issue 2386: don't log in on duplicates
            del session['codename']

            # Issue 4361: Delete 'logged_in' if it's in the session
            try:
                del session['logged_in']
            except KeyError:
                pass

            abort(500)
        else:
            os.mkdir(current_app.storage.path(filesystem_id))

        session['logged_in'] = True
        return redirect(url_for('.lookup'))

    @view.route('/lookup', methods=('GET',))
    @login_required
    def lookup():
        replies = []
        source_inbox = Reply.query.filter(Reply.source_id == g.source.id) \
                                  .filter(Reply.deleted_by_source == False).all()  # noqa

        for reply in source_inbox:
            reply_path = current_app.storage.path(
                g.filesystem_id,
                reply.filename,
            )
            try:
                with io.open(reply_path, "rb") as f:
                    contents = f.read()
                reply_obj = current_app.crypto_util.decrypt(g.codename, contents)
                if six.PY2:  # Python2
                    reply.decrypted = reply_obj.decode('utf-8')
                else:
                    reply.decrypted = reply_obj
            except UnicodeDecodeError:
                current_app.logger.error("Could not decode reply %s" %
                                         reply.filename)
            else:
                reply.date = datetime.utcfromtimestamp(
                    os.stat(reply_path).st_mtime)
                replies.append(reply)

        # Sort the replies by date
        replies.sort(key=operator.attrgetter('date'), reverse=True)

        # Generate a keypair to encrypt replies from the journalist
        # Only do this if the journalist has flagged the source as one
        # that they would like to reply to. (Issue #140.)
        if not current_app.crypto_util.getkey(g.filesystem_id) and \
                g.source.flagged:
            db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']
            async_genkey(current_app.crypto_util,
                         db_uri,
                         g.filesystem_id,
                         g.codename)

        return render_template(
            'lookup.html',
            codename=g.codename,
            replies=replies,
            flagged=g.source.flagged,
            new_user=session.get('new_user', None),
            haskey=current_app.crypto_util.getkey(
                g.filesystem_id))

    @view.route('/submit', methods=('POST',))
    @login_required
    def submit():
        msg = request.form['msg']
        fh = None
        if 'fh' in request.files:
            fh = request.files['fh']

        # Don't submit anything if it was an "empty" submission. #878
        if not (msg or fh):
            flash(gettext(
                "You must enter a message or choose a file to submit."),
                  "error")
            return redirect(url_for('main.lookup'))

        fnames = []
        journalist_filename = g.source.journalist_filename
        first_submission = g.source.interaction_count == 0

        if msg:
            g.source.interaction_count += 1
            fnames.append(
                current_app.storage.save_message_submission(
                    g.filesystem_id,
                    g.source.interaction_count,
                    journalist_filename,
                    msg))
        if fh:
            g.source.interaction_count += 1
            fnames.append(
                current_app.storage.save_file_submission(
                    g.filesystem_id,
                    g.source.interaction_count,
                    journalist_filename,
                    fh.filename,
                    fh.stream))

        if first_submission:
            msg = render_template('first_submission_flashed_message.html')
            flash(Markup(msg), "success")

        else:
            if msg and not fh:
                html_contents = gettext('Thanks! We received your message.')
            elif not msg and fh:
                html_contents = gettext('Thanks! We received your document.')
            else:
                html_contents = gettext('Thanks! We received your message and '
                                        'document.')

            msg = render_template('next_submission_flashed_message.html',
                                  html_contents=html_contents)
            flash(Markup(msg), "success")

        new_submissions = []
        for fname in fnames:
            submission = Submission(g.source, fname)
            db.session.add(submission)
            new_submissions.append(submission)

        if g.source.pending:
            g.source.pending = False

            # Generate a keypair now, if there's enough entropy (issue #303)
            # (gpg reads 300 bytes from /dev/random)
            entropy_avail = get_entropy_estimate()
            if entropy_avail >= 2400:
                db_uri = current_app.config['SQLALCHEMY_DATABASE_URI']

                async_genkey(current_app.crypto_util,
                             db_uri,
                             g.filesystem_id,
                             g.codename)
                current_app.logger.info("generating key, entropy: {}".format(
                    entropy_avail))
            else:
                current_app.logger.warn(
                        "skipping key generation. entropy: {}".format(
                                entropy_avail))

        g.source.last_updated = datetime.utcnow()
        db.session.commit()

        for sub in new_submissions:
            store.async_add_checksum_for_file(sub)

        normalize_timestamps(g.filesystem_id)

        return redirect(url_for('main.lookup'))

    @view.route('/delete', methods=('POST',))
    @login_required
    def delete():
        """This deletes the reply from the source's inbox, but preserves
        the history for journalists such that they can view conversation
        history.
        """

        query = Reply.query.filter_by(
            filename=request.form['reply_filename'],
            source_id=g.source.id)
        reply = get_one_or_else(query, current_app.logger, abort)
        reply.deleted_by_source = True
        db.session.add(reply)
        db.session.commit()

        flash(gettext("Reply deleted"), "notification")
        return redirect(url_for('.lookup'))

    @view.route('/delete-all', methods=('POST',))
    @login_required
    def batch_delete():
        replies = Reply.query.filter(Reply.source_id == g.source.id) \
                             .filter(Reply.deleted_by_source == False).all()  # noqa
        if len(replies) == 0:
            current_app.logger.error("Found no replies when at least one was "
                                     "expected")
            return redirect(url_for('.lookup'))

        for reply in replies:
            reply.deleted_by_source = True
            db.session.add(reply)
        db.session.commit()

        flash(gettext("All replies have been deleted"), "notification")
        return redirect(url_for('.lookup'))

    @view.route('/login', methods=('GET', 'POST'))
    def login():
        form = LoginForm()
        if form.validate_on_submit():
            codename = request.form['codename'].strip()
            if valid_codename(codename):
                session.update(codename=codename, logged_in=True)
                return redirect(url_for('.lookup', from_login='1'))
            else:
                current_app.logger.info(
                        "Login failed for invalid codename")
                flash(gettext("Sorry, that is not a recognized codename."),
                      "error")
        return render_template('login.html', form=form)

    @view.route('/logout')
    def logout():
        if logged_in():
            msg = render_template('logout_flashed_message.html')

            # Clear the session after we render the message so it's localized
            # If a user specified a locale, save it and restore it
            user_locale = g.locale
            session.clear()
            session['locale'] = user_locale

            flash(Markup(msg), "important hide-if-not-tor-browser")
        return redirect(url_for('.index'))

    return view

securedrop/source_app/main.py
diff --git a/securedrop/source_app/main.py b/securedrop/source_app/main.py
index 91481f3157..1ffbaa7ddd 100644
--- a/securedrop/source_app/main.py
+++ b/securedrop/source_app/main.py
@@ -67,6 +67,13 @@ def create():
 
             # Issue 2386: don't log in on duplicates
             del session['codename']
+
+            # Issue 4361: Delete 'logged_in' if it's in the session
+            try:
+                del session['logged_in']
+            except KeyError:
+                pass
+
             abort(500)
         else:
             os.mkdir(current_app.storage.path(filesystem_id))
diff --git a/securedrop/tests/test_source.py b/securedrop/tests/test_source.py
index 4499054642..ea9a04a847 100644
--- a/securedrop/tests/test_source.py
+++ b/securedrop/tests/test_source.py
@@ -146,7 +146,31 @@ def test_generate_too_long_codename(source_app):
     )
 
 
-def test_create_duplicate_codename(source_app):
+def test_create_duplicate_codename_logged_in_not_in_session(source_app):
+    with patch.object(source.app.logger, 'error') as logger:
+        with source_app.test_client() as app:
+            resp = app.get(url_for('main.generate'))
+            assert resp.status_code == 200
+
+            # Create a source the first time
+            resp = app.post(url_for('main.create'), follow_redirects=True)
+            assert resp.status_code == 200
+            codename = session['codename']
+
+        with source_app.test_client() as app:
+            # Attempt to add the same source
+            with app.session_transaction() as sess:
+                sess['codename'] = codename
+            resp = app.post(url_for('main.create'), follow_redirects=True)
+            logger.assert_called_once()
+            assert ("Attempt to create a source with duplicate codename"
+                    in logger.call_args[0][0])
+            assert resp.status_code == 500
+            assert 'codename' not in session
+            assert 'logged_in' not in session
+
+
+def test_create_duplicate_codename_logged_in_in_session(source_app):
     with patch.object(source.app.logger, 'error') as logger:
         with source_app.test_client() as app:
             resp = app.get(url_for('main.generate'))
@@ -157,12 +181,17 @@ def test_create_duplicate_codename(source_app):
             assert resp.status_code == 200
 
             # Attempt to add the same source
-            app.post(url_for('main.create'), follow_redirects=True)
+            resp = app.post(url_for('main.create'), follow_redirects=True)
             logger.assert_called_once()
             assert ("Attempt to create a source with duplicate codename"
                     in logger.call_args[0][0])
+            assert resp.status_code == 500
             assert 'codename' not in session
 
+            # Reproducer for bug #4361
+            resp = app.post(url_for('main.index'), follow_redirects=True)
+            assert 'logged_in' not in session
+
 
 def test_lookup(source_app):
     """Test various elements on the /lookup page."""

Source Interface requests fail with 500 error, due to session issue
## Description

In some situations, requests to the source interface may start to fail, returning the 500 error page. Once the 500 errors start, they continue until the Tor Browser cache is cleared, either explicitly or by starting a new browser session. With source error logging enabled, the following errors are seen on failing requests:

```
[Thu Apr 18 09:46:09.516056 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169] [2019-04-18 09:46:09,510] ERROR in app: Exception on / [GET]
[Thu Apr 18 09:46:09.516238 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169] Traceback (most recent call last):
[Thu Apr 18 09:46:09.516279 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 2292, in wsgi_app
[Thu Apr 18 09:46:09.516317 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     response = self.full_dispatch_request()
[Thu Apr 18 09:46:09.516363 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1815, in full_dispatch_request
[Thu Apr 18 09:46:09.516442 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     rv = self.handle_user_exception(e)
[Thu Apr 18 09:46:09.516479 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1718, in handle_user_exception
[Thu Apr 18 09:46:09.516514 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     reraise(exc_type, exc_value, tb)
[Thu Apr 18 09:46:09.516549 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1811, in full_dispatch_request
[Thu Apr 18 09:46:09.516584 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     rv = self.preprocess_request()
[Thu Apr 18 09:46:09.516619 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 2087, in preprocess_request
[Thu Apr 18 09:46:09.516654 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     rv = func()
[Thu Apr 18 09:46:09.516688 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/var/www/securedrop/source_app/decorators.py", line 23, in decorated_function
[Thu Apr 18 09:46:09.516724 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     return f(*args, **kwargs)
[Thu Apr 18 09:46:09.516758 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/var/www/securedrop/source_app/__init__.py", line 159, in setup_g
[Thu Apr 18 09:46:09.516793 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     g.codename = session['codename']
[Thu Apr 18 09:46:09.516828 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 377, in <lambda>
[Thu Apr 18 09:46:09.516864 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     __getitem__ = lambda x, i: x._get_current_object()[i]
[Thu Apr 18 09:46:09.516899 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]   File "/usr/local/lib/python2.7/dist-packages/flask/sessions.py", line 83, in __getitem__
[Thu Apr 18 09:46:09.516933 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169]     return super(SecureCookieSession, self).__getitem__(key)
[Thu Apr 18 09:46:09.516968 2019] [wsgi:error] [pid 7324:tid 3457186817792] [remote 127.0.0.1:29169] KeyError: 'codename'
```

## Steps to Reproduce
This error was initially hit by:
1) starting a source interface session on a 0.12.1 Xenial install
2) updating the 0.12.1 Xenial install to 0.12.2~rc1 via cron-apt
3) running a 0.12.1 db restore against the 0.12.2 database
4) attempting to continue the source session.

It's also been reproduced during a test session by creating multiple sources and logging in and out repeatedly (h/t @eloquence ), but is not reliably reproducible.

## Expected Behavior
Source Interface requests for valid URLs return the correct results.

## Actual Behavior
SI requests all return 500 errors.

## Comments


