import subprocess as sp
import signal
import threading
import os

SIGTERM_TIMEOUT = 1.0

class Command(object):
    def __init__(self, cmd, memlimit=None):
        self.cmd = cmd
        self.memlimit = memlimit
        self.process = None
        self.stdout = None
        self.stderr = None
        self.exitcode = None
        self.timed_out = False

    def run(self, timeout=None):
        def target():
            self.process = sp.Popen(self.cmd,
                                    stdout=sp.PIPE, stderr=sp.PIPE,
                                    shell=True, preexec_fn=os.setpgrp)
            self.stdout, self.stderr = self.process.communicate()
            self.exitcode = self.process.returncode

        thread = threading.Thread(target=target)
        thread.start()
        thread.join(float(timeout))
        if thread.is_alive():
            self.timed_out = True

            # Send the TERM signal to all the process groups
            os.killpg(self.process.pid, signal.SIGTERM)
            thread.join(SIGTERM_TIMEOUT)
            if thread.is_alive():
                # Send the KILL signal if the process hasn't exited by now.
                os.killpg(self.process.pid, signal.SIGKILL)
                self.process.kill()
                thread.join()

import subprocess as sp
import signal
import threading
import os

SIGTERM_TIMEOUT = 1.0

class Command(object):
    def __init__(self, cmd, memlimit=None):
        self.cmd = cmd
        self.memlimit = memlimit
        self.process = None
        self.stdout = None
        self.stderr = None
        self.exitcode = None
        self.timed_out = False

    def run(self, timeout=None):
        def target():
            cmd = self.cmd
            if self.memlimit:
                cmd = "ulimit -v %d; %s" % (self.memlimit, cmd)
            self.process = sp.Popen(cmd,
                                    stdout=sp.PIPE, stderr=sp.PIPE,
                                    shell=True, preexec_fn=os.setpgrp)
            self.stdout, self.stderr = self.process.communicate()
            self.exitcode = self.process.returncode

        thread = threading.Thread(target=target)
        thread.start()
        thread.join(float(timeout))
        if thread.is_alive():
            self.timed_out = True

            # Send the TERM signal to all the process groups
            os.killpg(self.process.pid, signal.SIGTERM)
            thread.join(SIGTERM_TIMEOUT)
            if thread.is_alive():
                # Send the KILL signal if the process hasn't exited by now.
                os.killpg(self.process.pid, signal.SIGKILL)
                self.process.kill()
                thread.join()

Enable Command to support memory limiting.


import pkg_resources

from sqlalchemy import MetaData, Table, create_engine, orm

from .tables import metadata

def connect(uri=None, **kwargs):
    """Connects to the requested URI.  Returns a session object.

    With the URI omitted, attempts to connect to a default SQLite database
    contained within the package directory.

    Calling this function also binds the metadata object to the created engine.
    """

    # Default to a URI within the package, which was hopefully created at some point
    if not uri:
        sqlite_path = pkg_resources.resource_filename('pokedex',
                                                      'data/pokedex.sqlite')
        uri = 'sqlite:///' + sqlite_path

    ### Do some fixery for MySQL
    if uri[0:5] == 'mysql':
        # MySQL uses latin1 for connections by default even if the server is
        # otherwise oozing with utf8; charset fixes this
        if 'charset' not in uri:
            uri += '?charset=utf8'

        # Tables should be InnoDB, in the event that we're creating them, and
        # use UTF-8 goddammit!
        for table in metadata.tables.values():
            table.kwargs['mysql_engine'] = 'InnoDB'
            table.kwargs['mysql_charset'] = 'utf8'

    ### Connect
    engine = create_engine(uri)
    conn = engine.connect()
    metadata.bind = engine

    session_args = dict(autoflush=True, autocommit=False, bind=engine)
    session_args.update(kwargs)
    sm = orm.sessionmaker(**session_args)
    session = orm.scoped_session(sm)

    return session

import pkg_resources

from sqlalchemy import MetaData, Table, create_engine, orm

from .tables import metadata

def connect(uri=None, session_args={}, engine_args={}):
    """Connects to the requested URI.  Returns a session object.

    With the URI omitted, attempts to connect to a default SQLite database
    contained within the package directory.

    Calling this function also binds the metadata object to the created engine.
    """

    # Default to a URI within the package, which was hopefully created at some point
    if not uri:
        sqlite_path = pkg_resources.resource_filename('pokedex',
                                                      'data/pokedex.sqlite')
        uri = 'sqlite:///' + sqlite_path

    ### Do some fixery for MySQL
    if uri[0:5] == 'mysql':
        # MySQL uses latin1 for connections by default even if the server is
        # otherwise oozing with utf8; charset fixes this
        if 'charset' not in uri:
            uri += '?charset=utf8'

        # Tables should be InnoDB, in the event that we're creating them, and
        # use UTF-8 goddammit!
        for table in metadata.tables.values():
            table.kwargs['mysql_engine'] = 'InnoDB'
            table.kwargs['mysql_charset'] = 'utf8'

    ### Connect
    engine = create_engine(uri, **engine_args)
    conn = engine.connect()
    metadata.bind = engine

    all_session_args = dict(autoflush=True, autocommit=False, bind=engine)
    all_session_args.update(session_args)
    sm = orm.sessionmaker(**all_session_args)
    session = orm.scoped_session(sm)

    return session

Allow passing engine arguments to connect().


# -*- coding: utf-8 -*-
"""
Crossfolium
-----------

"""

import crossfolium.marker_function as marker_function

from crossfolium.crossfolium import (
    Crossfilter,
    PieFilter,
    RowBarFilter,
    BarFilter,
    TableFilter,
    CountFilter,
    ResetFilter,
    GeoChoroplethFilter,
    )

from .map import (
    FeatureGroupFilter,
    HeatmapFilter,
    )

__version__ = "0.0.0"

__all__ = [
    '__version__',
    'marker_function',
    'Crossfilter',
    'PieFilter',
    'RowBarFilter',
    'BarFilter',
    'FeatureGroupFilter',
    'TableFilter',
    'CountFilter',
    'ResetFilter',
    'HeatmapFilter',
    'GeoChoroplethFilter',
    ]

# -*- coding: utf-8 -*-
"""
Crossfolium
-----------

"""
from __future__ import absolute_import

from crossfolium import marker_function

from crossfolium.crossfolium import (
    Crossfilter,
    PieFilter,
    RowBarFilter,
    BarFilter,
    TableFilter,
    CountFilter,
    ResetFilter,
    GeoChoroplethFilter,
    )

from crossfolium.map import (
    FeatureGroupFilter,
    HeatmapFilter,
    )

__version__ = "0.0.0"

__all__ = [
    '__version__',
    'marker_function',
    'Crossfilter',
    'PieFilter',
    'RowBarFilter',
    'BarFilter',
    'FeatureGroupFilter',
    'TableFilter',
    'CountFilter',
    'ResetFilter',
    'HeatmapFilter',
    'GeoChoroplethFilter',
    ]

Handle absolute import for py27

Handle absolute import for py27


import sqlite3
import os
import pandas as pd

TABLES = [['Natures', 'nature'],
          ['Experience'],
         ]

PATH = os.path.dirname(__file__)+"/"
CONNECTION = sqlite3.connect(PATH + 'serpyrior.db')

# insert a little jimmy drop tables here

for table in TABLES:
    table_name = table[0]
    print(table_name)
    try:
        table_index = table[1]
        write_index = False
    except IndexError:
        table_index = None
        write_index = True

    df = pd.read_csv(PATH + table_name + '.csv')
    df.to_sql(table_name, CONNECTION, index=write_index, index_label=table_index)

CONNECTION.commit()
CONNECTION.close()

# cur = conn.cursor()
# cur.execute("CREATE TABLE IF NOT EXISTS natures()")
# filename.encode('utf-8')
# with open(filename) as f:
#     reader = csv.reader(f)
#     for field in reader:
#         cur.execute("INSERT INTO natures VALUES (?,?,?,?,?,?,?);", field)
#
# conn.commit()
#
# df = pd.read_sql_query("SELECT * FROM natures", conn, index_col='nature')
#
# print(df.head(25))
# conn.close()

import sqlite3
import os
import pandas as pd

TABLES = [['Natures', 'nature'],
          ['Experience'],
         ]

PATH = os.path.dirname(__file__)+"/"

try: # Little Bobby Tables
    os.remove(PATH + 'serpyrior.db')
except FileNotFoundError:
    pass

CONNECTION = sqlite3.connect(PATH + 'serpyrior.db')

for table in TABLES:
    table_name = table[0]
    print(table_name)
    try:
        table_index = table[1]
        write_index = False
    except IndexError:
        table_index = None
        write_index = True

    df = pd.read_csv(PATH + table_name + '.csv')
    df.to_sql(table_name, CONNECTION, index=write_index, index_label=table_index)

CONNECTION.commit()
CONNECTION.close()

Remove db if it already exists

Remove db if it already exists


# coding: utf-8
from .base import *

LOGGING['handlers']['console'] = {
    'level': 'DEBUG',
    'class': 'logging.StreamHandler',
    'formatter': 'verbose'
}

INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar',)
MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')

# Comment out the line below to use `Django Debug Toolbar`
# INTERNAL_IPS = ['172.28.0.4']  # Change IP to KPI container's IP

ENV = 'dev'

# Expiration time in sec. after which paired data xml file must be regenerated
# Does not need to match KoBoCAT setting
PAIRED_DATA_EXPIRATION = 5

# Minimum size (in bytes) of files to allow fast calculation of hashes
# Should match KoBoCAT setting
HASH_BIG_FILE_SIZE_THRESHOLD = 200 * 1024  # 200 kB

# Chunk size in bytes to read per iteration when hash of a file is calculated
# Should match KoBoCAT setting
HASH_BIG_FILE_CHUNK = 5 * 1024  # 5 kB

# coding: utf-8
from .base import *

LOGGING['handlers']['console'] = {
    'level': 'DEBUG',
    'class': 'logging.StreamHandler',
    'formatter': 'verbose'
}

INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar',)
MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')

def show_toolbar(request):
    return env.bool("DEBUG_TOOLBAR", False)
DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": show_toolbar}

ENV = 'dev'

# Expiration time in sec. after which paired data xml file must be regenerated
# Does not need to match KoBoCAT setting
PAIRED_DATA_EXPIRATION = 5

# Minimum size (in bytes) of files to allow fast calculation of hashes
# Should match KoBoCAT setting
HASH_BIG_FILE_SIZE_THRESHOLD = 200 * 1024  # 200 kB

# Chunk size in bytes to read per iteration when hash of a file is calculated
# Should match KoBoCAT setting
HASH_BIG_FILE_CHUNK = 5 * 1024  # 5 kB

Enable django debug toolbar via env var

Enable django debug toolbar via env var


import json

import requests

from flask import request, jsonify

from dadd.worker import app
from dadd.worker.proc import ChildProcess

@app.route('/run/', methods=['POST'])
def run_process():
    proc = ChildProcess(request.json)
    proc.run()
    return jsonify(proc.info())

@app.route('/register/', methods=['POST'])
def register_with_master():
    register(app)
    return jsonify({'message': 'ok'})

def register(host, port):
    sess = requests.Session()

    if 'USERNAME' in app.config and 'PASSWORD' in app.config:
        sess.auth = (app.config['USERNAME'], app.config['PASSWORD'])
    sess.headers = {'content-type': 'application/json'}

    try:
        url = app.config['MASTER_URL'] + '/api/hosts/'
        resp = sess.post(url, data=json.dumps({
            'host': host, 'port': port
        }))
        if not resp.ok:
            app.logger.warning('Error registering with master: %s' %
                               app.config['MASTER_URL'])
    except Exception as e:
        app.logger.warning('Connection Error: %s' % e)

import json
import socket

import requests

from flask import request, jsonify

from dadd.worker import app
from dadd.worker.proc import ChildProcess

@app.route('/run/', methods=['POST'])
def run_process():
    proc = ChildProcess(request.json)
    proc.run()
    return jsonify(proc.info())

@app.route('/register/', methods=['POST'])
def register_with_master():
    register(app)
    return jsonify({'message': 'ok'})

def register(host, port):
    sess = requests.Session()

    if 'USERNAME' in app.config and 'PASSWORD' in app.config:
        sess.auth = (app.config['USERNAME'], app.config['PASSWORD'])
    sess.headers = {'content-type': 'application/json'}

    try:
        url = app.config['MASTER_URL'] + '/api/hosts/'
        resp = sess.post(url, data=json.dumps({
            'host': socket.getfqdn(), 'port': port
        }))
        if not resp.ok:
            app.logger.warning('Error registering with master: %s' %
                               app.config['MASTER_URL'])
    except Exception as e:
        app.logger.warning('Connection Error: %s' % e)

Use the fqdn when registering with the master.

Not all deployment systems will provide a specific hostname via an env
var so we'll avoid relying on it by asking the machine.


#!/usr/bin/env python

'''Implementation of a simple hash table.
The table has `hash`, `get` and `set` methods.
The hash function uses a very basic hash algorithm to insert the value
into the table.
'''

class HashItem(object):
    def __init__(self):
        pass

class Hash(object):
    def __init__(self, size=1024):
        self.table = []
        for i in range(size):
            self.table.append(list())

    def hash(self):
        pass

    def get(self):
        pass

    def set(self):
        pass

#!/usr/bin/env python

'''Implementation of a simple hash table.
The table has `hash`, `get` and `set` methods.
The hash function uses a very basic hash algorithm to insert the value
into the table.
'''

class HashItem(object):
    def __init__(self):
        pass

class Hash(object):
    def __init__(self, size=1024):
        self.table = []
        for i in range(size):
            self.table.append(list())

    def hash(self, key):
        hash_value = 0
        for i in key:
            hash_value += ord(key)
        return hash_value % len(self.table)

    def get(self):
        pass

    def set(self):
        pass

Build out hash function of hash table class


# coding: utf-8

from __future__ import unicode_literals
from django.contrib.auth.models import AbstractUser, UserManager
from django.db.models import BooleanField
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ungettext_lazy
from mptt.fields import TreeForeignKey
from mptt.managers import TreeManager
from mptt.models import MPTTModel
from cache_tools import cached_ugettext_lazy as _

class HierarchicUserManager(TreeManager, UserManager):
    pass

@python_2_unicode_compatible
class HierarchicUser(MPTTModel, AbstractUser):
    mentor = TreeForeignKey(
        'self', null=True, blank=True, related_name='disciples',
        verbose_name=_('mentor'),
        limit_choices_to={'willing_to_be_mentor__exact': True})
    willing_to_be_mentor = BooleanField(
        _('Veut être mentor'), default=False)

    objects = HierarchicUserManager()

    class MPTTMeta(object):
        parent_attr = 'mentor'
        order_insertion_by = ('username',)

    class Meta(object):
        verbose_name = ungettext_lazy('utilisateur', 'utilisateurs', 1)
        verbose_name_plural = ungettext_lazy('utilisateur', 'utilisateurs', 2)

    def __str__(self):
        return self.get_full_name() or self.get_username()

# coding: utf-8

from __future__ import unicode_literals
from django.contrib.auth.models import AbstractUser, UserManager
from django.db.models import BooleanField
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ungettext_lazy
from mptt.fields import TreeForeignKey
from mptt.managers import TreeManager
from mptt.models import MPTTModel
from cache_tools import cached_ugettext_lazy as _

class HierarchicUserManager(TreeManager, UserManager):
    pass

@python_2_unicode_compatible
class HierarchicUser(MPTTModel, AbstractUser):
    mentor = TreeForeignKey(
        'self', null=True, blank=True, related_name='disciples',
        verbose_name=_('mentor'),
        limit_choices_to={'willing_to_be_mentor__exact': True})
    willing_to_be_mentor = BooleanField(
        _('Veut être mentor'), default=False)

    objects = HierarchicUserManager()

    class MPTTMeta(object):
        parent_attr = 'mentor'
        order_insertion_by = ('last_name', 'first_name', 'username')

    class Meta(object):
        verbose_name = ungettext_lazy('utilisateur', 'utilisateurs', 1)
        verbose_name_plural = ungettext_lazy('utilisateur', 'utilisateurs', 2)

    def __str__(self):
        return self.get_full_name() or self.get_username()

Change l'ordre d'insertion des utilisateurs.


from ghostlines.storage.app_storage import AppStorage
from ghostlines.windows.account_details_window import AccountDetailsWindow
from ghostlines.windows.sign_in_window import SignInWindow

class AccountWindow(object):
    def __init__(self, sign_in=SignInWindow, account_details=AccountDetailsWindow):
        if self.is_logged_in:
            self.window = account_details(logout_window=sign_in)
        else:
            self.window = sign_in(success_window=account_details)

    @property
    def is_logged_in(self):
        token = AppStorage("pm.ghostlines.ghostlines.access_token").retrieve()
        # TODO: Retrieve returns NSNull if set to None. Empty string used to clear password for now.
        return token != '' and token is not None

    def open(self):
        self.window.open()

AccountWindow().open()

Remove call to open account window when loading module


import base64
import json
from os import path
import sys

sys.path.insert(0, path.dirname(path.dirname(path.abspath(__file__))))

api_file = 'my_api.json'
_api_file = '{}\{}'.format(path.dirname(path.abspath(__file__)), api_file)

with open(_api_file) as fin:
    cw_api_settings = json.load(fin)
API_URL = cw_api_settings['API_URL']
_cid = cw_api_settings['COMPANYID']
_pubk = cw_api_settings['PUBLICKEY']
_privk = cw_api_settings['PRIVATEKEY']
basic_auth = base64.b64encode("{}+{}:{}".format(_cid, _pubk, _privk).encode('utf-8'))
basic_auth = {'Authorization': 'Basic {}'.format(str(basic_auth, 'utf-8'))}

import base64
import json
from os import path
import sys

sys.path.insert(0, path.dirname(path.dirname(path.abspath(__file__))))

api_file = 'my_api.json'
_api_file = path.join(path.dirname(path.abspath(__file__)), api_file)

with open(_api_file) as fin:
    cw_api_settings = json.load(fin)
API_URL = cw_api_settings['API_URL']
_cid = cw_api_settings['COMPANYID']
_pubk = cw_api_settings['PUBLICKEY']
_privk = cw_api_settings['PRIVATEKEY']
basic_auth = base64.b64encode("{}+{}:{}".format(_cid, _pubk, _privk).encode('utf-8'))
basic_auth = {'Authorization': 'Basic {}'.format(str(basic_auth, 'utf-8'))}

Make api file path OS safe

Make api file path OS safe


import http.client
import os

from flask import Flask
from pymongo import MongoClient

MONGO_URL = os.environ.get('MONGO_URL', 'mongodb://mongo:27017/')
MONGO_DATABASE = os.environ.get('MONGO_DATABASE', 'whistleblower')
DATABASE = MongoClient(MONGO_URL)[MONGO_DATABASE]
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/facebook_webhook', methods=['POST'])
def facebook_webhook():
    DATABASE.facebook_webhook.insert(request.form)
    return ('', http.client.NO_CONTENT)

import http.client
import os

from flask import Flask
from pymongo import MongoClient

MONGO_URL = os.environ.get('MONGO_URL', 'mongodb://mongo:27017/')
MONGO_DATABASE = os.environ.get('MONGO_DATABASE', 'whistleblower')
DATABASE = MongoClient(MONGO_URL)[MONGO_DATABASE]
app = Flask(__name__)

@app.route('/')
def hello_world():
    return 'Hello, World!'

# @app.route('/facebook_webhook', methods=['POST'])
@app.route('/facebook_webhook')
def facebook_webhook():
    DATABASE.facebook_webhook.insert(request.form)
    return ('', http.client.NO_CONTENT)

Save every request coming in the facebook webroot endpoint


# ElasticMapping
# File: types.py
# Desc: base Elasticsearch types

class CallableDict(dict):
    BASE = None
    OVERRIDES = None

    def __call__(self, overrides):
        new_dict = CallableDict(self)
        new_dict.OVERRIDES = overrides
        new_dict.BASE = self
        return new_dict

BASE_TYPE = {
    'store': False,
    'doc_values': False
}

STRING = CallableDict({
    'type': 'string',
    'index': 'analyzed'
})

FLOAT = CallableDict({
    'type': 'float'
})

DOUBLE = CallableDict({
    'type': 'double'
})

INTEGER = CallableDict({
    'type': 'integer'
})

LONG = CallableDict({
    'type': 'long'
})

SHORT = CallableDict({
    'type': 'short'
})

BYTE = CallableDict({
    'type': 'byte'
})

BOOLEAN = CallableDict({
    'type': 'boolean'
})

DATE = CallableDict({
    'type': 'date',
    'format': 'date'
})

DATETIME = CallableDict({
    'type': 'date',
    'format': 'date_hour_minute_second_fraction'
})

TYPES = {
    name: type
    for name, type in locals().items()
    if isinstance(type, CallableDict)
}

# ElasticMapping
# File: types.py
# Desc: base Elasticsearch types

class CallableDict(dict):
    BASE = None
    OVERRIDES = None

    def __call__(self, overrides):
        new_dict = CallableDict(self)
        new_dict.OVERRIDES = overrides
        new_dict.BASE = self
        return new_dict

BASE_TYPE = {
    'store': False,
    'doc_values': False
}

STRING = CallableDict({
    'type': 'string',
    'index': 'analyzed'
})

FLOAT = CallableDict({
    'type': 'float'
})

DOUBLE = CallableDict({
    'type': 'double'
})

INTEGER = CallableDict({
    'type': 'integer'
})

LONG = CallableDict({
    'type': 'long'
})

SHORT = CallableDict({
    'type': 'short'
})

BYTE = CallableDict({
    'type': 'byte'
})

BOOLEAN = CallableDict({
    'type': 'boolean'
})

DATE = CallableDict({
    'type': 'date',
    'format': 'date'
})

TIME = CallableDict({
    'type': 'date',
    'format': 'time'
})

DATETIME = CallableDict({
    'type': 'date',
    'format': 'date_optional_time'
})

TYPES = {
    name: type
    for name, type in locals().items()
    if isinstance(type, CallableDict)
}

Switch default to actual ES default (date_optional_time) and add TIME type


"""distutils

The main package for the Python Module Distribution Utilities.  Normally
used from a setup script as

   from distutils.core import setup

   setup (...)
"""

# Distutils version
#
# Updated automatically by the Python release process.
#
#--start constants--
__version__ = "3.3.0rc3"
#--end constants--

"""distutils

The main package for the Python Module Distribution Utilities.  Normally
used from a setup script as

   from distutils.core import setup

   setup (...)
"""

# Distutils version
#
# Updated automatically by the Python release process.
#
#--start constants--
__version__ = "3.3.0"
#--end constants--

Bump version to 3.3.0 final.

Bump version to 3.3.0 final.


from django.core.management.base import BaseCommand, CommandError
from app.models import Author, Location, AutoComment, Comment, Idea, Vote

class Command(BaseCommand):
    def handle(self, *args, **options):
        self.stdout.write('Starting to clean app tables...')
        try:
            Idea.objects.all().delete()
            self.stdout.write('Ideas deleted')
            Comment.objects.all().delete()
            self.stdout.write('Comments deleted')
            Vote.objects.all().delete()
            self.stdout.write('Votes deleted')
            Location.objects.all().delete()
            self.stdout.write('Locations deleted')
            Author.objects.all().delete()
            self.stdout.write('Authors deleted')
            AutoComment.objects.all().delete()
            self.stdout.write('Automatic Comments deleted')
        except Exception as e:
            raise CommandError('The cleaning procedure couldn\'t finished. Error {}'.format(e))

        self.stdout.write('The procedure has finished successfully...')

from django.core.management.base import BaseCommand, CommandError
from app.models import Author, Location, AutoComment, Comment, Idea, Vote, SocialNetworkAppUser

class Command(BaseCommand):
    def handle(self, *args, **options):
        self.stdout.write('Starting to clean app tables...')
        try:
            Idea.objects.all().delete()
            self.stdout.write('Ideas were deleted')
            Comment.objects.all().delete()
            self.stdout.write('Comments were deleted')
            Vote.objects.all().delete()
            self.stdout.write('Votes were deleted')
            Location.objects.all().delete()
            self.stdout.write('Locations were deleted')
            Author.objects.all().delete()
            self.stdout.write('Authors were deleted')
            AutoComment.objects.all().delete()
            self.stdout.write('Automatic Comments were deleted')
            SocialNetworkAppUser.objects.all().delete()
            self.stdout.write('App users were deleted')
        except Exception as e:
            raise CommandError('The cleaning procedure couldn\'t finished. Error {}'.format(e))

        self.stdout.write('The procedure has finished successfully...')

Add the deletion of the app users

Add the deletion of the app users


from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
from flask import request as flask_request
from flask import abort

import logging
import os

def gen_auth_token(id,expiration=10000):
    """Generate auth token"""
    try:
        s = Serializer(os.environ['API_KEY'],expires_in=expiration)
    except KeyError:
        logging.fatal("No API_KEY env")
        abort(500)

    return s.dumps({'id':id})

def verify_auth_token(token):
    """Verify auth token"""
    try:
        s = Serializer(os.environ['API_KEY'])
    except KeyError:
        logging.fatal("No API_KEY env")
        abort(500)
    # check the token and throw respective exception
    try:
        user = s.loads(token)
    except Exception as e:
        logging.info(e)
        abort(401)
    return user

def enable_auth(func):
    """Decorator to enable auth"""
    def wrapper(*args,**kwargs):
        re = flask_request
        # deny if not authorized
        if not re.headers.has_key("Authorization"):
            logging.info("No token found")
            abort(401)
        auth = re.headers.get("Authorization").split(" ")
        # proces token
        validate = verify_auth_token(auth[1])
        logging.debug("Valid auth! Yay")
        return func(*args,**kwargs)
    return wrapper

from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
from flask import request as flask_request
from flask import abort

import logging
import os

def gen_auth_token(id,expiration=10000):
    """Generate auth token"""
    try:
        s = Serializer(os.environ['API_KEY'],expires_in=expiration)
    except KeyError:
        logging.fatal("No API_KEY env")
        abort(500)

    return s.dumps({'id':id})

def verify_auth_token(token):
    """Verify auth token"""
    try:
        s = Serializer(os.environ['API_KEY'])
    except KeyError:
        logging.fatal("No API_KEY env")
        abort(500)
    # check the token and throw respective exception
    try:
        user = s.loads(token)
    except Exception as e:
        logging.warning("Bad token")
        abort(401)
    return user

def enable_auth(func):
    """Decorator to enable auth"""
    def wrapper(*args,**kwargs):
        re = flask_request
        # deny if not authorized
        if not re.headers.has_key("Authorization"):
            logging.warning("No token found")
            abort(401)
        auth = re.headers.get("Authorization").split(" ")
        # proces token
        validate = verify_auth_token(auth[1])
        logging.debug("Valid auth! Yay")
        return func(*args,**kwargs)
    return wrapper

Add warning log level for auth checks

Add warning log level for auth checks


#-*- coding:utf-8 -*-

"""
Base Recommender Models.
"""

# Authors: Marcel Caraciolo <marcel@muricoca.com>
#          Bruno Melo <bruno@muricoca.com>
# License: BSD Style.

import unittest
import sys

sys.path.append('/Users/marcelcaraciolo/Desktop/Orygens/crab/crab/scikits/craba')

from base import BaseRecommender

#test classes

class MyRecommender(BaseRecommender):
    def __init__(self,model):
        BaseRecommender.__init__(self,model)

################################################################################
# The tests

class testBaseRecommender(unittest.TestCase):
    pass

if __name__ == '__main__':
    unittest.main()

#-*- coding:utf-8 -*-

"""
Base Recommender Models.
"""

# Authors: Marcel Caraciolo <marcel@muricoca.com>
#          Bruno Melo <bruno@muricoca.com>
# License: BSD Style.

import unittest

from base import BaseRecommender

#test classes

class MyRecommender(BaseRecommender):
    def __init__(self,model):
        BaseRecommender.__init__(self,model)

################################################################################
# The tests

class testBaseRecommender(unittest.TestCase):
    pass

if __name__ == '__main__':
    unittest.main()

Fix the test removing paths.

Fix the test removing paths.


"""
Python 2 Compatibility Layer

This module exists to work around compatibility issues between Python 2 and
Python 3. The main code-base will use Python 3 idioms and this module will
patch Python 2 code to support those changes. When Python 2 support is
dropped this module can be removed and imports can be updated.
"""

try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser

    class ConfigParser(SafeConfigParser):

        def read_file(self, fp):
            return self.readfp(fp)

# Only used in tests
try:
    from unittest.mock import Mock, MagicMock, call
except ImportError:
    try:
        from mock import Mock, MagicMock, call, patch
    except ImportError:
        pass

"""
Python 2 Compatibility Layer

This module exists to work around compatibility issues between Python 2 and
Python 3. The main code-base will use Python 3 idioms and this module will
patch Python 2 code to support those changes. When Python 2 support is
dropped this module can be removed and imports can be updated.
"""

try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser

    class ConfigParser(SafeConfigParser):

        def read_file(self, fp):
            return self.readfp(fp)

# Only used in tests
try:
    from unittest.mock import Mock, MagicMock, call, patch
except ImportError:
    try:
        from mock import Mock, MagicMock, call, patch
    except ImportError:
        pass

Add missing import for unittest.mock.patch.


import requests
from django.conf import settings
from rest_framework import serializers
from rest_framework.validators import UniqueTogetherValidator
from server.api.serializers.chapters import ChapterSerializer
from server.core.models.rides import Ride, RideRiders

class RideSerializer(serializers.ModelSerializer):
    chapter = ChapterSerializer()
    riders = serializers.PrimaryKeyRelatedField(source='registered_riders', many=True, read_only=True)

    class Meta:
        model = Ride
        fields = ('id', 'name', 'slug', 'description_html', 'start_location', 'end_location', 'start_date', 'end_date',
                  'chapter', 'rider_capacity', 'riders', 'spaces_left', 'price', 'full_cost', 'currency', 'is_over',
                  'fundraising_total', 'fundraising_target')

class RideRiderSerializer(serializers.ModelSerializer):
    user = serializers.PrimaryKeyRelatedField(read_only=True, default=serializers.CurrentUserDefault())

    class Meta:
        model = RideRiders
        fields = ('ride', 'user', 'signup_date', 'signup_expires', 'status', 'paid', 'expired', 'payload')
        validators = [
            UniqueTogetherValidator(
                queryset=RideRiders.objects.all(),
                fields=('user', 'ride'),
                message='You have already registered for this ride.'
            )
        ]

import requests
from django.conf import settings
from rest_framework import serializers
from rest_framework.validators import UniqueTogetherValidator
from server.api.serializers.chapters import ChapterSerializer
from .riders import RiderSerializer
from server.core.models.rides import Ride, RideRiders

class RideSerializer(serializers.ModelSerializer):
    chapter = ChapterSerializer()
    riders = RiderSerializer(source='registered_riders', many=True, read_only=True)

    class Meta:
        model = Ride
        fields = ('id', 'name', 'slug', 'description_html', 'start_location', 'end_location', 'start_date', 'end_date',
                  'chapter', 'rider_capacity', 'riders', 'spaces_left', 'price', 'full_cost', 'currency', 'is_over',
                  'fundraising_total', 'fundraising_target')

class RideRiderSerializer(serializers.ModelSerializer):
    user = serializers.PrimaryKeyRelatedField(read_only=True, default=serializers.CurrentUserDefault())

    class Meta:
        model = RideRiders
        fields = ('ride', 'user', 'signup_date', 'signup_expires', 'status', 'paid', 'expired', 'payload')
        validators = [
            UniqueTogetherValidator(
                queryset=RideRiders.objects.all(),
                fields=('user', 'ride'),
                message='You have already registered for this ride.'
            )
        ]

Send through serialised users with each ride api request


__version_info__ = (0, 14, 0)
__version__ = '.'.join(map(str, __version_info__))

__version_info__ = (0, 14, 1)
__version__ = '.'.join(map(str, __version_info__))

Cut 0.14.1 because setup.py bug

Cut 0.14.1 because setup.py bug


from pyramid.settings import asbool

from .client import ElasticClient

def client_from_config(settings, prefix='elastic.'):
    """
    Instantiate and configure an Elasticsearch from settings.

    In typical Pyramid usage, you shouldn't use this directly: instead, just
    include ``pyramid_es`` and use the :py:func:`get_client` function to get
    access to the shared :py:class:`.client.ElasticClient` instance.
    """
    return ElasticClient(
        servers=settings.get(prefix + 'servers', ['localhost:9200']),
        timeout=settings.get(prefix + 'timeout', 1.0),
        index=settings[prefix + 'index'],
        use_transaction=asbool(settings.get(prefix + 'use_transaction', True)),
        disable_indexing=settings.get(prefix + 'disable_indexing', False))

def includeme(config):
    registry = config.registry
    settings = registry.settings

    client = client_from_config(settings)
    client.ensure_index()

    registry.pyramid_es_client = client

def get_client(request):
    """
    Get the registered Elasticsearch client. The supplied argument can be
    either a ``Request`` instance or a ``Registry``.
    """
    registry = getattr(request, 'registry', None)
    if registry is None:
        registry = request
    return registry.pyramid_es_client

from pyramid.settings import asbool

from .client import ElasticClient

def client_from_config(settings, prefix='elastic.'):
    """
    Instantiate and configure an Elasticsearch from settings.

    In typical Pyramid usage, you shouldn't use this directly: instead, just
    include ``pyramid_es`` and use the :py:func:`get_client` function to get
    access to the shared :py:class:`.client.ElasticClient` instance.
    """
    return ElasticClient(
        servers=settings.get(prefix + 'servers', ['localhost:9200']),
        timeout=settings.get(prefix + 'timeout', 1.0),
        index=settings[prefix + 'index'],
        use_transaction=asbool(settings.get(prefix + 'use_transaction', True)),
        disable_indexing=settings.get(prefix + 'disable_indexing', False))

def includeme(config):
    registry = config.registry
    settings = registry.settings

    client = client_from_config(settings)
    if asbool(settings.get('elastic.ensure_index_on_start')):
        client.ensure_index()

    registry.pyramid_es_client = client

def get_client(request):
    """
    Get the registered Elasticsearch client. The supplied argument can be
    either a ``Request`` instance or a ``Registry``.
    """
    registry = getattr(request, 'registry', None)
    if registry is None:
        registry = request
    return registry.pyramid_es_client

Add a settings key to ensure index at start


from subprocess import PIPE

from .monkey import Popen
from .exceptions import Failure

class Result(object):
    def __init__(self, stdout=None, stderr=None, exited=None):
        self.exited = self.return_code = exited
        self.stdout = stdout
        self.stderr = stderr

    def __nonzero__(self):
        # Holy mismatch between name and implementation, Batman!
        return self.exited == 0

def run(command, warn=False):
    """
    Execute ``command`` in a local subprocess.

    By default, raises an exception if the subprocess terminates with a nonzero
    return code. This may be disabled by setting ``warn=True``.
    """
    process = Popen(command,
        shell=True,
        stdout=PIPE,
        stderr=PIPE
    )
    stdout, stderr = process.communicate()
    result = Result(stdout=stdout, stderr=stderr, exited=process.returncode)
    if not (result or warn):
        raise Failure(result)
    return result

from subprocess import PIPE

from .monkey import Popen
from .exceptions import Failure

class Result(object):
    def __init__(self, stdout=None, stderr=None, exited=None):
        self.exited = self.return_code = exited
        self.stdout = stdout
        self.stderr = stderr

    def __nonzero__(self):
        # Holy mismatch between name and implementation, Batman!
        return self.exited == 0

    def __str__(self):
        ret = ["Command exited with status %s." % self.exited]
        for x in ('stdout', 'stderr'):
            val = getattr(self, x)
            ret.append("""=== %s ===
%s
""" % (x, val.rstrip()) if val else "(no %s)" % x)
        return "\n".join(ret)

def run(command, warn=False):
    """
    Execute ``command`` in a local subprocess.

    By default, raises an exception if the subprocess terminates with a nonzero
    return code. This may be disabled by setting ``warn=True``.
    """
    process = Popen(command,
        shell=True,
        stdout=PIPE,
        stderr=PIPE
    )
    stdout, stderr = process.communicate()
    result = Result(stdout=stdout, stderr=stderr, exited=process.returncode)
    if not (result or warn):
        raise Failure(result)
    return result

Add semi-useful `__str__` for Result

Add semi-useful `__str__` for Result


from pythonforandroid.recipe import CompiledComponentsPythonRecipe

class RegexRecipe(CompiledComponentsPythonRecipe):
    name = 'regex'
    version = '2017.07.28'
    url = 'https://pypi.python.org/packages/d1/23/5fa829706ee1d4452552eb32e0bfc1039553e01f50a8754c6f7152e85c1b/regex-{version}.tar.gz'

    depends = ['setuptools']

recipe = RegexRecipe()

from pythonforandroid.recipe import CompiledComponentsPythonRecipe

class RegexRecipe(CompiledComponentsPythonRecipe):
    name = 'regex'
    version = '2017.07.28'
    url = 'https://pypi.python.org/packages/d1/23/5fa829706ee1d4452552eb32e0bfc1039553e01f50a8754c6f7152e85c1b/regex-{version}.tar.gz'

    depends = ['setuptools']
    call_hostpython_via_targetpython = False

recipe = RegexRecipe()

Fix compilation for regex recipe

[recipes] Fix compilation for  regex recipe

The error was: build/other_builds/hostpython3/desktop/hostpython3/Include/Python.h:39:19: fatal error: crypt.h: No such file or directory


import json
from relay_api.core.relay import relay
from relay_api.conf.config import relays

def init_relays():
    for r in relays:
        relays[r]["object"] = relay(relays[r]["gpio"])
        relays[r]["state"] = relays[r]["object"].get_state()

def get_all_relays():
    relays_dict = __get_relay_dict()
    return json.dumps(relays_dict, indent=4)

def get_relay(relay_name):
    if relay_name not in relays:
        return None
    relay_dict = __get_relay_dict(relay_name)
    return json.dumps(relay_dict, indent=4)

def __get_relay_dict(relay_name=None):
    if relay_name:
        relay_dict = dict.copy(relays["relay_name"])
        del(relay_dict["object"])
        return relay_dict
    relays_dict = dict.copy(relays)
    for r in relays_dict:
        del(relays_dict[r]["object"])
    return relays_dict

import json
import copy
from relay_api.core.relay import relay
from relay_api.conf.config import relays

def init_relays():
    for r in relays:
        relays[r]["object"] = relay(relays[r]["gpio"])
        relays[r]["state"] = relays[r]["object"].get_state()

def get_all_relays():
    relays_dict = __get_relay_dict()
    return json.dumps(relays_dict, indent=4)

def get_relay(relay_name):
    if relay_name not in relays:
        return None
    relay_dict = __get_relay_dict(relay_name)
    return json.dumps(relay_dict, indent=4)

def __get_relay_dict(relay_name=None):
    if relay_name:
        relay_dict = copy.deepcopy(relays[relay_name])
        del(relay_dict["object"])
        return relay_dict
    relays_dict = copy.deepcopy(relays)
    for r in relays_dict:
        del(relays_dict[r]["object"])
    return relays_dict

Use deppcopy to copy dicts

Use deppcopy to copy dicts


import time
from sr import *

print "Hello, world!"

R = Robot()

# This is the configuration for Elizabeth.
# TODO: make this generic
BOARD_RIGHT = R.motors["SR0HL17"]
BOARD_LEFT = R.motors["SR0YK1C"]

WHEEL_FRONT_LEFT = BOARD_LEFT.m1 # positive is towards the front of the robot
WHEEL_FRONT_RIGHT = BOARD_RIGHT.m0 # positive is towards the front of the robot
WHEEL_BACK = BOARD_RIGHT.m1 # positive is to the right of the robot

# enable the brakes
WHEEL_FRONT_LEFT.use_brake = True
WHEEL_FRONT_RIGHT.use_brake = True
WHEEL_BACK.use_brake = True

WHEEL_FRONT_LEFT_CALIBRATION = -1
WHEEL_FRONT_RIGHT_CALIBRATION = -1
WHEEL_BACK_CALIBRATION = 1

def set_motors(front_left, front_right, back):
    WHEEL_FRONT_LEFT.power = int(front_left * WHEEL_FRONT_LEFT_CALIBRATION)
    WHEEL_FRONT_RIGHT.power = int(front_right * WHEEL_FRONT_RIGHT_CALIBRATION)
    WHEEL_BACK.power = int(back * WHEEL_BACK_CALIBRATION)

def forward(speed):
    set_motors(speed, speed, 0)

def reverse(speed):
    forward(-speed)

def stop():
    forward(0)

def rotate(speed):
    set_motors(speed, -speed, speed)

def can_see_block():
    ACCEPTABLE_MARKER_TYPES = (MARKER_TOKEN_TOP,
                               MARKER_TOKEN_BOTTOM,
                               MARKER_TOKEN_SIDE)
    markers = R.see()
    return any(marker.info.marker_type in ACCEPTABLE_MARKER_TYPES
                for marker in R.see())

# FIXME: debug while we don't have a marker
search_count = 0
def can_see_block():
    global search_count
    search_count += 1
    return search_count % 5 == 0

def state_search():
    rotate(40)
    time.sleep(0.4)
    stop()
    time.sleep(0.3)
    return state_advance if can_see_block() else state_search

def state_advance():
    forward(30)
    time.sleep(1)
    stop()
    time.sleep(0.8)
    return state_advance if can_see_block() else state_backoff

def state_backoff():
    reverse(20)
    time.sleep(2)
    stop()
    rotate(-80)
    time.sleep(4)
    stop()
    return state_search

current_state = state_search
while True:
    current_state = current_state()

Implement a basic search-and-grab algorithm


#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Andrew Grim
# Copyright (c) 2014 Andrew Grim
#
# License: MIT
#

"""This module exports the Puppet plugin class."""

from SublimeLinter.lint import Linter, util

class Puppet(Linter):

    """Provides an interface to puppet."""

    syntax = 'puppet'
    cmd = ('puppet', 'parser', 'validate', '--color=false')
    regex = r'^(?P<error>Error:).+?(?P<message>Syntax error at \'(?P<near>.+?)\'; expected \'.+\').+?line (?P<line>\d+)'
    error_stream = util.STREAM_STDERR
    defaults = {}

#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Andrew Grim
# Copyright (c) 2014 Andrew Grim
#
# License: MIT
#

"""This module exports the Puppet plugin class."""

from SublimeLinter.lint import Linter, util

class Puppet(Linter):

    """Provides an interface to puppet."""

    syntax = 'puppet'
    cmd = ('puppet', 'parser', 'validate', '--color=false')
    regex = r'^Error:.+?(?P<message>Syntax error at \'(?P<near>.+?)\'; expected \'.+\').+?line (?P<line>\d+)'
    error_stream = util.STREAM_STDERR

Remove unnecessary defaults and simplify regexp


#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jack Brewer
# Copyright (c) 2015 Jack Brewer
#
# License: MIT

"""Exports the Stylint plugin class."""

from SublimeLinter.lint import NodeLinter, util

class Stylint(NodeLinter):

    """Provides an interface to stylint."""

    npm_name = 'stylint'
    syntax = ('stylus', 'vue')
    selectors = {'vue': 'source.stylus.embedded.html'}
    cmd = 'stylint @ *'
    executable = 'stylint'
    version_requirement = '>= 1.5.0'
    regex = r'''(?xi)
        # Comments show example output for each line of a Stylint warning
        # /path/to/file/example.styl
        ^.*$\s*
        # 177:24 colors warning hexidecimal color should be a variable
        ^(?P<line>\d+):?(?P<col>\d+)?\s*((?P<warning>warning)|(?P<error>error))\s*(?P<message>.+)$\s*
    '''
    multiline = True
    error_stream = util.STREAM_STDOUT
    tempfile_suffix = 'styl'
    config_file = ('--config', '.stylintrc', '~')

#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jack Brewer
# Copyright (c) 2015 Jack Brewer
#
# License: MIT

"""Exports the Stylint plugin class."""

from SublimeLinter.lint import NodeLinter, util

class Stylint(NodeLinter):

    """Provides an interface to stylint."""

    npm_name = 'stylint'
    syntax = ('stylus', 'vue')
    selectors = {'vue': 'source.stylus.embedded.html'}
    cmd = 'stylint @ *'
    executable = 'stylint'
    version_requirement = '>= 1.5.0'
    regex = r'''(?xi)
        # Comments show example output for each line of a Stylint warning
        # /path/to/file/example.styl
        ^.*$\s*
        # 177:24 colors warning hexidecimal color should be a variable
        ^(?P<line>\d+):?(?P<col>\d+)?\s*(?P<rule>\w+)?\s*((?P<warning>warning)|(?P<error>error))\s*(?P<message>.+)$\s*
    '''
    multiline = True
    error_stream = util.STREAM_STDOUT
    tempfile_suffix = 'styl'
    config_file = ('--config', '.stylintrc', '~')

Handle case where rule shows before severity

Thank you @suprMax !


from gratipay.wireup import db, env
from gratipay.models.team import AlreadyMigrated

db = db(env())

teams = db.all("""
    SELECT distinct ON (t.slug) t.*::teams
      FROM teams t
      JOIN tips ON t.owner = tips.tippee    -- Only fetch teams whose owner have tips.
     WHERE t.is_approved IS TRUE            -- Only fetch approved teams.
       AND NOT EXISTS (                     -- Make sure not already migrated.
            SELECT 1
              FROM payment_instructions pi
             WHERE t.slug = pi.team
               AND pi.ctime < t.ctime
	     )
""")

for team in teams:
    try:
        ntips = team.migrate_tips()
        print("Migrated {} tip(s) for '{}'".format(ntips, team.slug))
    except AlreadyMigrated:
        print("'%s' already migrated." % team.slug)

print("Done.")

from gratipay.wireup import db, env
from gratipay.models.team import AlreadyMigrated

db = db(env())

teams = db.all("""
    SELECT distinct ON (t.slug) t.*::teams
      FROM teams t
      JOIN tips ON t.owner = tips.tippee    -- Only fetch teams whose owners had tips under Gratipay 1.0
     WHERE t.is_approved IS TRUE            -- Only fetch approved teams
       AND NOT EXISTS (                     -- Make sure tips haven't been migrated for any teams with same owner
            SELECT 1
              FROM payment_instructions pi
              JOIN teams t2 ON t2.slug = pi.team
             WHERE t2.owner = t.owner
               AND pi.ctime < t2.ctime
       )
""")

for team in teams:
    try:
        ntips = team.migrate_tips()
        print("Migrated {} tip(s) for '{}'".format(ntips, team.slug))
    except AlreadyMigrated:
        print("'%s' already migrated." % team.slug)

print("Done.")

Exclude teams if owner has other teams with migrated tips


# Generated by Django 2.1.4 on 2019-01-05 15:00

import django.contrib.postgres.fields
from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        (
            "peering",
            "0026_autonomoussystem_potential_internet_exchange_peering_sessions",
        )
    ]

    operations = [
        migrations.AlterField(
            model_name="autonomoussystem",
            name="potential_internet_exchange_peering_sessions",
            field=django.contrib.postgres.fields.ArrayField(
                base_field=models.GenericIPAddressField(),
                blank=True,
                default=list,
                size=None,
            ),
        )
    ]

# Generated by Django 2.1.4 on 2019-01-05 15:00

import django.contrib.postgres.fields
from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        (
            "peering",
            "0026_autonomoussystem_potential_internet_exchange_peering_sessions",
        )
    ]

    def forwards_func(apps, schema_editor):
        AutonomousSystem = apps.get_model("peering", "AutonomousSystem")
        db_alias = schema_editor.connection.alias
        AutonomousSystem.objects.using(db_alias).filter(
            potential_internet_exchange_peering_sessions=None
        ).update(potential_internet_exchange_peering_sessions=[])

    def reverse_func(apps, schema_editor):
        AutonomousSystem = apps.get_model("peering", "AutonomousSystem")
        db_alias = schema_editor.connection.alias
        AutonomousSystem.objects.using(db_alias).filter(
            potential_internet_exchange_peering_sessions=[]
        ).update(potential_internet_exchange_peering_sessions=None)

    operations = [
        migrations.AlterField(
            model_name="autonomoussystem",
            name="potential_internet_exchange_peering_sessions",
            field=django.contrib.postgres.fields.ArrayField(
                base_field=models.GenericIPAddressField(),
                blank=True,
                default=list,
                size=None,
            ),
        ),
        migrations.RunPython(forwards_func, reverse_func),
    ]

Fix issue with migrations introduced lately.


# coding=utf-8
from __future__ import unicode_literals
from .. import Provider as SsnProvider

# Note: as there no SSN in Ukraine
# we get value added tax identification number (VATIN) here.
# It is also called "Ідентифікаційний номер платника податків" (in ukrainian).
# It contains only digits and length if 12.

class Provider(SsnProvider):
    ssn_formats = ("############",)

# coding=utf-8
from __future__ import unicode_literals

from datetime import date

from .. import Provider as SsnProvider
from faker.providers.date_time import Provider as DateTimeProvider

class Provider(SsnProvider):
    @classmethod
    def ssn(cls):
        """
        Ukrainian "Реєстраційний номер облікової картки платника податків"
        also known as "Ідентифікаційний номер фізичної особи".
        """
        digits = []

        # Number of days between 1899-12-31 and a birth date
        for digit in str((DateTimeProvider.date_object() -
                          date(1899, 12, 31)).days):
            digits.append(int(digit))

        # Person's sequence number
        for _ in range(4):
            digits.append(cls.random_int(0, 9))

        checksum = (digits[0]*-1 + digits[1]*5 + digits[2]*7 + digits[3]*9 +
                    digits[4]*4 + digits[5]*6 + digits[6]*10 + digits[7]*5 +
                    digits[8]*7)
        # Remainder of a checksum divided by 11 or 1 if it equals to 10
        digits.append(checksum % 11 % 10)

        return ''.join(str(digit) for digit in digits)

Make the Ukrainian SSN provider realer

Make the Ukrainian SSN  provider realer


import pytest
import requests
from time import sleep

COMPONENT_INDEX_URL = "http://identidock:5000"

COMPONENT_MONSTER_BASE_URL = COMPONENT_INDEX_URL + '/monster'

def test_get_mainpage():
	print('component tester sleeping for 1 sec to let the identidock app to be ready adn also start its server')
	sleep(1)
	page = requests.get(COMPONENT_INDEX_URL)
	assert page.status_code == 200
	assert 'Joe Bloggs' in str(page.text)

def test_post_mainpage():
	page = requests.post(COMPONENT_INDEX_URL, data=dict(name="Moby Dock"))
	assert page.status_code == 200
	assert 'Moby Dock' in str(page.text)

def test_mainpage_html_escaping():
	page = requests.post(COMPONENT_INDEX_URL, data=dict(name='"><b>TEST</b><!--'))
	assert page.status_code == 200
	assert '<b>' not in str(page.text)

def test_get_identicon_with_valid_name_and_invalid_post_method_should_return_405():
	name_hash = 'ABCDEF123456789'

	page = requests.post('{0}/{1}'.format(COMPONENT_MONSTER_BASE_URL, name_hash))

	assert page.status_code == 405

def test_get_identicon_with_valid_name_and_cache_miss():
	name_hash = 'ABCDEF123456789'
	page = requests.get('{0}/{1}'.format(COMPONENT_MONSTER_BASE_URL, name_hash))

	# print('page.content : {0}'.format(page.content))
	assert page.status_code == 200

def test_get_identicon_with_valid_name_and_cache_hit():
	name_hash = 'ABCDEF123456789'
	page = requests.get('{0}/{1}'.format(COMPONENT_MONSTER_BASE_URL, name_hash))

	# print('page.content : {0}'.format(page.content))
	assert page.status_code == 200

def test_get_identicon_with_insecure_and_unescaped_invalid_name_hash():
	invalid_name_hash = '<b>;i_am_invalid|name <{"'

	page = requests.get('{0}/{1}'.format(COMPONENT_MONSTER_BASE_URL, invalid_name_hash))

	# print('page.content : {0}'.format(page.content))
	assert page.status_code == 200

if __name__ == '__main__':
	# unittest.main()
	pytest.main()

Add component test functions using pytest


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest

import const
import uniformdh

import obfsproxy.network.buffer as obfs_buf

class UniformDHTest( unittest.TestCase ):

    def setUp( self ):
        weAreServer = True
        self.udh = uniformdh.new("A" * const.SHARED_SECRET_LENGTH, weAreServer)

    def test1_createHandshake( self ):
        handshake = self.udh.createHandshake()
        self.failUnless((const.PUBLIC_KEY_LENGTH +
                         const.MARKER_LENGTH +
                         const.HMAC_LENGTH) <= len(handshake) <=
                        (const.MARKER_LENGTH +
                         const.HMAC_LENGTH +
                         const.MAX_PADDING_LENGTH))

    def test2_receivePublicKey( self ):
        buf = obfs_buf.Buffer(self.udh.createHandshake())

        self.failUnless(self.udh.receivePublicKey(buf, lambda x: x) == True)

        publicKey = self.udh.getRemotePublicKey()
        self.failUnless(len(publicKey) == const.PUBLIC_KEY_LENGTH)

    def test3_invalidHMAC( self ):
        # Make the HMAC invalid.
        handshake = self.udh.createHandshake()
        if handshake[-1] != 'a':
            handshake = handshake[:-1] + 'a'
        else:
            handshake = handshake[:-1] + 'b'

        buf = obfs_buf.Buffer(handshake)

        self.failIf(self.udh.receivePublicKey(buf, lambda x: x) == True)

if __name__ == '__main__':
    unittest.main()

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest

import const
import uniformdh

import obfsproxy.network.buffer as obfs_buf

class UniformDHTest( unittest.TestCase ):

    def setUp( self ):
        weAreServer = True
        self.udh = uniformdh.new("A" * const.SHARED_SECRET_LENGTH, weAreServer)

    def test1_createHandshake( self ):
        handshake = self.udh.createHandshake()
        self.failUnless((const.PUBLIC_KEY_LENGTH +
                         const.MARKER_LENGTH +
                         const.HMAC_LENGTH) <= len(handshake) <=
                        (const.MARKER_LENGTH +
                         const.HMAC_LENGTH +
                         const.MAX_PADDING_LENGTH))

    def test2_receivePublicKey( self ):
        buf = obfs_buf.Buffer(self.udh.createHandshake())

        def callback( masterKey ):
            self.failUnless(len(masterKey) == const.MASTER_KEY_LENGTH)

        self.failUnless(self.udh.receivePublicKey(buf, callback) == True)

        publicKey = self.udh.getRemotePublicKey()
        self.failUnless(len(publicKey) == const.PUBLIC_KEY_LENGTH)

    def test3_invalidHMAC( self ):
        # Make the HMAC invalid.
        handshake = self.udh.createHandshake()
        if handshake[-1] != 'a':
            handshake = handshake[:-1] + 'a'
        else:
            handshake = handshake[:-1] + 'b'

        buf = obfs_buf.Buffer(handshake)

        self.failIf(self.udh.receivePublicKey(buf, lambda x: x) == True)

if __name__ == '__main__':
    unittest.main()

Extend UniformDH test to also verify the length of the shared master secret.


from flask_script import Manager
from flask_skeleton_api.main import app
import subprocess
import os

manager = Manager(app)

@manager.command
def unittest(report=False):
    """Run unit tests"""

    if report:
        subprocess.call(["py.test", "--junitxml=test-output/unit-test-output.xml",
                         "--cov-report=html:test-output/unit-test-cov-report"])
    else:
        subprocess.call(["py.test"])

@manager.command
def integrationtest(report=False):
    """Run integration tests"""

    if report:
        subprocess.call(["py.test", "--junitxml=test-output/integration-test-output.xml",
                         "integration_tests"])
    else:
        subprocess.call(["py.test", "integration_tests"])

@manager.command
def runserver(port=9998):
    """Run the app using flask server"""

    os.environ["PYTHONUNBUFFERED"] = "yes"
    os.environ["LOG_LEVEL"] = "DEBUG"
    os.environ["COMMIT"] = "LOCAL"

    app.run(debug=True, port=int(port))

if __name__ == "__main__":
    manager.run()

from flask_script import Manager
from flask_skeleton_api.main import app
import subprocess
import os
# Using Alembic?
# See what extra lines are needed here:
# http://192.168.249.38/gadgets/gadget-api/blob/master/manage.py

manager = Manager(app)

@manager.command
def unittest(report=False):
    """Run unit tests"""

    if report:
        subprocess.call(["py.test", "--junitxml=test-output/unit-test-output.xml",
                         "--cov-report=html:test-output/unit-test-cov-report"])
    else:
        subprocess.call(["py.test"])

@manager.command
def integrationtest(report=False):
    """Run integration tests"""

    if report:
        subprocess.call(["py.test", "--junitxml=test-output/integration-test-output.xml",
                         "integration_tests"])
    else:
        subprocess.call(["py.test", "integration_tests"])

@manager.command
def runserver(port=9998):
    """Run the app using flask server"""

    os.environ["PYTHONUNBUFFERED"] = "yes"
    os.environ["LOG_LEVEL"] = "DEBUG"
    os.environ["COMMIT"] = "LOCAL"

    app.run(debug=True, port=int(port))

if __name__ == "__main__":
    manager.run()

Add link to gadget for Alembic code

Add link to gadget for Alembic code


# -*- coding: utf-8 -*-

from learning_journal.models import Entry, DBSession

def test_create_entry(dbtransaction):
    """Assert entry was entered into database."""
    new_entry = Entry(title="Entry1", text="Hey, this works. Awesome.")
    assert new_entry.id is None
    DBSession.flush
    assert new_entry.id is not None

# -*- coding: utf-8 -*-

from learning_journal.models import Entry, DBSession

def test_create_entry(dbtransaction):
    """Assert entry was entered into database."""
    new_entry = Entry(title="Entry1", text="Hey, this works. Awesome.")
    assert new_entry.id is None
    DBSession.add(new_entry)
    DBSession.flush
    assert new_entry.id is not None

Modify test file, still doesn't work. Messed around in pshell.


import asyncio
import json

import aiohttp

import SLA_bot.config as cf

class AlertFeed:
    source_url = 'http://pso2emq.flyergo.eu/api/v2/'

    async def download(url):
        try:
            async with aiohttp.get(url) as response:
                return await response.json()
        except json.decoder.JSONDecodeError:
            pass

    def parse_data(data):
        latest_alert = data[0]['text']
        lines = latest_alert.splitlines()
        header = '-' * len(lines[0])
        lines.insert(1, header)
        text = '\n'.join(lines)
        return '```fix\n{}\n```'.format(text)

    async def fetch():
        header = cf.get('PSO2 Feed', 'header')
        raw_data = await AlertFeed.download(AlertFeed.source_url)
        return '** **\n' + header + '\n' + AlertFeed.parse_data(raw_data)

import asyncio
import json

import aiohttp

import SLA_bot.config as cf

class AlertFeed:
    source_url = 'http://pso2emq.flyergo.eu/api/v2/'

    async def download(url):
        try:
            async with aiohttp.get(url) as response:
                return await response.json()
        except json.decoder.JSONDecodeError:
            pass

    def parse_data(data):
        latest_alert = data[0]['text']
        lines = latest_alert.splitlines()
        code_color = 'fix' if len(lines) >= 10 else ''
        header = '-' * len(lines[0])
        lines.insert(1, header)
        text = '\n'.join(lines)
        return '```{}\n{}\n```'.format(code_color, text)

    async def fetch():
        header = cf.get('PSO2 Feed', 'header')
        raw_data = await AlertFeed.download(AlertFeed.source_url)
        return '** **\n' + header + '\n' + AlertFeed.parse_data(raw_data)

Remove text coloring in AlertFeed if it seems like scheduled text


from setuptools import setup

v = '4.10.1'
setup(
    name='antlr4-python2-runtime',
    version=v,
    url='http://www.antlr.org',
    license='BSD',
    packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'],
    package_dir={'': 'src'},
    author='Eric Vergnaud, Terence Parr, Sam Harwell',
    author_email='eric.vergnaud@wanadoo.fr',
    description=f'ANTLR {v} runtime for Python 2.7.12'
)

from setuptools import setup

v = '4.10.1'
setup(
    name='antlr4-python2-runtime',
    version=v,
    url='http://www.antlr.org',
    license='BSD',
    packages=['antlr4', 'antlr4.atn', 'antlr4.dfa', 'antlr4.tree', 'antlr4.error', 'antlr4.xpath'],
    package_dir={'': 'src'},
    author='Eric Vergnaud, Terence Parr, Sam Harwell',
    author_email='eric.vergnaud@wanadoo.fr',
    description='ANTLR %s runtime for Python 2.7.12' % v
)

Fix SyntaxError due to F string

[py2] Fix SyntaxError due to F string

Signed-off-by: Travis Thieman <f1ef50ba1343ab5680bff0994219d82815f791bd@gmail.com>


from os import path

from devops.utils.version import Version

PROJECT_NAME = 'python-clean-architecture'
PACKAGE_NAME = 'pca'
VERSION = Version(0, 0, 3)

PACKAGE_DIR = path.dirname(__file__)
PROJECT_DIR = path.dirname(PACKAGE_DIR)

from os import path

from devops.utils.version import Version

PROJECT_NAME = 'python-clean-architecture'
PACKAGE_NAME = 'pca'
VERSION = Version(0, 0, 4)

PACKAGE_DIR = path.dirname(__file__)
PROJECT_DIR = path.dirname(PACKAGE_DIR)

Bump version: 0.0.3 -> 0.0.4

Bump version: 0.0.3 -> 0.0.4


# Use the codon variables you defined previously
S = "TCT"
L = "CTT"
Y = "TAT"
C = "TGT"

# Create a list for the protein sequence CLYSY
codons = [C, L, Y, S, Y]

# Print the DNA sequence of the protein
print("DNA sequence:", codons)

# Print the DNA sequence of the last amino acid
print("Last codon:", codons[-1])

# Create two more variables containing the DNA sequence for a stop codon and a start codon
start = "ATG"
stop = "TGA"

# Replace the first element of the list with the start codon
codons[0] = start

# Append the stop codon to the end of the list
codons.append(stop)

# Print the resulting DNA sequence
print("DNA sequence after alteration:", "".join(codons))

# Use the codon variables you defined previously
S = "TCT"
L = "CTT"
Y = "TAT"
C = "TGT"

# Create a list for the protein sequence CLYSY
codons = [C, L, Y, S, Y]

# Print the DNA sequence of the protein
print("DNA sequence:", codons)

# Print the DNA sequence of the last amino acid
print("Last codon:", codons[-1])

# Create two more variables containing the DNA sequence for a stop codon and a start codon
start = "ATG"
stop = "TGA"

# Replace the first element of the list with the start codon
codons[0] = start

# Append the stop codon to the end of the list
codons.append(stop)

# Print the resulting DNA sequence
print("DNA sequence after alteration:", codons)

Remove join in exercise 1.3.1 not seen yet in course


from django.db import models
from django.utils import timezone
from datetime import timedelta

class Poll(models.Model):
    text = models.CharField(max_length=200)
    created_ts = models.DateTimeField()
    updated_ts = models.DateTimeField(null=True, default=None)
    is_published = models.BooleanField(default=False)
    publication_date = models.DateTimeField(
        'date published',
        default=None,
        )

    def __str__(self):
        return self.text

    def was_published_recently(self):
        return self.publication_date >= timezone.now() - timedelta(days=1)

    def save(self, *args, **kwargs):
        ''' On save, update timestamps '''
        if not self.id:
            self.created_ts = timezone.now()
            self.updated_ts = timezone.now()
            return super(Poll, self).save(*args, **kwargs)

class Choice(models.Model):
    poll = models.ForeignKey(Poll, on_delete=models.CASCADE)
    text = models.CharField(max_length=200)
    votes = models.IntegerField(default=0)

    def __str__(self):
        return self.text

from django.db import models
from django.utils import timezone
from datetime import timedelta

class Poll(models.Model):
    text = models.CharField(max_length=200)
    created_ts = models.DateTimeField()
    updated_ts = models.DateTimeField(null=True, default=None)
    is_published = models.BooleanField(default=False)
    publication_date = models.DateTimeField(
        'date published',
        default=None,
        )

    def __str__(self):
        return self.text

    def was_published_recently(self):
        now = timezone.now()
        return now - timedelta(days=1) <= self.publication_date <= now

    def save(self, *args, **kwargs):
        ''' On save, update timestamps '''
        if not self.id:
            self.created_ts = timezone.now()
            self.updated_ts = timezone.now()
            return super(Poll, self).save(*args, **kwargs)

class Choice(models.Model):
    poll = models.ForeignKey(Poll, on_delete=models.CASCADE)
    text = models.CharField(max_length=200)
    votes = models.IntegerField(default=0)

    def __str__(self):
        return self.text

Fix was_published_recently reporting polls from the future


from motobot import command, Notice, split_response, IRCBot

@command('commands')
def commands_command(bot, database, context, message, args):
    userlevel = bot.get_userlevel(context.channel, context.nick)

    valid_command = lambda plugin: plugin.type == IRCBot.command_plugin \
        and plugin.level <= userlevel and not plugin.arg.hidden
    key = lambda plugin: (plugin.arg.trigger, plugin.func)

    command_groups = {}
    for command, func in map(key, filter(valid_command, bot.plugins)):
        value = command_groups.get(func, [])
        value.append(command)
        command_groups[func] = value

    format_group = lambda group: '({})'.format(', '.join(group)) \
        if len(group) != 1 else group[0]
    commands = map(format_group, sorted(command_groups.values(), key=lambda x: x[0]))
    response = split_response(commands, "Bot Commands: {};")

    return response, Notice(context.nick)

from motobot import command, Notice, split_response, IRCBot
from collections import defaultdict

def filter_plugins(plugins, userlevel):
    return map(
        lambda plugin: (plugin.arg.trigger, plugin.func), filter(
            lambda plugin: plugin.type == IRCBot.command_plugin and
                           plugin.level <= userlevel and not plugin.arg.hidden,
            plugins
        )
    )

def format_group(group):
    return '({})'.format(', '.join(group)) if len(group) != 1 else group[0]

@command('commands')
def commands_command(bot, database, context, message, args):
    userlevel = bot.get_userlevel(context.channel, context.nick)
    groups = defaultdict(lambda: [])

    for command, func in filter_plugins(bot.plugins, userlevel):
        groups[func].append(command)

    commands = map(format_group, sorted(groups.values(), key=lambda x: x[0]))
    response = split_response(commands, "Bot Commands: {};")

    return response, Notice(context.nick)

Revert "Revert "Cleans up split_response""

This reverts commit c3c62feb9fbd8b7ff35d70eaaa5fecfb2093dbb0.


from twisted.internet import endpoints, reactor
from txampext.multiplexing import ProxyingFactory

def connectProxy(namespace, identifier, _reactor=reactor):
    """Start listening on some free local port; connections will be
    proxied to the virtual server with the given identifier.

    """
    remote = namespace.get("remote")
    if remote is None:
        raise RuntimeError("You are not connected to the exercise server. "
                           "Call ``connect``.")

    factory = ProxyingFactory(remote, identifier)
    endpoint = endpoints.TCP4ServerEndpoint(_reactor, 0, interface="localhost")
    d = endpoint.listen(factory)
    d.addCallback(_listening, namespace, identifier)
    return None

def _listening(listeningPort, namespace, identifier):
    """Started listening; report success to terminal.

    """
    host = listeningPort.getHost()
    template = "{id} is now listening on {h.host}:{h.port}"
    namespace["manhole"].writeLine(template.format(h=host, id=identifier))

from stanczyk.util import _getRemote
from twisted.internet import endpoints, reactor
from txampext.multiplexing import ProxyingFactory

def connectProxy(namespace, identifier, _reactor=reactor):
    """Start listening on some free local port; connections will be
    proxied to the virtual server with the given identifier.

    """
    endpoint = endpoints.TCP4ServerEndpoint(_reactor, 0, interface="localhost")
    factory = ProxyingFactory(_getRemote(namespace), identifier)
    d = endpoint.listen(factory)
    d.addCallback(_listening, namespace, identifier)
    return None

def _listening(listeningPort, namespace, identifier):
    """Started listening; report success to terminal.

    """
    host = listeningPort.getHost()
    template = "{id} is now listening on {h.host}:{h.port}"
    namespace["manhole"].writeLine(template.format(h=host, id=identifier))

Use the new fancy refactored remote logic


import os
import subprocess

try:
    VERSION = __import__('pkg_resources') \
        .get_distribution('changes').version
except Exception, e:
    VERSION = 'unknown'

def _get_git_revision(path):
    try:
        r = subprocess.check_output('git rev-parse HEAD', cwd=path, shell=True)
    except Exception:
        return None
    return r.strip()

def get_revision():
    """
    :returns: Revision number of this branch/checkout, if available. None if
        no revision number can be determined.
    """
    package_dir = os.path.dirname(__file__)
    checkout_dir = os.path.normpath(os.path.join(package_dir, os.pardir))
    path = os.path.join(checkout_dir, '.git')
    if os.path.exists(path):
        return _get_git_revision(path)
    return None

def get_version():
    base = VERSION
    if __build__:
        base = '%s (%s)' % (base, __build__)
    return base

__build__ = get_revision()
__docformat__ = 'restructuredtext en'

import os
import subprocess

try:
    VERSION = __import__('pkg_resources') \
        .get_distribution('changes').version
except Exception:
    VERSION = 'unknown'

def _get_git_revision(path):
    try:
        r = subprocess.check_output('git rev-parse HEAD', cwd=path, shell=True)
    except Exception:
        return None
    return r.strip()

def get_revision():
    """
    :returns: Revision number of this branch/checkout, if available. None if
        no revision number can be determined.
    """
    package_dir = os.path.dirname(__file__)
    checkout_dir = os.path.normpath(os.path.join(package_dir, os.pardir))
    path = os.path.join(checkout_dir, '.git')
    if os.path.exists(path):
        return _get_git_revision(path)
    return None

def get_version():
    base = VERSION
    if __build__:
        base = '%s (%s)' % (base, __build__)
    return base

__build__ = get_revision()
__docformat__ = 'restructuredtext en'

Update exception syntax to be py3 compat


#!/usr/bin/env python

#
# Curriculum Module Status Script
# - Run while the job is running
# - Run -outside- of the allocation
# - onramp_run_params.ini file is available in current working directory
#
import sys
import re

#
# Display any special message you want the user to see, or leave blank if nothing.
# Please restrict status messages to 1 line of text.
#

# Read in the output file
lines = [line.rstrip('\n') for line in open('onramp/output.txt')]

# If the file is empty then nothing to do
if len(lines) <= 0:
    sys.exit(0)

#print "Status) Number of Lines: %d" % len(lines)

# Find the last line of 'Increment value'
last_status = None
for line in lines:
    searchObj = re.search( r'Increment value (.*)', line)
    if searchObj:
        last_status = searchObj.group(1).strip()

if last_status is not None:
    print "%s" % last_status

#
# Exit successfully
#
sys.exit(0)

#!/usr/bin/env python

#
# Curriculum Module Status Script
# - Run while the job is running
# - Run -outside- of the allocation
# - onramp_run_params.ini file is available in current working directory
#
import sys
import re

#
# Display any special message you want the user to see, or leave blank if nothing.
# Please restrict status messages to 1 line of text.
#

# Read in the output file
lines = [line.rstrip('\n') for line in open('output.txt')]

# If the file is empty then nothing to do
if len(lines) <= 0:
    sys.exit(0)

#print "Status) Number of Lines: %d" % len(lines)

# Find the last line of 'Increment value'
last_status = None
for line in lines:
    searchObj = re.search( r'Increment value (.*)', line)
    if searchObj:
        last_status = searchObj.group(1).strip()

if last_status is not None:
    print "%s" % last_status

#
# Exit successfully
#
sys.exit(0)

Update the status.py to look for the output.txt in the new location


"""
Test the importing module.
This module relies heavily on an external service and requires
quite a bit of mocking.
"""
import json

from django.test import TestCase

from molo.core.tests.base import MoloTestCaseMixin
from molo.core.api import importers
from molo.core.api.tests import constants

class ArticleImportTestCase(MoloTestCaseMixin, TestCase):

    def setUp(self):
        self.mk_main()

    def test_importer_initializtion(self):
        content = json.dumps(constants.AVAILABLE_ARTICLES)
        importer = importers.ArticlePageImporter(content=content)

        self.assertEqual(importer.articles(), content["items"])

Write test for importer initialisation

Write test for importer initialisation


import argparse
import sys

from pipreq.command import Command

def create_parser():
    parser = argparse.ArgumentParser(
        description='Manage Python package requirements across multiple environments using '
                    'per-environment requirements files.')

    parser.add_argument('-g', '--generate', action='store_true', default=False,
                        help='Generate requirements files')
    parser.add_argument('-c', '--create', action='store_true', default=False,
                        help='Create or update rc file (requires list of packages)')
    parser.add_argument('-U', '--upgrade', action='store_true', default=False,
                        help='Upgrade packages (requires list of packages)')
    parser.add_argument('packages', nargs='?', type=argparse.FileType('r'), default=sys.stdin)

    return parser

def verify_args(args):
    if not args.create and not args.generate and not args.upgrade:
        return u'Must specify generate (-g) or create/upgrade (-[cu]) with packages'
    return None

def error(parser, message):
        parser.print_help()
        parser.exit(message="\nERROR: %s\n" % message)

def main():
    try:
        parser = create_parser()
        parsed_args = parser.parse_args()
        error_message = verify_args(parsed_args)
        if error_message:
            error(parser, error_message)
        command = Command(parsed_args, ".requirementsrc")
        command.run()
    except KeyboardInterrupt:
        sys.exit()

import argparse
import sys

from pipreq.command import Command

def create_parser():
    parser = argparse.ArgumentParser(
        description='Manage Python package requirements across multiple environments using '
                    'per-environment requirements files.')

    parser.add_argument('-g', '--generate', action='store_true', default=False,
                        help='Generate requirements files')
    parser.add_argument('-c', '--create', action='store_true', default=False,
                        help='Create or update rc file (requires list of packages)')
    parser.add_argument('-U', '--upgrade', action='store_true', default=False,
                        help='Upgrade packages (requires list of packages)')
    parser.add_argument('packages', nargs='?', type=argparse.FileType('r'), default=sys.stdin)

    return parser

def verify_args(args):
    if not args.create and not args.generate and not args.upgrade:
        return 'Must specify generate (-g) or create/upgrade (-[cu]) with packages'
    return None

def error(parser, message):
        parser.print_help()
        parser.exit(message="\nERROR: %s\n" % message)

def main():
    try:
        parser = create_parser()
        parsed_args = parser.parse_args()
        error_message = verify_args(parsed_args)
        if error_message:
            error(parser, error_message)
        command = Command(parsed_args, ".requirementsrc")
        command.run()
    except KeyboardInterrupt:
        sys.exit()

Remove unnecessary u on string

Remove unnecessary u on string


#!/usr/bin/env python
"""
Processing routines for the waveFlapper case.

"""

import foampy
import numpy as np
import matplotlib.pyplot as plt

width_2d = 0.1
width_3d = 3.66

def plot_force():
    """Plots the streamwise force on the paddle over time."""

def plot_moment():
    data = foampy.load_forces_moments()
    i = 10
    t = data["time"][i:]
    m = data["moment"]["pressure"]["z"] + data["moment"]["viscous"]["z"]
    m = m[i:]*width_3d/width_2d
    plt.figure()
    plt.plot(t, m)
    plt.xlabel("t (s)")
    plt.ylabel("Flapper moment (Nm)")
    print("Max moment from CFD =", m.max(), "Nm")
    print("Theoretical max moment (including inertia) =", 5500*3.3, "Nm")
    plt.show()

if __name__ == "__main__":
    plot_moment()

#!/usr/bin/env python
"""
Processing routines for the waveFlapper case.

"""

import foampy
import numpy as np
import matplotlib.pyplot as plt

width_2d = 0.1
width_3d = 3.66
m_paddle = 1270.0 # Paddle mass in kg, from OMB manual
h_piston = 3.3147
I_paddle = 1/3*m_paddle*h_piston**2

def plot_force():
    """Plots the streamwise force on the paddle over time."""

def plot_moment():
    data = foampy.load_forces_moments()
    i = 10
    t = data["time"][i:]
    m = data["moment"]["pressure"]["z"] + data["moment"]["viscous"]["z"]
    m = m[i:]*width_3d/width_2d
    period = 2.2
    omega = 2*np.pi/period
    theta = 0.048*np.sin(omega*t)
    theta_doubledot = -0.048*omega**2*np.sin(omega*t)
    m_inertial = I_paddle*theta_doubledot
    m += m_inertial
    plt.figure()
    plt.plot(t, m)
    plt.xlabel("t (s)")
    plt.ylabel("Flapper moment (Nm)")
    print("Max moment from CFD =", m.max(), "Nm")
    print("Theoretical max moment (including inertia) =", 5500*3.3, "Nm")
    plt.show()

if __name__ == "__main__":
    plot_moment()

Add paddle inertia to calculations

Add paddle inertia to calculations


from Crypto.PublicKey import RSA

from django.conf import settings
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = 'Randomly generate a new RSA key for the OpenID server'

    def handle(self, *args, **options):
        try:
            key = RSA.generate(1024)
            file_path = settings.BASE_DIR + '/OIDC_RSA_KEY.pem'
            with open(file_path, 'w') as f:
                f.write(key.exportKey('PEM'))
            self.stdout.write('RSA key successfully created at: ' + file_path)
        except Exception as e:
            self.stdout.write('Something goes wrong: {0}'.format(e))

from Crypto.PublicKey import RSA

from django.conf import settings
from django.core.management.base import BaseCommand

class Command(BaseCommand):
    help = 'Randomly generate a new RSA key for the OpenID server'

    def handle(self, *args, **options):
        try:
            key = RSA.generate(1024)
            file_path = settings.BASE_DIR + '/OIDC_RSA_KEY.pem'
            with open(file_path, 'wb') as f:
                f.write(key.exportKey('PEM'))
            self.stdout.write('RSA key successfully created at: ' + file_path)
        except Exception as e:
            self.stdout.write('Something goes wrong: {0}'.format(e))

Append binary file mode to write RSA exported key needed by Python 3


#!/usr/bin/python
#
# SCRIPT FOR POPULATING TEST SOLR SERVER CORE WITH TEST DOCUMENTS
#
# Usage: python setup-test-docs.py <Solr Endpoint Url>
#
# Solr endpoint URL should be in the form:
# https://example.com/solr/<core-name>/
#
# .txt files in the directory ./txt/ will be committed to user-provided Solr
# core matching the name <core-name>.

import os
from os import listdir
from os.path import isfile, join
import json
import sys

arguments = sys.argv

solrApiUrl = arguments[1]

filePaths = [f for f in listdir('txt') if isfile(join('txt', f))]

TEMPLATE = """
{
    "add": {
        "doc":
            {"title":"%s", "body": %s},
        "boost":1.0,
        "overwrite":true,
        "commitWithin":1000
    }
}
"""

headers = {'Content-type': 'application/json'}

for i, path in enumerate(filePaths):
    print str(i) + '\tProcessing ' + path
    f = open('txt/' + path)
    text = f.read()

    commandJson = TEMPLATE % (path.replace('.txt', ''), json.dumps(text))

    os.system("curl " + solrApiUrl + "update?commit=true -H 'Content-type:application/json' -d '%s'" % commandJson)
    print '\nDone.\n----------------------------------'

#!/usr/bin/python
#
# SCRIPT FOR POPULATING TEST SOLR SERVER CORE WITH TEST DOCUMENTS
#
# Usage: python setup-test-docs.py <Solr Endpoint Url>
#
# Solr endpoint URL should be in the form:
# https://example.com/solr/<core-name>/
#
# .txt files in the directory ./txt/ will be committed to user-provided Solr
# core matching the name <core-name>.

import os
from os import listdir
from os.path import isfile, join
import json
import sys

TEST_DOC_DIR = 'test_docs'

arguments = sys.argv

solrApiUrl = arguments[1]

filePaths = [f for f in listdir(TEST_DOC_DIR) if isfile(join(TEST_DOC_DIR, f))]

TEMPLATE = """
{
    "add": {
        "doc":
            {"title":"%s", "body": %s},
        "boost":1.0,
        "overwrite":true,
        "commitWithin":1000
    }
}
"""

headers = {'Content-type': 'application/json'}

for i, path in enumerate(filePaths):
    print str(i) + '\tProcessing ' + path
    f = open(TEST_DOC_DIR + '/' + path)
    text = f.read()

    commandJson = TEMPLATE % (path.replace('.txt', ''), json.dumps(text))

    os.system("curl " + solrApiUrl + "update?commit=true -H 'Content-type:application/json' -d '%s'" % commandJson)
    print '\nDone.\n----------------------------------'

Use test_docs as directory for test documents for solr server


"""
A utility class which wraps the RateLimitMixin 3rd party class to do bad request counting
which can be used for rate limiting
"""
from ratelimitbackend.backends import RateLimitMixin
from django.conf import settings

if settings.FEATURES.get('EDRAAK_RATELIMIT_APP', False):
    from edraak_ratelimit.backends import EdraakRateLimitMixin
    RateLimitMixin = EdraakRateLimitMixin

class BadRequestRateLimiter(RateLimitMixin):
    """
    Use the 3rd party RateLimitMixin to help do rate limiting on the Password Reset flows
    """

    def is_rate_limit_exceeded(self, request):
        """
        Returns if the client has been rated limited
        """
        counts = self.get_counters(request)
        is_exceeded = sum(counts.values()) >= self.requests

        if is_exceeded:
            self.db_log_failed_attempt(request)

        return is_exceeded

    def tick_bad_request_counter(self, request):
        """
        Ticks any counters used to compute when rate limt has been reached
        """
        self.cache_incr(self.get_cache_key(request))

"""
A utility class which wraps the RateLimitMixin 3rd party class to do bad request counting
which can be used for rate limiting
"""
from ratelimitbackend.backends import RateLimitMixin
from django.conf import settings

if settings.FEATURES.get('EDRAAK_RATELIMIT_APP', False):
    from edraak_ratelimit.backends import EdraakRateLimitMixin
    RateLimitMixin = EdraakRateLimitMixin

class BadRequestRateLimiter(RateLimitMixin):
    """
    Use the 3rd party RateLimitMixin to help do rate limiting on the Password Reset flows
    """

    def is_rate_limit_exceeded(self, request):
        """
        Returns if the client has been rated limited
        """
        counts = self.get_counters(request)
        is_exceeded = sum(counts.values()) >= self.requests

        if is_exceeded and settings.FEATURES.get('EDRAAK_RATELIMIT_APP', False):
            self.db_log_failed_attempt(request)

        return is_exceeded

    def tick_bad_request_counter(self, request):
        """
        Ticks any counters used to compute when rate limt has been reached
        """
        self.cache_incr(self.get_cache_key(request))

Fix object has no db_log_failed_attempt

Fix object has no db_log_failed_attempt


from api.scrapers.item import scrape_item_by_id
from api import app

from flask.json import loads

import unittest

app.config['TESTING'] = True

class ScrapeItem(unittest.TestCase):

    def test_scrape_item_by_id(self):
        item = scrape_item_by_id('d19447e548d')
        self.assertEqual('d19447e548d', item.lodestone_id)
        self.assertEqual('Thyrus Zenith', item.name)
        self.assertEqual('Two-handed Conjurer\'s Arm', item.type)
        self.assertEqual(90, item.ilvl)

    def test_item_json(self):
        with app.test_client() as client:
            response = client.get('/scrape/item/cada9ec7074')
            self.assertEqual(response.status_code, 200)

            json = loads(response.data)
            self.assertEqual(json, {
                "id": "cada9ec7074",
                "ilvl": 110,
                "name": "Arachne Robe",
                "type": "Body"
            })

    def test_invalid_lodestone_id(self):
        with app.test_client() as client:
            response = client.post('/scrape/item/23fh032hf0oi1so3a012r1')
            self.assertEqual(response.status_code, 405)

from api.scrapers.item import scrape_item_by_id
from api import app, db

from flask.json import loads

import unittest

app.config['TESTING'] = True
db.create_all()

class ScrapeItem(unittest.TestCase):

    def test_scrape_item_by_id(self):
        item = scrape_item_by_id('d19447e548d')
        self.assertEqual('d19447e548d', item.lodestone_id)
        self.assertEqual('Thyrus Zenith', item.name)
        self.assertEqual('Two-handed Conjurer\'s Arm', item.type)
        self.assertEqual(90, item.ilvl)

    def test_item_json(self):
        with app.test_client() as client:
            response = client.get('/scrape/item/cada9ec7074')
            self.assertEqual(response.status_code, 200)

            json = loads(response.data)
            self.assertEqual(json, {
                "id": "cada9ec7074",
                "ilvl": 110,
                "name": "Arachne Robe",
                "type": "Body"
            })

    def test_invalid_lodestone_id(self):
        with app.test_client() as client:
            response = client.post('/scrape/item/23fh032hf0oi1so3a012r1')
            self.assertEqual(response.status_code, 405)

Create tables in database before running tests


from django.db import transaction, connection

class TransactionMiddleware(object):
    def process_job(self, job):
        if not connection.in_atomic_block:
            transaction.set_autocommit(False)

    def process_result(self, job, result, duration):
        if not connection.in_atomic_block:
            transaction.commit()

    def process_exception(self, job, time_taken, *exc_info):
        if not connection.in_atomic_block:
            transaction.rollback()

# Legacy
if not hasattr(connection, 'in_atomic_block'):
    class TransactionMiddleware(object):
        def process_job(self, job):
            transaction.enter_transaction_management()
            transaction.managed(True)

        def process_result(self, job, result, duration):
            if not transaction.is_managed():
                return
            if transaction.is_dirty():
                transaction.commit()
            transaction.leave_transaction_management()

        def process_exception(self, job, time_taken, *exc_info):
            if transaction.is_dirty():
                transaction.rollback()
            transaction.leave_transaction_management()

Add a legacy version for older versions of Django.

Signed-off-by: Chris Lamb <711c73f64afdce07b7e38039a96d2224209e9a6c@thread.com>


"""Django haystack `SearchIndex` module."""
from pages.models import Page, Content

from haystack.indexes import SearchIndex, CharField, DateTimeField
from haystack import site

import datetime

class PageIndex(SearchIndex):
    """Search index for pages content."""
    text = CharField(document=True, use_template=True)
    title = CharField(model_attr='title')
    publication_date = DateTimeField(model_attr='publication_date')

    def get_queryset(self):
        """Used when the entire index for model is updated."""
        return Page.objects.published()

site.register(Page, PageIndex)

"""Django haystack `SearchIndex` module."""
from pages.models import Page, Content

from haystack.indexes import SearchIndex, CharField, DateTimeField
from haystack import site

import datetime

class PageIndex(SearchIndex):
    """Search index for pages content."""
    text = CharField(document=True, use_template=True)
    title = CharField(model_attr='title')
    url = CharField(model_attr='get_absolute_url')
    publication_date = DateTimeField(model_attr='publication_date')

    def get_queryset(self):
        """Used when the entire index for model is updated."""
        return Page.objects.published()

site.register(Page, PageIndex)

Add a url attribute to the SearchIndex for pages.

This is useful when displaying a list of search results because we
can create a link to the result without having to hit the database
for every object in the result list.


import asyncio
from unittest import mock

import aiohttp
import pytest

from bottery.app import App

def test_app_session():
    app = App()
    assert isinstance(app.session, aiohttp.ClientSession)

def test_app_already_configured_session():
    app = App()
    app._session = 'session'
    assert app.session == 'session'

def test_app_loop():
    app = App()
    assert isinstance(app.loop, asyncio.AbstractEventLoop)

def test_app_already_configured_loop():
    app = App()
    app._loop = 'loop'
    assert app.loop == 'loop'

@mock.patch('bottery.app.settings')
def test_app_configure_without_platforms(mocked_settings):
    """Should raise Exception if no platform was found at settings"""

    mocked_settings.PLATFORMS = {}
    app = App()
    with pytest.raises(Exception):
        app.configure_platforms()

import asyncio
import sys
from unittest import mock

import aiohttp
import pytest

from bottery.app import App

@pytest.fixture
def mocked_engine():
    mocked_engine_module = mock.MagicMock()
    mocked_engine_instance = mocked_engine_module.engine.return_value
    mocked_engine_instance.tasks.return_value = [(mock.MagicMock(), )]
    sys.modules['tests.fake_engine'] = mocked_engine_module

    yield {
        'module': mocked_engine_module,
        'instance': mocked_engine_instance
    }

    del sys.modules['tests.fake_engine']

def test_app_session():
    app = App()
    assert isinstance(app.session, aiohttp.ClientSession)

def test_app_already_configured_session():
    app = App()
    app._session = 'session'
    assert app.session == 'session'

def test_app_loop():
    app = App()
    assert isinstance(app.loop, asyncio.AbstractEventLoop)

def test_app_already_configured_loop():
    app = App()
    app._loop = 'loop'
    assert app.loop == 'loop'

@mock.patch('bottery.app.settings')
def test_app_configure_without_platforms(mocked_settings):
    """Should raise Exception if no platform was found at settings"""

    mocked_settings.PLATFORMS = {}
    app = App()
    with pytest.raises(Exception):
        app.configure_platforms()

@mock.patch('bottery.app.settings')
def test_app_configure_with_platforms(mocked_settings, mocked_engine):
    """Should call the platform interface methods"""

    mocked_settings.PLATFORMS = {
        'test': {
            'ENGINE': 'tests.fake_engine',
            'OPTIONS': {
                'token': 'should-be-a-valid-token'
            }
        }
    }

    app = App()
    app.configure_platforms()

    mocked_engine['module'].engine.assert_called_with(
        session=app.session,
        token='should-be-a-valid-token'
    )
    mocked_engine['instance'].configure.assert_called_with()
    mocked_engine['instance'].tasks.assert_called_with()

Increase the code coverage of App.configure_platforms method


from __future__ import unicode_literals
from datetime import datetime
from .logic_adapter import LogicAdapter

class TimeLogicAdapter(LogicAdapter):
    """
    The TimeLogicAdapter returns the current time.
    """

    def __init__(self, **kwargs):
        super(TimeLogicAdapter, self).__init__(**kwargs)
        from textblob.classifiers import NaiveBayesClassifier

        training_data = [
            ('what time is it', 1),
            ('do you know the time', 1),
            ('do you know what time it is', 1),
            ('what is the time', 1),
            ('it is time to go to sleep', 0),
            ('what is your favorite color', 0),
            ('i had a great time', 0),
            ('what is', 0)
        ]

        self.classifier = NaiveBayesClassifier(training_data)

    def process(self, statement):
        from chatterbot.conversation import Statement

        now = datetime.now()

        confidence = self.classifier.classify(statement.text.lower())
        response = Statement('The current time is ' + now.strftime('%I:%M %p'))

        return confidence, response

from __future__ import unicode_literals
from datetime import datetime
from .logic_adapter import LogicAdapter

class TimeLogicAdapter(LogicAdapter):
    """
    The TimeLogicAdapter returns the current time.
    """

    def __init__(self, **kwargs):
        super(TimeLogicAdapter, self).__init__(**kwargs)
        from nltk import NaiveBayesClassifier

        self.positive = [
            'what time is it',
            'do you know the time',
            'do you know what time it is',
            'what is the time'
        ]

        self.negative = [
            'it is time to go to sleep',
            'what is your favorite color',
            'i had a great time',
            'what is'
        ]

        labeled_data = (
            [(name, 0) for name in self.negative] +
            [(name, 1) for name in self.positive]
        )

        # train_set = apply_features(self.time_question_features, training_data)
        train_set = [(self.time_question_features(n), text) for (n, text) in labeled_data]

        self.classifier = NaiveBayesClassifier.train(train_set)

    def time_question_features(self, text):
        """
        Provide an analysis of significan features in the string.
        """
        features = {}

        all_words = " ".join(self.positive + self.negative).split()

        for word in text.split():
            features['contains({})'.format(word)] = (word in all_words)

        for letter in 'abcdefghijklmnopqrstuvwxyz':
            features['count({})'.format(letter)] = text.lower().count(letter)
            features['has({})'.format(letter)] = (letter in text.lower())

        return features

    def process(self, statement):
        from chatterbot.conversation import Statement

        now = datetime.now()

        time_features = self.time_question_features(statement.text.lower())
        confidence = self.classifier.classify(time_features)
        response = Statement('The current time is ' + now.strftime('%I:%M %p'))

        return confidence, response

Remove textblob dependency in time logic adapter


# -*- coding: utf-8 -*-
# © 2011 Akretion Sébastien BEAU <sebastien.beau@akretion.com>
# © 2013 Camptocamp SA (author: Guewen Baconnier)
# © 2016 Sodexis
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import models, fields

class AccountInvoice(models.Model):
    _inherit = "account.invoice"

    workflow_process_id = fields.Many2one(
        comodel_name='sale.workflow.process',
        string='Sale Workflow Process'
    )

# -*- coding: utf-8 -*-
# © 2011 Akretion Sébastien BEAU <sebastien.beau@akretion.com>
# © 2013 Camptocamp SA (author: Guewen Baconnier)
# © 2016 Sodexis
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
from odoo import models, fields

class AccountInvoice(models.Model):
    _inherit = "account.invoice"

    workflow_process_id = fields.Many2one(
        comodel_name='sale.workflow.process',
        string='Sale Workflow Process',
        copy=False,
    )

[FIX] Fix issue on account.invoice about workflow_process_id: if a user duplicate an invoice, it copy also the workflow and validations (the reason of bugs)