<NME> aggregator.py
<BEF> import argparse
import mongoengine
import redis
import sys

import rmc.models as m
import rmc.shared.constants as c
import rmc.shared.facebook as facebook
import rmc.shared.util as rmc_util
import rmc.data.crawler as rmc_crawler
import rmc.data.processor as rmc_processor

# TODO(mack): remove duplication of fields throughout code
# TODO(mack): deprecate overall rating

r = redis.StrictRedis(host=c.REDIS_HOST, port=c.REDIS_PORT, db=c.REDIS_DB)

PROFESSOR_RATING_FIELDS = [
    'easiness',
    'clarity',
    'passion',
]

COURSE_RATING_FIELDS = [
    'easiness',
    'interest',
    'usefulness',
    'overall',
]


def increment_ratings(courses, get_rating_fn, get_fields_fn, ucs):
    for uc in ucs:
        ratings = get_rating_fn(courses, uc)
        if not ratings:
            continue
        for field_key, field_value in get_fields_fn(uc):
            if field_value is not None:
                ratings[field_key].add_rating(field_value)


def increment_aggregate_ratings(courses, get_rating_fn, get_fields_fn, ucs):
    for uc in ucs:
        ratings = get_rating_fn(courses, uc)
        if not ratings:
            continue
        for field_key, field_value in get_fields_fn(uc):
            if field_value is not None:
                ratings[field_key].add_aggregate_rating(field_value)


def update_mongo_course_rating():
    # course => ratings
    def get_rating_fn(courses, uc):
        if uc.course_id not in courses:
            obj = {}
            for field in COURSE_RATING_FIELDS:
                obj[field] = m.AggregateRating()
            courses[uc.course_id] = obj
        return courses[uc.course_id]

    def get_fields_fn(uc):
        easiness = uc.course_review.easiness
        interest = uc.course_review.interest
        usefulness = uc.course_review.usefulness
        if easiness and interest:
            overall = (easiness + interest) / 2
        elif easiness:
            overall = easiness
        else:
            overall = interest

        return [
        interest = uc.interest

        def calculate_overall_rating(e, i):
            return (e.count * e.rating + i.count * i.rating) / (e.count + i.count)

        # heuristic for getting the overall rating:
        # 1. the count will max of the count for each attribute
        easiness = uc.easiness
        interest = uc.interest
        # TODO(mack): add usefulness metric

        def calculate_overall_rating(e, i):
            return ((e.count * e.rating + i.count * i.rating) /
                     max(1, (e.count + i.count)))

        # heuristic for getting the overall rating:
        # 1. the count will max of the count for each attribute
        # 2. the rating will be average
        overall = m.AggregateRating(
            count=max(easiness.count, interest.count),
            rating=calculate_overall_rating(easiness, interest),
        )

        return [
            ('easiness', easiness),
            ('interest', interest),
            ('overall', overall),
        ]

    courses = {}
    args = [courses, get_rating_fn]
    menlo_ucs = m.MenloCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)
    flow_ucs = m.UserCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)

    increment_ratings(*(args + [get_fields_fn, menlo_ucs]))
    increment_ratings(*(args + [get_fields_fn, flow_ucs]))

    count = [0]

    def set_course_ratings_in_mongo(courses):
        for course_id, ratings in courses.items():
            course = m.Course.objects.with_id(course_id)
            if not course:
                print 'could not find course %s in mongo' % course_id
                continue

            course.easiness = ratings['easiness']
            course.interest = ratings['interest']
            course.usefulness = ratings['usefulness']
            course.overall = ratings['overall']

            course.save()
            count[0] += 1

    set_course_ratings_in_mongo(courses)
    print 'saved ratings for %d courses in mongodb' % count[0]


def update_mongo_course_professors():

    count = 0
    for course in m.Course.objects.only('professor_ids'):
        def get_professor_ids(course, coll):
            course_prof_ids_only = (coll.objects(course_id=course.id)
                                        .only('professor_id'))
            return set(
                [x.professor_id for x in course_prof_ids_only
                 if x.professor_id]
            )
        professor_ids = get_professor_ids(course, m.UserCourse).union(
                get_professor_ids(course, m.MenloCourse))
        # TODO(mack): Looks like add_to_set doesn't validate that each item
        # in the list meets the schema since it seemed to be letting me
        # writing lists that contained None. Investigate if this is what it
        # is doing.
        course.update(add_to_set__professor_ids=list(professor_ids))
        count += 1

    print 'added professors for %d courses in mongodb' % count


def update_redis_course_professor_rating():
    # course => professors => ratings
    def get_rating_fn(courses, uc):
        if uc.professor_id is None:
            return None

        if uc.course_id not in courses:
            courses[uc.course_id] = {}
        professors = courses[uc.course_id]

        if uc.professor_id not in professors:
            obj = {}
            for field in PROFESSOR_RATING_FIELDS:
                obj[field] = m.AggregateRating()
            professors[uc.professor_id] = obj
        return professors[uc.professor_id]

    def get_fields_fn(uc):
        return [
            ('easiness', uc.course_review.easiness),
            ('clarity', uc.professor_review.clarity),
            ('passion', uc.professor_review.passion),
        ]

    def get_aggregate_fields_fn(uc):
        return [
            ('easiness', uc.easiness),
            ('clarity', uc.clarity),
            ('passion', uc.passion),
        ]

    courses = {}
    args = [courses, get_rating_fn]
    menlo_ucs = m.MenloCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)
    flow_ucs = m.UserCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)

    increment_ratings(*(args + [get_fields_fn, menlo_ucs]))
    increment_ratings(*(args + [get_fields_fn, flow_ucs]))

    count = [0]

    def set_course_professor_ratings_in_redis(courses):
        for course_id, professors in courses.items():
            for professor_id, ratings in professors.items():
                if professor_id is None:
                    continue
                professor = m.Professor.objects.with_id(professor_id)
                if not professor:
                    continue
                for rating_type, aggregate_rating in ratings.items():
                    professor.set_course_rating_in_redis(
                            course_id, rating_type, aggregate_rating)
                    count[0] += 1

    set_course_professor_ratings_in_redis(courses)
    print 'set %d course professor rating keys in redis' % count[0]


def update_all_fb_friend_list():
    for user in m.User.objects():
        # TODO(Sandy): Batch requests for performance
        if user.fbid and not user.is_fb_token_expired:
            try:
                user.update_fb_friends(
                        facebook.get_friend_list(user.fb_access_token))
                user.save()
            except facebook.FacebookOAuthException as e:
                user.fb_access_token_invalid = True
                user.save()
            except Exception as e:
                print "get_friend_list failed for %s with: %s" % (user.id,
                e.message)


# TODO(mack): test it when we get data to test with
# TODO(mack): currently sort of duplicate logic in
# User.cache_mutual_course_ids()
def update_redis_friend_mutual_courses():
    # TODO(Sandy): Use friend real time updates after it. There's a fb updates
    # branch for this, pending on:
    # https://developers.facebook.com/bugs/374296595988186?browse=search_50990ddb8a19d9316431973
    # Rate limit is 600 calls / 600 seconds / token:
    # http://stackoverflow.com/questions/8805316/facebook-graph-api-rate-limit-and-batch-requests
    update_all_fb_friend_list()

    courses_by_user = {}
    for user in m.User.objects.only('friend_ids', 'course_history'):
        friend_ids = [str(friend_id) for friend_id in user.friend_ids]
        ucs = (m.UserCourse.objects(id__in=user.course_history)
                                  .only('course_id'))
        course_ids = [uc.course_id for uc in ucs]
        courses_by_user[str(user.id)] = [friend_ids, set(course_ids)]

    count = 0
    user_pair = set()
    for user_id, (friend_ids, courses) in courses_by_user.iteritems():
        for friend_id in friend_ids:
            if user_id < friend_id:
                first_id = user_id
                second_id = friend_id
            else:
                first_id = friend_id
                second_id = user_id
            if (first_id, second_id) in user_pair:
                continue

            friend_courses = courses_by_user[friend_id][1]
            mutual_courses = courses.intersection(friend_courses)
            if mutual_courses:
                count += 1
                redis_key = m.User.cls_mutual_courses_redis_key(
                    first_id, second_id)
                r.sadd(redis_key, *list(mutual_courses))
            user_pair.add((first_id, second_id))

    print 'set %d friend pair keys in redis' % count


def update_mongo_points():
    total_points = 0

    num_course_comments = 0
    num_course_ratings = 0
    num_course_shares = 0
    num_professor_comments = 0
    num_professor_ratings = 0
    num_professor_shares = 0
    num_invites = 0

    for user in m.User.objects.only(
            'num_invites', 'course_history', 'num_points'):
        num_points = 0
        if user.num_invites:
            num_points += m.PointSource.FIRST_INVITE
            num_invites += 1
        for uc in m.UserCourse.objects(id__in=user.course_history):
            num_points += uc.num_points

            if uc.course_review.has_commented:
                num_course_comments += 1
            if uc.course_review.has_been_rated:
                num_course_ratings += 1
            if uc.course_review.has_shared:
                num_course_shares += 1

            if uc.professor_review.has_commented:
                num_professor_comments += 1
            if uc.professor_review.has_been_rated:
                num_professor_ratings += 1
            if uc.professor_review.has_shared:
                num_professor_shares += 1

        user.update(set__num_points=num_points)
        total_points += num_points

    r.set('total_points', total_points)

    print ' ===update_mongo_points ==='
    print 'num_course_comments', num_course_comments
    print 'num_course_ratings', num_course_ratings
    print 'num_course_shares', num_course_shares
    print 'num_professor_comments', num_professor_comments
    print 'num_professor_ratings', num_professor_ratings
    print 'num_professor_shares', num_professor_shares
    print 'num_invites', num_invites


def update_exam_schedule():
    # Crawl data and store on disk
    rmc_crawler.get_opendata_exam_schedule()

    # Process the data on disk
    errors = rmc_processor.import_opendata_exam_schedules()

    print "%d exam schedule items found" % m.Exam.objects().count()
    print "%d exam schedule items skipped" % len(errors)


def update_sections():
    # Fetch data from OpenData API and cache to files.
    rmc_crawler.get_opendata_sections()

    # Import from files to DB.
    rmc_processor.import_opendata_sections()

    # Send push notifications about seat openings.
    num_sent = m.GcmCourseAlert.send_eligible_alerts()
    num_expired = m.GcmCourseAlert.delete_expired()
    print 'Sent %s push notifications and expired %s' % (num_sent, num_expired)


def update_courses():
    # First get an up to date list of departments and write to a text file
    print "Fetching departments"
    rmc_crawler.get_departments()

    # Import any departments we don't already have into Mongo
    print "Loading departments into Mongo"
    rmc_processor.import_departments()

    # Hit the endpoints of the OpenData API for each department
    print "Fetching courses"
    rmc_crawler.get_opendata2_courses()

    # Load the data into Mongo
    print "Loading courses into Mongo"
    rmc_processor.import_courses()


def update_professors_departments():
    """Update the departments_taught field for each professor in Mongo"""
    for prof in m.Professor.objects():
        prof.departments_taught = prof.get_departments_taught()
        prof.save()


def update_scholarships():
    """Update the scholarships available in Mongo"""
    print "Fetching scholarships"
    rmc_crawler.get_scholarships()

    print "Loading scholarships into Mongo"
    rmc_processor.import_scholarships()


if __name__ == '__main__':
    mongoengine.connect(c.MONGO_DB_RMC)

    parser = argparse.ArgumentParser()
    mode_mapping = {
        'redis_course_professor_rating': update_redis_course_professor_rating,
        'redis_friend_mutual_courses': update_redis_friend_mutual_courses,
        'mongo_course_rating': update_mongo_course_rating,
        'mongo_course_professors': update_mongo_course_professors,
        'mongo_points': update_mongo_points,
        'exam_schedule': update_exam_schedule,
        'sections': update_sections,
        'courses': update_courses,
        'prof_departments': update_professors_departments,
        'scholarships': update_scholarships
    }
    parser.add_argument('mode',
            help='one of %s' % ','.join(mode_mapping.keys() + ['daily']))
    args = parser.parse_args()

    if args.mode == 'daily':
        daily_functions = [
            update_redis_course_professor_rating,
            update_redis_friend_mutual_courses,
            update_mongo_course_rating,
            update_mongo_course_professors,
            update_mongo_points,
            update_exam_schedule,
            update_professors_departments
        ]
        for func in daily_functions:
            try:
                func()
            except Exception as exp:
                print "aggregator.py: function %s threw an exception" % (func)
                print exp
    elif args.mode in mode_mapping:
        func = mode_mapping[args.mode]
        func()
    else:
        sys.exit('The mode %s is not supported' % args.mode)

<MSG> Guarded against division by 0 in aggregator.py

<DFF> @@ -74,7 +74,7 @@ def import_mongo_course_rating():
         interest = uc.interest
 
         def calculate_overall_rating(e, i):
-            return (e.count * e.rating + i.count * i.rating) / (e.count + i.count)
+            return (e.count * e.rating + i.count * i.rating) / max(1, (e.count + i.count))
 
         # heuristic for getting the overall rating:
         # 1. the count will max of the count for each attribute

{'commit': 'diff --git a/data/aggregator.py b/data/aggregator.py\nindex f1811a09..6039e333 100644\n--- a/data/aggregator.py\n+++ b/data/aggregator.py\n@@ -74,7 +74,7 @@ def import_mongo_course_rating():\n         interest = uc.interest\n \n         def calculate_overall_rating(e, i):\n-            return (e.count * e.rating + i.count * i.rating) / (e.count + i.count)\n+            return (e.count * e.rating + i.count * i.rating) / max(1, (e.count + i.count))\n \n         # heuristic for getting the overall rating:\n         # 1. the count will max of the count for each attribute\n', 'commit_subject': 'Guarded against division by 0 in aggregator.py', 'repo_name': 'UWFlow/rmc', 'committer': {'name': 'Jiayi Sandy Wu', 'email': 'e5f17d009649570b6fb0d9a3cdffc384771c1a3c@gmail.com', 'time_sec': '1348568525', 'tz_offset': '-240', 'date': {'seconds': '1348568525'}}, 'commit_tree': '1ca15222649f7c26478bccdf29256b66b594b1d2', 'commit_message': 'Guarded against division by 0 in aggregator.py\n', 'language_name': 'Shell', 'license': 'mit', 'hunks': '@@ -74,7 +74,7 @@ def import_mongo_course_rating():\n         interest = uc.interest\n \n         def calculate_overall_rating(e, i):\n-            return (e.count * e.rating + i.count * i.rating) / (e.count + i.count)\n+            return (e.count * e.rating + i.count * i.rating) / max(1, (e.count + i.count))\n \n         # heuristic for getting the overall rating:\n         # 1. the count will max of the count for each attribute\n', 'addition_count': 1, 'deletion_count': 1, 'src_file': 'a/data/aggregator.py', 'tgt_file': 'b/data/aggregator.py', 'file_extension': '.py', 'before_file': 'import argparse\nimport mongoengine\nimport redis\nimport sys\n\nimport rmc.models as m\nimport rmc.shared.constants as c\nimport rmc.shared.facebook as facebook\nimport rmc.shared.util as rmc_util\nimport rmc.data.crawler as rmc_crawler\nimport rmc.data.processor as rmc_processor\n\n# TODO(mack): remove duplication of fields throughout code\n# TODO(mack): deprecate overall rating\n\nr = redis.StrictRedis(host=c.REDIS_HOST, port=c.REDIS_PORT, db=c.REDIS_DB)\n\nPROFESSOR_RATING_FIELDS = [\n    \'easiness\',\n    \'clarity\',\n    \'passion\',\n]\n\nCOURSE_RATING_FIELDS = [\n    \'easiness\',\n    \'interest\',\n    \'usefulness\',\n    \'overall\',\n]\n\n\ndef increment_ratings(courses, get_rating_fn, get_fields_fn, ucs):\n    for uc in ucs:\n        ratings = get_rating_fn(courses, uc)\n        if not ratings:\n            continue\n        for field_key, field_value in get_fields_fn(uc):\n            if field_value is not None:\n                ratings[field_key].add_rating(field_value)\n\n\ndef increment_aggregate_ratings(courses, get_rating_fn, get_fields_fn, ucs):\n    for uc in ucs:\n        ratings = get_rating_fn(courses, uc)\n        if not ratings:\n            continue\n        for field_key, field_value in get_fields_fn(uc):\n            if field_value is not None:\n                ratings[field_key].add_aggregate_rating(field_value)\n\n\ndef update_mongo_course_rating():\n    # course => ratings\n    def get_rating_fn(courses, uc):\n        if uc.course_id not in courses:\n            obj = {}\n            for field in COURSE_RATING_FIELDS:\n                obj[field] = m.AggregateRating()\n            courses[uc.course_id] = obj\n        return courses[uc.course_id]\n\n    def get_fields_fn(uc):\n        easiness = uc.course_review.easiness\n        interest = uc.course_review.interest\n        usefulness = uc.course_review.usefulness\n        if easiness and interest:\n            overall = (easiness + interest) / 2\n        elif easiness:\n            overall = easiness\n        else:\n            overall = interest\n\n        return [\n        interest = uc.interest\n\n        def calculate_overall_rating(e, i):\n            return (e.count * e.rating + i.count * i.rating) / (e.count + i.count)\n\n        # heuristic for getting the overall rating:\n        # 1. the count will max of the count for each attribute\n        easiness = uc.easiness\n        interest = uc.interest\n        # TODO(mack): add usefulness metric\n\n        def calculate_overall_rating(e, i):\n            return ((e.count * e.rating + i.count * i.rating) /\n                     max(1, (e.count + i.count)))\n\n        # heuristic for getting the overall rating:\n        # 1. the count will max of the count for each attribute\n        # 2. the rating will be average\n        overall = m.AggregateRating(\n            count=max(easiness.count, interest.count),\n            rating=calculate_overall_rating(easiness, interest),\n        )\n\n        return [\n            (\'easiness\', easiness),\n            (\'interest\', interest),\n            (\'overall\', overall),\n        ]\n\n    courses = {}\n    args = [courses, get_rating_fn]\n    menlo_ucs = m.MenloCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)\n    flow_ucs = m.UserCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)\n\n    increment_ratings(*(args + [get_fields_fn, menlo_ucs]))\n    increment_ratings(*(args + [get_fields_fn, flow_ucs]))\n\n    count = [0]\n\n    def set_course_ratings_in_mongo(courses):\n        for course_id, ratings in courses.items():\n            course = m.Course.objects.with_id(course_id)\n            if not course:\n                print \'could not find course %s in mongo\' % course_id\n                continue\n\n            course.easiness = ratings[\'easiness\']\n            course.interest = ratings[\'interest\']\n            course.usefulness = ratings[\'usefulness\']\n            course.overall = ratings[\'overall\']\n\n            course.save()\n            count[0] += 1\n\n    set_course_ratings_in_mongo(courses)\n    print \'saved ratings for %d courses in mongodb\' % count[0]\n\n\ndef update_mongo_course_professors():\n\n    count = 0\n    for course in m.Course.objects.only(\'professor_ids\'):\n        def get_professor_ids(course, coll):\n            course_prof_ids_only = (coll.objects(course_id=course.id)\n                                        .only(\'professor_id\'))\n            return set(\n                [x.professor_id for x in course_prof_ids_only\n                 if x.professor_id]\n            )\n        professor_ids = get_professor_ids(course, m.UserCourse).union(\n                get_professor_ids(course, m.MenloCourse))\n        # TODO(mack): Looks like add_to_set doesn\'t validate that each item\n        # in the list meets the schema since it seemed to be letting me\n        # writing lists that contained None. Investigate if this is what it\n        # is doing.\n        course.update(add_to_set__professor_ids=list(professor_ids))\n        count += 1\n\n    print \'added professors for %d courses in mongodb\' % count\n\n\ndef update_redis_course_professor_rating():\n    # course => professors => ratings\n    def get_rating_fn(courses, uc):\n        if uc.professor_id is None:\n            return None\n\n        if uc.course_id not in courses:\n            courses[uc.course_id] = {}\n        professors = courses[uc.course_id]\n\n        if uc.professor_id not in professors:\n            obj = {}\n            for field in PROFESSOR_RATING_FIELDS:\n                obj[field] = m.AggregateRating()\n            professors[uc.professor_id] = obj\n        return professors[uc.professor_id]\n\n    def get_fields_fn(uc):\n        return [\n            (\'easiness\', uc.course_review.easiness),\n            (\'clarity\', uc.professor_review.clarity),\n            (\'passion\', uc.professor_review.passion),\n        ]\n\n    def get_aggregate_fields_fn(uc):\n        return [\n            (\'easiness\', uc.easiness),\n            (\'clarity\', uc.clarity),\n            (\'passion\', uc.passion),\n        ]\n\n    courses = {}\n    args = [courses, get_rating_fn]\n    menlo_ucs = m.MenloCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)\n    flow_ucs = m.UserCourse.get_publicly_visible(rmc_util.MIN_NUM_RATINGS)\n\n    increment_ratings(*(args + [get_fields_fn, menlo_ucs]))\n    increment_ratings(*(args + [get_fields_fn, flow_ucs]))\n\n    count = [0]\n\n    def set_course_professor_ratings_in_redis(courses):\n        for course_id, professors in courses.items():\n            for professor_id, ratings in professors.items():\n                if professor_id is None:\n                    continue\n                professor = m.Professor.objects.with_id(professor_id)\n                if not professor:\n                    continue\n                for rating_type, aggregate_rating in ratings.items():\n                    professor.set_course_rating_in_redis(\n                            course_id, rating_type, aggregate_rating)\n                    count[0] += 1\n\n    set_course_professor_ratings_in_redis(courses)\n    print \'set %d course professor rating keys in redis\' % count[0]\n\n\ndef update_all_fb_friend_list():\n    for user in m.User.objects():\n        # TODO(Sandy): Batch requests for performance\n        if user.fbid and not user.is_fb_token_expired:\n            try:\n                user.update_fb_friends(\n                        facebook.get_friend_list(user.fb_access_token))\n                user.save()\n            except facebook.FacebookOAuthException as e:\n                user.fb_access_token_invalid = True\n                user.save()\n            except Exception as e:\n                print "get_friend_list failed for %s with: %s" % (user.id,\n                e.message)\n\n\n# TODO(mack): test it when we get data to test with\n# TODO(mack): currently sort of duplicate logic in\n# User.cache_mutual_course_ids()\ndef update_redis_friend_mutual_courses():\n    # TODO(Sandy): Use friend real time updates after it. There\'s a fb updates\n    # branch for this, pending on:\n    # https://developers.facebook.com/bugs/374296595988186?browse=search_50990ddb8a19d9316431973\n    # Rate limit is 600 calls / 600 seconds / token:\n    # http://stackoverflow.com/questions/8805316/facebook-graph-api-rate-limit-and-batch-requests\n    update_all_fb_friend_list()\n\n    courses_by_user = {}\n    for user in m.User.objects.only(\'friend_ids\', \'course_history\'):\n        friend_ids = [str(friend_id) for friend_id in user.friend_ids]\n        ucs = (m.UserCourse.objects(id__in=user.course_history)\n                                  .only(\'course_id\'))\n        course_ids = [uc.course_id for uc in ucs]\n        courses_by_user[str(user.id)] = [friend_ids, set(course_ids)]\n\n    count = 0\n    user_pair = set()\n    for user_id, (friend_ids, courses) in courses_by_user.iteritems():\n        for friend_id in friend_ids:\n            if user_id < friend_id:\n                first_id = user_id\n                second_id = friend_id\n            else:\n                first_id = friend_id\n                second_id = user_id\n            if (first_id, second_id) in user_pair:\n                continue\n\n            friend_courses = courses_by_user[friend_id][1]\n            mutual_courses = courses.intersection(friend_courses)\n            if mutual_courses:\n                count += 1\n                redis_key = m.User.cls_mutual_courses_redis_key(\n                    first_id, second_id)\n                r.sadd(redis_key, *list(mutual_courses))\n            user_pair.add((first_id, second_id))\n\n    print \'set %d friend pair keys in redis\' % count\n\n\ndef update_mongo_points():\n    total_points = 0\n\n    num_course_comments = 0\n    num_course_ratings = 0\n    num_course_shares = 0\n    num_professor_comments = 0\n    num_professor_ratings = 0\n    num_professor_shares = 0\n    num_invites = 0\n\n    for user in m.User.objects.only(\n            \'num_invites\', \'course_history\', \'num_points\'):\n        num_points = 0\n        if user.num_invites:\n            num_points += m.PointSource.FIRST_INVITE\n            num_invites += 1\n        for uc in m.UserCourse.objects(id__in=user.course_history):\n            num_points += uc.num_points\n\n            if uc.course_review.has_commented:\n                num_course_comments += 1\n            if uc.course_review.has_been_rated:\n                num_course_ratings += 1\n            if uc.course_review.has_shared:\n                num_course_shares += 1\n\n            if uc.professor_review.has_commented:\n                num_professor_comments += 1\n            if uc.professor_review.has_been_rated:\n                num_professor_ratings += 1\n            if uc.professor_review.has_shared:\n                num_professor_shares += 1\n\n        user.update(set__num_points=num_points)\n        total_points += num_points\n\n    r.set(\'total_points\', total_points)\n\n    print \' ===update_mongo_points ===\'\n    print \'num_course_comments\', num_course_comments\n    print \'num_course_ratings\', num_course_ratings\n    print \'num_course_shares\', num_course_shares\n    print \'num_professor_comments\', num_professor_comments\n    print \'num_professor_ratings\', num_professor_ratings\n    print \'num_professor_shares\', num_professor_shares\n    print \'num_invites\', num_invites\n\n\ndef update_exam_schedule():\n    # Crawl data and store on disk\n    rmc_crawler.get_opendata_exam_schedule()\n\n    # Process the data on disk\n    errors = rmc_processor.import_opendata_exam_schedules()\n\n    print "%d exam schedule items found" % m.Exam.objects().count()\n    print "%d exam schedule items skipped" % len(errors)\n\n\ndef update_sections():\n    # Fetch data from OpenData API and cache to files.\n    rmc_crawler.get_opendata_sections()\n\n    # Import from files to DB.\n    rmc_processor.import_opendata_sections()\n\n    # Send push notifications about seat openings.\n    num_sent = m.GcmCourseAlert.send_eligible_alerts()\n    num_expired = m.GcmCourseAlert.delete_expired()\n    print \'Sent %s push notifications and expired %s\' % (num_sent, num_expired)\n\n\ndef update_courses():\n    # First get an up to date list of departments and write to a text file\n    print "Fetching departments"\n    rmc_crawler.get_departments()\n\n    # Import any departments we don\'t already have into Mongo\n    print "Loading departments into Mongo"\n    rmc_processor.import_departments()\n\n    # Hit the endpoints of the OpenData API for each department\n    print "Fetching courses"\n    rmc_crawler.get_opendata2_courses()\n\n    # Load the data into Mongo\n    print "Loading courses into Mongo"\n    rmc_processor.import_courses()\n\n\ndef update_professors_departments():\n    """Update the departments_taught field for each professor in Mongo"""\n    for prof in m.Professor.objects():\n        prof.departments_taught = prof.get_departments_taught()\n        prof.save()\n\n\ndef update_scholarships():\n    """Update the scholarships available in Mongo"""\n    print "Fetching scholarships"\n    rmc_crawler.get_scholarships()\n\n    print "Loading scholarships into Mongo"\n    rmc_processor.import_scholarships()\n\n\nif __name__ == \'__main__\':\n    mongoengine.connect(c.MONGO_DB_RMC)\n\n    parser = argparse.ArgumentParser()\n    mode_mapping = {\n        \'redis_course_professor_rating\': update_redis_course_professor_rating,\n        \'redis_friend_mutual_courses\': update_redis_friend_mutual_courses,\n        \'mongo_course_rating\': update_mongo_course_rating,\n        \'mongo_course_professors\': update_mongo_course_professors,\n        \'mongo_points\': update_mongo_points,\n        \'exam_schedule\': update_exam_schedule,\n        \'sections\': update_sections,\n        \'courses\': update_courses,\n        \'prof_departments\': update_professors_departments,\n        \'scholarships\': update_scholarships\n    }\n    parser.add_argument(\'mode\',\n            help=\'one of %s\' % \',\'.join(mode_mapping.keys() + [\'daily\']))\n    args = parser.parse_args()\n\n    if args.mode == \'daily\':\n        daily_functions = [\n            update_redis_course_professor_rating,\n            update_redis_friend_mutual_courses,\n            update_mongo_course_rating,\n            update_mongo_course_professors,\n            update_mongo_points,\n            update_exam_schedule,\n            update_professors_departments\n        ]\n        for func in daily_functions:\n            try:\n                func()\n            except Exception as exp:\n                print "aggregator.py: function %s threw an exception" % (func)\n                print exp\n    elif args.mode in mode_mapping:\n        func = mode_mapping[args.mode]\n        func()\n    else:\n        sys.exit(\'The mode %s is not supported\' % args.mode)\n', 'message': 'Guarded against division by 0 in aggregator.py\n', 'author_name': 'Jiayi Sandy Wu', 'lang': 'py'}