text: #!/usr/bin/env python
  # -*- coding: utf-8 -*-
  # File: fargate_task.py
  #
  # Copyright 2019 Markus Muhr
  #
  # Permission is hereby granted, free of charge, to any person obtaining a copy
  #  of this software and associated documentation files (the "Software"), to
  #  deal in the Software without restriction, including without limitation the
  #  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  #  sell copies of the Software, and to permit persons to whom the Software is
  #  furnished to do so, subject to the following conditions:
  #
  # The above copyright notice and this permission notice shall be included in
  #  all copies or substantial portions of the Software.
  #
  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  #  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  #  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  #  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  #  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  #  DEALINGS IN THE SOFTWARE.
  #
  
  """
  Implementation for aggregate Fargate Task
  
  .. _Google Python Style Guide:
     http://google.github.io/styleguide/pyguide.html
  """
  
  
  import logging
  import sys
  import io
  import gzip
  import json
  import csv
  
  
  __author__ = '''Markus Muhr <mmuhr@schubergphilis.com>'''
  __docformat__ = '''google'''
  __date__ = '''05-03-2019'''
  __copyright__ = '''Copyright 2019, Markus Muhr'''
  __license__ = '''MIT'''
  __maintainer__ = '''Markus Muhr'''
  __email__ = '''<mmuhr@schubergphilis.com>'''
  __status__ = '''Development'''  # "Prototype", "Development", "Production".
  
  
  # This is the main prefix used for logging
  LOGGER_BASENAME = 'aggregate-fargate-task'
  LOGGER = logging.getLogger(LOGGER_BASENAME)
  LOGGER.addHandler(logging.StreamHandler(sys.stdout))
  LOGGER.setLevel(logging.INFO)
  
  
  def load_tsv(file_name):
      """
      Load a tsv file.
  
      This method loads a tab separated values file from the local input folder.
      """
      gzip_file = gzip.GzipFile(mode='r', filename=file_name)
      return csv.reader(io.TextIOWrapper(gzip_file), delimiter='\t')
  
  
  def find_title_ids(prefix_folder, years):
      """
      Find title ids matching years.
  
      This method filters all title ids for titles starting in the provided years
      """
      rows = load_tsv(prefix_folder+'input/title.basics.tsv.gz')
      filtered_rows = filter(lambda row: any(year in row[5] for year in years), rows)
      title_ids = map(lambda row: row[0], filtered_rows)
      return set(title_ids)
  
  
  def calculate_averages(filtered_ratings):
      """
      Find average ratings and num votes for given ratings.
  
      This method calculates the mean for ratings and number of votes for the provided ratings.
      """
      filtered_number = len(filtered_ratings)
      average_rating = sum([float(rating[1]) for rating in filtered_ratings]) / filtered_number
      average_votes = sum([float(rating[2]) for rating in filtered_ratings]) / filtered_number
      LOGGER.info("Average ratings {} - average votes {} !".format(average_rating, average_votes))
      return average_rating, average_votes
  
  
  def enrich_ratings(prefix_folder, title_ids):
      """
      Enrich ratings.
  
      This method enriches the ratings by finding the mean of the ratings and normalising each rating.
      """
      title_ratings = load_tsv(prefix_folder + 'input/title.ratings.tsv.gz')
      filtered_ratings = list(filter(lambda rating: rating[0] in title_ids, title_ratings))
  
      average_rating, average_votes = calculate_averages(filtered_ratings)
  
      enriched_ratings = map(lambda row: {"tconst": row[0], "rnorm_averageRating": float(row[1]) - average_rating,
                                          "rnorm_numVotes": float(row[2]) - average_votes}, filtered_ratings)
  
      return list(enriched_ratings)
  
  
  def run(prefix_folder, years):
      """
      Run method.
  
      This method is the starting point of the actual logic
      """
      LOGGER.info("Started calculation!")
  
      title_ids = find_title_ids(prefix_folder, years)
      LOGGER.info("Finished title ids in years!")
  
      enriched_ratings = enrich_ratings(prefix_folder, title_ids)
      LOGGER.info("Finished enriching ratings!")
  
      with open(prefix_folder + 'output/rnorm-aggregate.json', 'w') as outfile:
          json.dump(enriched_ratings, outfile)
  
      LOGGER.info("Finished calculation!")
  
  
  def main():
      """
      Main method.
  
      This method is the entry point for local testing.
      """
      prefix_folder = "/Users/mmuhr/Documents/repository/funwithserverless/"
      years = ["2017,2018"]
      run(prefix_folder, years)
  
  
  if __name__ == "__main__":
      main()
  