class_docstr: Handle polling an SQS queue and running Athena queries for updating tables
class_docstr_tok: ["Handle","polling","an","SQS","queue","and","running","Athena","queries","for","updating","tables"]
docstr: Get the partitions that need to be added for the Athena tables
  
  Returns:
      (dict): representation of tables, partitions and locations to be added
          Example:
              {
                  'alerts': {
                      '(dt = '2018-08-01-01')': 's3://streamalert.alerts/2018/08/01/01'
                  }
              }
docstr_tok: ["Get","the","partitions","that","need","to","be","added","for","the","Athena","tables","Returns",":","(","dict",")",":","representation","of","tables",",","partitions","and","locations","to","be","added","Example",":","{","'alerts","'",":","{","'","(","dt","=","'2018-08-01-01","'",")","'",":","'s3",":","\/\/streamalert.alerts\/2018\/08\/01\/01'","}","}"]
code: def _get_partitions_from_keys(self):
      """Get the partitions that need to be added for the Athena tables
  
      Returns:
          (dict): representation of tables, partitions and locations to be added
              Example:
                  {
                      'alerts': {
                          '(dt = \'2018-08-01-01\')': 's3://streamalert.alerts/2018/08/01/01'
                      }
                  }
      """
      partitions = defaultdict(dict)
  
      LOGGER.info("Processing new Hive partitions...")
      for bucket, keys in self._s3_buckets_and_keys.items():
          athena_table = self._athena_buckets.get(bucket)
          if not athena_table:
              # TODO(jacknagz): Add this as a metric
              LOGGER.error(
                  "'%s' not found in 'buckets' config. Please add this "
                  "bucket to enable additions of Hive partitions.",
                  bucket,
              )
              continue
  
          # Iterate over each key
          for key in keys:
              match = None
              key = key.decode("utf-8")
              for pattern in (self._data_regex, self._alerts_regex):
                  match = pattern.search(key)
                  if match:
                      break
  
              if not match:
                  LOGGER.warning(
                      "The key %s does not match any regex, skipping",
                      key,
                  )
                  continue
  
              # Get the path to the objects in S3
              path = posixpath.dirname(key)
              # The config does not need to store all possible tables
              # for enabled log types because this can be inferred from
              # the incoming S3 bucket notification.  Only enabled
              # log types will be sending data to Firehose.
              # This logic extracts out the name of the table from the
              # first element in the S3 path, as that's how log types
              # are configured to send to Firehose.
              if athena_table != "alerts":
                  athena_table = (
                      # when file_format is json, s3 file path is
                      #   s3://bucketname/[data-type]/YYYY/MM/DD/hh/*.gz
                      # when file_format is parquet, s3 file path is
                      #   s3://bucketname/parquet/[data-type]/dt=YYYY-MM-DD-hh/*.parquet
                      path.split("/")[1]
                      if self._file_format == "parquet"
                      else path.split("/")[0]
                  )
  
              # Example:
              # PARTITION (dt = '2017-01-01-01') LOCATION 's3://bucket/path/'
              partition = "(dt = '{year}-{month}-{day}-{hour}')".format(
                  **match.groupdict()
              )
              location = "'s3://{bucket}/{path}'".format(
                  bucket=bucket, path=path
              )
              # By using the partition as the dict key, this ensures that
              # Athena will not try to add the same partition twice.
              # TODO(jacknagz): Write this dictionary to SSM/DynamoDB
              # to increase idempotence of this Lambda function
              partitions[athena_table][partition] = location
  
      return partitions
  
imports: from collections import defaultdict
  import json
  import posixpath
  import re
  import urllib.request
  import urllib.parse
  import urllib.error
  from streamalert.shared.utils import get_database_name, get_data_file_format
  from streamalert.shared.athena import AthenaClient
  from streamalert.shared.config import athena_partition_buckets, load_config
  from streamalert.shared.exceptions import ConfigError
  from streamalert.shared.logger import get_logger
label_desc: Use image node_id 3 for calling the AthenaPartitioner obj's underlying member method code with example usage: obj._get_partitions_from_keys() and returns: partitions