<NME> task.py
<BEF> # Copyright 2019 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Executes model training and evaluation."""

import argparse
import logging
import sys

from sklearn import model_selection

from trainer import input_util
from trainer import model


def _train_and_evaluate(estimator, dataset, output_dir):
  """Runs model training and evalation."""


  utils.dump_object(estimator, model_output_path)
  utils.dump_object(scores, metric_output_path)

  # The default name of the metric is training/hptuning/metric.
  # We recommend that you assign a custom name
  # The only functional difference is that if you use a custom name,
  # you must set the hyperparameterMetricTag value in the
  # HyperparameterSpec object in your job request to match your chosen name.
  hpt = hypertune.HyperTune()
  hpt.report_hyperparameter_tuning_metric(
      hyperparameter_metric_tag='my_metric_tag',
      metric_value=np.mean(scores),
      global_step=1000)


def run_experiment(flags):
  """Testbed for running model training and evaluation."""
  # Get data for training and evaluation

  dataset = utils.read_df_from_bigquery(
      flags.input, num_samples=flags.num_samples)

  # Get model
  estimator = model.get_estimator(flags)

  # Run training and evaluation
  _train_and_evaluate(estimator, dataset, flags.job_dir)


def _parse_args(argv):
  """Parses command-line arguments."""

  parser = argparse.ArgumentParser()

  parser.add_argument(
      '--input',
      help='''Dataset to use for training and evaluation.
              Can be BigQuery table or a file (CSV).
              If BigQuery table, specify as as PROJECT_ID.DATASET.TABLE_NAME.
            ''',
      required=True,
  )

  parser.add_argument(
      '--job-dir',
      help='Output directory for exporting model and other metadata.',
      required=True,
  )

  parser.add_argument(
      '--log_level',
      help='Logging level.',
      choices=[
          'DEBUG',
          'ERROR',
          'FATAL',
          'INFO',
          'WARN',
      ],
      default='INFO',
  )

  parser.add_argument(
      '--num_samples',
      help='Number of samples to read from `input`',
      type=int,
      default=None,
  )

  parser.add_argument(
      '--n_estimators',
      help='Number of trees in the forest.',
      default=10,
      type=int,
  )

  parser.add_argument(
      '--max_depth',
      help='The maximum depth of the tree.',
      type=int,
      default=None,
  )

  parser.add_argument(
      '--min_samples_leaf',
      help='The minimum number of samples required to be at a leaf node.',
      default=1,
      type=int,
  )

  parser.add_argument(
      '--criterion',
      help='The function to measure the quality of a split.',
      choices=[
          'gini',
          'entropy',
      ],
      default='gini',
  )

  return parser.parse_args(argv)


def main():
  """Entry point."""

  flags = _parse_args(sys.argv[1:])
  logging.basicConfig(level=flags.log_level.upper())
  run_experiment(flags)


if __name__ == '__main__':
  main()

<MSG> add function to upload file from local to GCS

<DFF> @@ -18,13 +18,40 @@
 import argparse
 import logging
 import sys
+from urllib.parse import urlparse
 
 from sklearn import model_selection
 
+from google.cloud import storage
+
 from trainer import input_util
 from trainer import model
 
 
+def _upload_gcs_file(local_path, gcs_path):
+  """
+  Upload local file to Google Cloud Storage
+  Args:
+    local_path: (string) Local file
+    gcs_path: (string) Google Cloud Storage destination
+
+  Returns:
+    None
+  """
+  storage_client = storage.Client()
+  parse_result = urlparse(gcs_path)
+
+  # Parse bucket name
+  gcs_path = parse_result.path
+  bucket_name = parse_result.hostname
+  bucket = storage_client.get_bucket(bucket_name)
+
+  blob_path = gcs_path[1:] if gcs_path[0] == '/' else gcs_path
+  blob = bucket.blob(blob_path)
+
+  blob.upload_from_filename(local_path)
+
+
 def _train_and_evaluate(estimator, dataset, output_dir):
   """Runs model training and evalation."""
 

{'addition_count': 27, 'commit_subject': 'add function to upload file from local to GCS', 'deletion_count': 0, 'file_extension': '.py', 'lang': 'py', 'license': 'apache-2.0', 'repo_name': 'GoogleCloudPlatform/cloudml-samples'}