beam/proto_to_bq/pipeline.py
def run(argv=None):
    """The main function which creates the pipeline and runs it."""

    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--input',
        dest='input',
        required=False,
        help='Input file to read. This can be a local file or '
        'a file in a Google Storage Bucket.',
        default='gs://jk-bq-dev-datasets/test.tfrecords')

    parser.add_argument('--output',
                        dest='output',
                        required=False,
                        help='Output BQ table to write results to.',
                        default='sandbox.person')

    known_args, pipeline_args = parser.parse_known_args(argv)

    table_spec = known_args.output 


#    p = beam.Pipeline(options=PipelineOptions(pipeline_args))
#    _ = (p
    with beam.Pipeline(options=PipelineOptions(pipeline_args)) as p:
        (p
        | 'Read from a TFRecords' >> beam.io.ReadFromTFRecord(known_args.input)
        | 'Transform to Dict' >>  beam.ParDo(ProtoToDict())
        | 'Write to BigQuery' >> beam.io.WriteToBigQuery(
                table_spec,
                #schema=table_schema,
                write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
                create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER))
The main function which creates the pipeline and runs it.
The main function which creates the pipeline and runs it.
The main function which creates the pipeline and runs it .
def run(argv=None):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--input',
        dest='input',
        required=False,
        help='Input file to read. This can be a local file or '
        'a file in a Google Storage Bucket.',
        default='gs://jk-bq-dev-datasets/test.tfrecords')
    parser.add_argument('--output',
                        dest='output',
                        required=False,
                        help='Output BQ table to write results to.',
                        default='sandbox.person')
    known_args, pipeline_args = parser.parse_known_args(argv)
    table_spec = known_args.output 
    with beam.Pipeline(options=PipelineOptions(pipeline_args)) as p:
        (p
        | 'Read from a TFRecords' >> beam.io.ReadFromTFRecord(known_args.input)
        | 'Transform to Dict' >>  beam.ParDo(ProtoToDict())
        | 'Write to BigQuery' >> beam.io.WriteToBigQuery(
                table_spec,
                write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
                create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER))
def run ( argv = None ) : parser = argparse . ArgumentParser ( ) parser . add_argument ( '--input' , dest = 'input' , required = False , help = 'Input file to read. This can be a local file or ' 'a file in a Google Storage Bucket.' , default = 'gs://jk-bq-dev-datasets/test.tfrecords' ) parser . add_argument ( '--output' , dest = 'output' , required = False , help = 'Output BQ table to write results to.' , default = 'sandbox.person' ) known_args , pipeline_args = parser . parse_known_args ( argv ) table_spec = known_args . output with beam . Pipeline ( options = PipelineOptions ( pipeline_args ) ) as p : ( p | 'Read from a TFRecords' >> beam . io . ReadFromTFRecord ( known_args . input ) | 'Transform to Dict' >> beam . ParDo ( ProtoToDict ( ) ) | 'Write to BigQuery' >> beam . io . WriteToBigQuery ( table_spec , write_disposition = beam . io . BigQueryDisposition . WRITE_APPEND , create_disposition = beam . io . BigQueryDisposition . CREATE_NEVER ) )
The main function which creates the pipeline and runs it.
The main function which creates the pipeline and runs it .
"""The main function which creates the pipeline and runs it.""" #    p = beam.Pipeline(options=PipelineOptions(pipeline_args)) #    _ = (p #schema=table_schema,
argv