title: Connect PySpark to Kafka from Docker container
body: <p>I have a Kafka cluster that I'm managing with Docker.</p>
  <p>I have a container where I'm running the broker and another one where I run the pyspark program which is supposed to connect to the kafka topic inside the broker container.</p>
  <p>If I run the pyspark script in my local laptop everything runs perfectly but if I try to run the same code from inside the pyspark container I get the following error:</p>
  <pre><code>AnalysisException: Failed to find data source: kafka. Please deploy the application as per the deployment section of &quot;Structured Streaming + Kafka Integration Guide&quot;
  </code></pre>
  <p>Does anybody know how can I access the Kafka topic running in a docker container from another external container?</p>
  <p>This is the PySpark code</p>
  <pre><code>df = spark \
      .readStream \
      .format(&quot;kafka&quot;) \
      .option(&quot;kafka.bootstrap.servers&quot;, &quot;localhost:9092&quot;) \
      .option(&quot;failOnDataLoss&quot;, &quot;false&quot;) \
      .option(&quot;subscribe&quot;, &quot;tweets&quot;) \
      .option(&quot;startingOffsets&quot;, &quot;earliest&quot;) \
      .load()
  </code></pre>
  <p>This is my docker-compose.yml</p>
  <pre><code>version: &quot;2&quot;
  services:
    zookeeper:
      image: confluentinc/cp-zookeeper:6.1.0
      hostname: zookeeper
      container_name: zookeeper
      ports:
        - &quot;2181:2181&quot;
      environment:
        ZOOKEEPER_CLIENT_PORT: 2181
        ZOOKEEPER_TICK_TIME: 2000
  
    broker:
      image: confluentinc/cp-server:6.1.0
      hostname: broker
      container_name: broker
      depends_on:
        - zookeeper
      ports:
        - &quot;9092:9092&quot;
        - &quot;9101:9101&quot;
      environment:
        KAFKA_BROKER_ID: 1
        KAFKA_ZOOKEEPER_CONNECT: &quot;zookeeper:2181&quot;
        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
        KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
        KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
        KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
        KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
        KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
        KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
        KAFKA_JMX_PORT: 9101
        KAFKA_JMX_HOSTNAME: localhost
        KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
        CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
        CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
        CONFLUENT_METRICS_ENABLE: &quot;true&quot;
        CONFLUENT_SUPPORT_CUSTOMER_ID: &quot;anonymous&quot;
  
    schema-registry:
      image: confluentinc/cp-schema-registry:6.1.0
      hostname: schema-registry
      container_name: schema-registry
      depends_on:
        - broker
      ports:
        - &quot;8081:8081&quot;
      environment:
        SCHEMA_REGISTRY_HOST_NAME: schema-registry
        SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: &quot;broker:29092&quot;
        SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
  
    connect:
      image: cnfldemos/cp-server-connect-datagen:latest
      hostname: connect
      container_name: connect
      build:
        context: .
        dockerfile: Dockerfile_connect
      depends_on:
        - broker
        - schema-registry
      ports:
        - &quot;8083:8083&quot;
      environment:
        CONNECT_BOOTSTRAP_SERVERS: &quot;broker:29092&quot;
        CONNECT_REST_ADVERTISED_HOST_NAME: connect
        CONNECT_REST_PORT: 8083
        CONNECT_GROUP_ID: compose-connect-group
        CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
        CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
        CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
        CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
        CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
        CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
        CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
        CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
        CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
        CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
        # CLASSPATH required due to CC-2422
        CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.1.0.jar
        CONNECT_PRODUCER_INTERCEPTOR_CLASSES: &quot;io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor&quot;
        CONNECT_CONSUMER_INTERCEPTOR_CLASSES: &quot;io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor&quot;
        CONNECT_PLUGIN_PATH: &quot;/usr/share/java,/usr/share/confluent-hub-components&quot;
        CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
  
    control-center:
      image: confluentinc/cp-enterprise-control-center:6.1.0
      hostname: control-center
      container_name: control-center
      depends_on:
        - broker
        - schema-registry
        - connect
        - ksqldb-server
      ports:
        - &quot;9021:9021&quot;
      environment:
        CONTROL_CENTER_BOOTSTRAP_SERVERS: &quot;broker:29092&quot;
        CONTROL_CENTER_CONNECT_CLUSTER: &quot;connect:8083&quot;
        CONTROL_CENTER_KSQL_KSQLDB1_URL: &quot;http://ksqldb-server:8088&quot;
        CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: &quot;http://localhost:8088&quot;
        CONTROL_CENTER_SCHEMA_REGISTRY_URL: &quot;http://schema-registry:8081&quot;
        CONTROL_CENTER_REPLICATION_FACTOR: 1
        CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
        CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
        CONFLUENT_METRICS_TOPIC_REPLICATION: 1
        PORT: 9021
  
    ksqldb-server:
      image: confluentinc/cp-ksqldb-server:6.1.0
      hostname: ksqldb-server
      container_name: ksqldb-server
      depends_on:
        - broker
        - connect
      ports:
        - &quot;8088:8088&quot;
      environment:
        KSQL_CONFIG_DIR: &quot;/etc/ksql&quot;
        KSQL_BOOTSTRAP_SERVERS: &quot;broker:29092&quot;
        KSQL_HOST_NAME: ksqldb-server
        KSQL_LISTENERS: &quot;http://0.0.0.0:8088&quot;
        KSQL_CACHE_MAX_BYTES_BUFFERING: 0
        KSQL_KSQL_SCHEMA_REGISTRY_URL: &quot;http://schema-registry:8081&quot;
        KSQL_PRODUCER_INTERCEPTOR_CLASSES: &quot;io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor&quot;
        KSQL_CONSUMER_INTERCEPTOR_CLASSES: &quot;io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor&quot;
        KSQL_KSQL_CONNECT_URL: &quot;http://connect:8083&quot;
        KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
        KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: &quot;true&quot;
        KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: &quot;true&quot;
  
    ksqldb-cli:
      image: confluentinc/cp-ksqldb-cli:6.1.0
      container_name: ksqldb-cli
      depends_on:
        - broker
        - connect
        - ksqldb-server
      entrypoint: /bin/sh
      tty: true
  
    ksql-datagen:
      image: confluentinc/ksqldb-examples:6.1.0
      hostname: ksql-datagen
      container_name: ksql-datagen
      depends_on:
        - ksqldb-server
        - broker
        - schema-registry
        - connect
      command: &quot;bash -c 'echo Waiting for Kafka to be ready... &amp;&amp; \
        cub kafka-ready -b broker:29092 1 40 &amp;&amp; \
        echo Waiting for Confluent Schema Registry to be ready... &amp;&amp; \
        cub sr-ready schema-registry 8081 40 &amp;&amp; \
        echo Waiting a few seconds for topic creation to finish... &amp;&amp; \
        sleep 11 &amp;&amp; \
        tail -f /dev/null'&quot;
      environment:
        KSQL_CONFIG_DIR: &quot;/etc/ksql&quot;
        STREAMS_BOOTSTRAP_SERVERS: broker:29092
        STREAMS_SCHEMA_REGISTRY_HOST: schema-registry
        STREAMS_SCHEMA_REGISTRY_PORT: 8081
  
    rest-proxy:
      image: confluentinc/cp-kafka-rest:6.1.0
      depends_on:
        - broker
        - schema-registry
      ports:
        - 8082:8082
      hostname: rest-proxy
      container_name: rest-proxy
      environment:
        KAFKA_REST_HOST_NAME: rest-proxy
        KAFKA_REST_BOOTSTRAP_SERVERS: &quot;broker:29092&quot;
        KAFKA_REST_LISTENERS: &quot;http://0.0.0.0:8082&quot;
        KAFKA_REST_SCHEMA_REGISTRY_URL: &quot;http://schema-registry:8081&quot;
  
    elasticsearch:
      image: docker.elastic.co/elasticsearch/elasticsearch:7.11.2
      container_name: elasticsearch
      environment:
        - xpack.security.enabled=false
        - discovery.type=single-node
      ulimits:
        memlock:
          soft: -1
          hard: -1
        nofile:
          soft: 65536
          hard: 65536
      cap_add:
        - IPC_LOCK
      volumes:
        - elasticsearch-data:/usr/share/elasticsearch/data
      ports:
        - 9200:9200
        - 9300:9300
  
    kibana:
      container_name: kibana
      image: docker.elastic.co/kibana/kibana:7.11.2
      environment:
        - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      ports:
        - 5601:5601
      depends_on:
        - elasticsearch
  
    pyspark:
      image: jupyter/pyspark-notebook:latest
      hostname: pyspark
      container_name: pyspark
      ports:
        - 8888:8888
      #volumes:
      #  - pySpark: pySpark/
      depends_on:
        - broker
  
  volumes:
    elasticsearch-data:
      driver: local
    pySpark:
      driver: local
  </code></pre>