Using the Streams for Apache Kafka Proxy
Using the Streams for Apache Kafka Proxy Red Hat Streams for Apache Kafka 2.7 Enhancing Kafka with specialized features
cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1 > vault.root.token export VAULT_TOKEN=USD(cat vault.root.token) helm repo add hashicorp https://helm.releases.hashicorp.com helm install vault hashicorp/vault --create-namespace --namespace=vault --version <helm_version> --values vault/helm-dev-values.yaml --set server.dev.devRootToken=USD{VAULT_TOKEN} --wait get pods -n vault NAME READY STATUS RESTARTS vault-0 1/1 Running 0 export VAULT_ADDR=USD(oc get route -n vault vault --template='https://{{.spec.host}}') vault secrets enable transit export VAULT_TRANSIT_URL=USD{VAULT_ADDR}/v1/transit export VAULT_ADDR=https://<vault server>:8200 export VAULT_NAMESPACE=<namespaces> vault secrets enable transit export VAULT_TRANSIT_URL=USD{VAULT_ADDR}/v1/USD{VAULT_NAMESPACE}/transit sed -i "s/\(vaultTransitEngineUrl:\).*USD/\1 USD{VAULT_TRANSIT_URL}/" */proxy/proxy-config.yaml vault policy write amqstreams_proxy_encryption_filter_policy vault/amqstreams_proxy_encryption_filter_policy.hcl vault token create -display-name "amqstreams-proxy encryption filter" -policy=amqstreams_proxy_encryption_filter_policy -no-default-policy -orphan -field=token > vault.encryption.token create secret generic proxy-encryption-vault-token -n proxy --from-file=encryption-vault-token.txt=vault.encryption.token --dry-run=client -o yaml > base/proxy/proxy-encryption-vault-token-secret.yaml run -n <my_proxy_namespace> -ti proxy-producer --image=registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0 --rm=true --restart=Never -- bin/kafka-topics.sh --bootstrap-server proxy-service:9092 --create -topic my-topic vault write -f transit/keys/KEK_my-topic filters: - type: RecordEncryption config: kms: VaultKmsService 1 kmsConfig: vaultTransitEngineUrl: http://vault.vault.svc.cluster.local:8200/v1/transit 2 vaultToken: passwordFile: /opt/proxy/encryption/token.txt 3 selector: TemplateKekSelector 4 selectorConfig: template: "KEK_USD{topicName}" 5 # cd /examples/proxy/record-encryption/ apply -k cluster-ip cd /examples/proxy/record-encryption/ apply -k loadbalancer LOAD_BALANCER_ADDRESS=USD(oc get service -n <my_proxy_namespace> proxy-service --template='{{(index .status.loadBalancer.ingress 0).hostname}}') sed -i "s/\(brokerAddressPattern:\).*USD/\1 USD{LOAD_BALANCER_ADDRESS}/" load-balancer/proxy/proxy-config.yaml apply -k load-balancer && oc delete pod -n <my_proxy_namespace> --all get pods -n <my_proxy_namespace> NAME READY STATUS RESTARTS my-cluster-kafka-0 1/1 Running 0 my-cluster-kafka-1 1/1 Running 0 my-cluster-kafka-2 1/1 Running 0 my-cluster-proxy-<pod_id> 1/1 Running 0 run -n <my_proxy_namespace> -ti proxy-producer --image=registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0 --rm=true --restart=Never -- bin/kafka-console-producer.sh --bootstrap-server proxy-service:9092 --topic my-topic run -n my-cluster -ti cluster-consumer --image=registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0 --rm=true --restart=Never -- ./bin/kafka-console-consumer.sh --bootstrap-server my-cluster-kafka-bootstrap:9092 --topic my-topic --from-beginning --timeout-ms 10000 run -n <my_proxy_namespace> -ti proxy-consumer --image=registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0 --rm=true --restart=Never -- ./bin/kafka-console-consumer.sh --bootstrap-server proxy-service:9092 --topic my-topic --from-beginning --timeout-ms 10000 kafka-console-producer --bootstrap-server <load_balancer_address>:9092 --topic my-topic run -n my-cluster -ti cluster-consumer --image=registry.redhat.io/amq-streams/kafka-37-rhel9:2.7.0 --rm=true --restart=Never -- ./bin/kafka-console-consumer.sh --bootstrap-server my-cluster-kafka-bootstrap:9092 --topic my-topic --from-beginning --timeout-ms 10000 kafka-console-consumer --bootstrap-server <load_balancer_address>:9092 --topic my-topic --from-beginning --timeout-ms 10000 apiVersion: v1 kind: ConfigMap metadata: name: proxy-config data: config.yaml: | adminHttp: 1 endpoints: prometheus: {} virtualClusters: 2 my-cluster-proxy: 3 targetCluster: bootstrap_servers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9093 4 tls: 5 trust: storeFile: /opt/proxy/trust/ca.p12 storePassword: passwordFile: /opt/proxy/trust/ca.password clusterNetworkAddressConfigProvider: 6 type: SniRoutingClusterNetworkAddressConfigProvider Config: bootstrapAddress: mycluster-proxy.kafka:9092 brokerAddressPattern: brokerUSD(nodeId).mycluster-proxy.kafka logNetwork: false 7 logFrames: false tls: 8 key: storeFile: /opt/proxy/server/key-material/keystore.p12 storePassword: passwordFile: /opt/proxy/server/keystore-password/storePassword filters: 9 - type: EnvelopeEncryption 10 config: 11 kms: VaultKmsService kmsConfig: vaultTransitEngineUrl: https://vault.vault.svc.cluster.local:8200/v1/transit vaultToken: passwordFile: /opt/proxy/server/token.txt tls: 12 key: storeFile: /opt/cert/server.p12 storePassword: passwordFile: /opt/cert/store.password keyPassword: passwordFile: /opt/cert/key.password storeType: PKCS12 selector: TemplateKekSelector selectorConfig: template: "USD{topicName}" virtualClusters: my-cluster-proxy: tls: key: storeFile: <path>/server.p12 1 storePassword: passwordFile: <path>/store.password 2 keyPassword: passwordFile: <path>/key.password 3 storeType: PKCS12 4 # virtualClusters: my-cluster-proxy: tls: key: privateKeyFile: <path>/server.key 1 certificateFile: <path>/server.crt 2 keyPassword: passwordFile: <path>/key.password 3 ... virtualClusters: demo: targetCluster: bootstrap_servers: myprivatecluster:9092 tls: trust: insecure: true 1 # ... virtualClusters: my-cluster-proxy: targetCluster: bootstrap_servers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9093 tls: {} # virtualClusters: my-cluster-proxy: targetCluster: bootstrap_servers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9093 tls: trust: storeFile: <path>/trust.p12 1 storePassword: passwordFile: <path>/store.password 2 storeType: PKCS12 3 # virtualClusters: my-cluster-proxy: targetCluster: bootstrap_servers: my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9093:9092 tls: key: privateKeyFile: <path>/client.key 1 certificateFile: <path>/client.crt 2 trust: storeFile: <path>/server.crt storeType: PEM virtualClusters: my-cluster-proxy: targetCluster: bootstrap_servers: myprivatecluster:9092 tls: trust: insecure: true # clusterNetworkAddressConfigProvider: type: PortPerBrokerClusterNetworkAddressConfigProvider config: bootstrapAddress: mycluster.kafka.com:9192 1 brokerAddressPattern: mybroker-USD(nodeId).mycluster.kafka.com 2 brokerStartPort: 9193 3 numberOfBrokerPorts: 3 4 bindAddress: 192.168.0.1 5 mybroker-0.mycluster.kafka.com:9193 mybroker-1.mycluster.kafka.com:9194 mybroker-2.mycluster.kafka.com:9194 clusterNetworkAddressConfigProvider: type: SniRoutingClusterNetworkAddressConfigProvider config: bootstrapAddress: mycluster.kafka.com:9192 1 brokerAddressPattern: mybroker-USD(nodeId).mycluster.kafka.com bindAddress: 192.168.0.1 adminHttp: endpoints: prometheus: {} adminHttp: host: localhost port: 9999 endpoints: prometheus: {} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: name: proxy labels: app: proxy spec: selector: matchLabels: app: proxy namespaceSelector: matchNames: - proxy podMetricsEndpoints: - path: /metrics port: metrics dnf install <package_name> dnf install <path_to_download_package>