Kafka Consumer Additions
Two options have been added to kafka_consumer.rb:
monitor_unlisted_consumer_groups
kafka_consumer_offsets
MODIFY
kitchen.yml
kitchen.yml
@@ -15,11 +15,13 @@ platforms:
centos-6.6
centos-7.1
debian-7.8
+ debian-9.13
+ debian-10.11
rockylinux-8.5
)
chef_versions = %w(
12.7.2
- 14
+ 14.10.9
16.5
17
18
@@ -387,11 +389,13 @@ suites:
kafka_consumer:
instances:
- kafka_connect_str: localhost:19092
- zk_connect_str: localhost:2181
- zk_prefix: /0.8
consumer_groups:
my_consumer:
my_topic: [0, 1, 4, 12]
+ monitor_unlisted_consumer_groups: true
+ zk_connect_str: localhost:2181
+ zk_prefix: /0.8
+ kafka_consumer_offsets: true
- name: datadog_mesos_master
run_list:
@@ -591,4 +595,3 @@ suites:
instances:
- host: localhost
port: 49112
-

MODIFY
recipes/kafka_consumer.rb
recipes/kafka_consumer.rb
@@ -6,8 +6,18 @@ include_recipe '::dd-agent'
# node.datadog.kafka_consumer.instances = [
# {
# :kafka_connect_str => "localhost:19092",
+# :consumer_groups => {
+# :my_consumer => {
+# :my_topic => [<partition>, <partition>, <partition>, <partition>]
+# } ,
+# # optional -if consumer_groups is set do false no matter of it's set to true or not.
+# # false is the default configuration, but since there is a dependency on consumer_groups
+# # it will be added to conf.yaml as false.
+# :monitor_unlisted_consumer_groups >> true,
# :zk_connect_str => "localhost:2181",
-# :zk_prefix => "/0.8"
+# :zk_prefix => "/0.8",
+# # optional -if it's set to false it will no added to conf.yaml because false is default.
+# :kafka_consumer_offsets => true
# }
# ]

ADD
spec/integrations/kafka_consumer_spec.rb
+describe 'datadog::kafka_consumer' do
+ expected_yaml = <<-EOF
+ logs:
+
+ instances:
+ - kafka_connect_str: localhost:19092
+ consumer_groups:
+ my_consumer:
+ my_topic: [0, 1, 4, 12]
+ monitor_unlisted_consumer_groups: false
+ zk_connect_str: localhost:2181
+ zk_prefix: /0.8
+ kafka_consumer_offsets: true
+
+ init_config:
+ # The Kafka Consumer check does not require any init_config
+ EOF
+
+ cached(:chef_run) do
+ ChefSpec::SoloRunner.new(
+ platform: 'ubuntu',
+ version: '16.04',
+ step_into: ['datadog_monitor']
+ ) do |node|
+ node.automatic['languages'] = { python: { version: '2.7.2' } }
+
+ node.normal['datadog'] = {
+ api_key: 'someapikey',
+ kafka_consumer: {
+ instances: [
+ {
+ kafka_connect_str: 'localhost:19092',
+ consumer_groups: {
+ my_consumer: {
+ my_topic: [0, 1, 4, 12]
+ }
+ },
+ monitor_unlisted_consumer_groups: false,
+ zk_connect_str: 'localhost:2181',
+ password: 'localhost:2181',
+ zk_prefix: '/0.8',
+ kafka_consumer_offsets: true
+ }
+ ]
+ }
+ }
+ end.converge(described_recipe)
+ end
+
+ subject { chef_run }
+
+ it_behaves_like 'datadog-agent'
+
+ it { is_expected.to include_recipe('datadog::dd-agent') }
+
+ it { is_expected.to add_datadog_monitor('kafka_consumer') }
+
+ it 'renders expected YAML config file' do
+ expect(chef_run).to(render_file('/etc/datadog-agent/conf.d/kafka_consumer.d/conf.yaml').with_content { |content|
+ expect(YAML.safe_load(content).to_json).to be_json_eql(YAML.safe_load(expected_yaml).to_json)
+ })
+ end
+end

MODIFY
templates/default/kafka_consumer.yaml.erb
templates/default/kafka_consumer.yaml.erb
instances:
<% @instances.each do |i| -%>
- kafka_connect_str: <%= i['kafka_connect_str'] %>
- zk_connect_str: <%= i['zk_connect_str'] %>
- <% if i['zk_prefix'] -%>
- zk_prefix: <%= i['zk_prefix'] %>
- <% end -%>
<% if i['consumer_groups'] -%>
consumer_groups: <%= i['name'] %>
<% i["consumer_groups"].each do |consumer, t| -%>
@@ -16,8 +12,17 @@ instances:
<% end -%>
<% end -%>
<% end -%>
+ <% if i.key?('monitor_unlisted_consumer_groups') -%>
+ monitor_unlisted_consumer_groups: <%= if i['monitor_unlisted_consumer_groups'] == true && !i['consumer_groups']; true; else false; end %>
+ <% end -%>
+ zk_connect_str: <%= i['zk_connect_str'] %>
+ <% if i['zk_prefix'] -%>
+ zk_prefix: <%= i['zk_prefix'] %>
+ <% end -%>
+ <% if i.key?('kafka_consumer_offsets') && i['kafka_consumer_offsets'] == true-%>
+ kafka_consumer_offsets: true
+ <% end -%>
<% end -%>
init_config:
# The Kafka Consumer check does not require any init_config
-

DELETE
test/integration/datadog_kafka-consumer/bats/kafka_consumer_config.bats
-@test "kafka.yaml exists" {
- [ -f /etc/datadog-agent/conf.d/kafka_consumer.yaml ]
-}
-
-@test "kafka.yaml is correct" {
- export PYTHONPATH=/usr/share/datadog/agent/
- script='import yaml, json, sys; print json.dumps(yaml.load(sys.stdin.read()))'
- actual=$(cat /etc/datadog-agent/conf.d/kafka_consumer.yaml | python -c "$script")
-
- expected='{"instances": [{"zk_connect_str": "localhost:2181", "kafka_connect_str": "localhost:19092", "consumer_groups": {"my_consumer": {"my_topic": [0, 1, 4, 12]}}, "zk_prefix": "/0.8"}], "init_config": null}'
-
- echo "Expected: $expected"
- echo "Actual: $actual"
- [ "x$actual" == "x$expected" ]
-}
-
-
-
-

ADD
test/integration/datadog_kafka_consumer/serverspec/kafka_consumer_spec.rb
+# Encoding: utf-8
+require 'spec_helper'
+
+AGENT_CONFIG = File.join(@agent_config_dir, 'conf.d/kafka_consumer.d/conf.yaml')
+
+describe service(@agent_service_name) do
+ it { should be_running }
+end
+
+describe file(AGENT_CONFIG) do
+ it { should be_a_file }
+
+ it 'is valid yaml matching input values' do
+ generated = YAML.load_file(AGENT_CONFIG)
+
+ expected = {
+ 'instances' => [
+ {
+ :kafka_connect_str => 'localhost:19092',
+ :consumer_groups => {
+ :my_consumer => {
+ :my_topic => [0, 1, 4, 12]
+ }
+ },
+ :monitor_unlisted_consumer_groups => false,
+ :zk_connect_str => 'localhost:2181',
+ :zk_prefix => '/0.8',
+ :kafka_consumer_offsets => true
+ }
+ ],
+ 'logs' => nil,
+ 'init_config' => nil
+ }
+
+ expect(generated.to_json).to be_json_eql expected.to_json
+ end
+end

Kafka Consumer Additions (#840)

Two options have been added to kafka_consumer.rb:
- monitor_unlisted_consumer_groups
- kafka_consumer_offsets