Kafka recipe: allow setting any conf
MODIFY
recipes/kafka.rb
recipes/kafka.rb
@@ -5,9 +5,11 @@ include_recipe 'datadog::dd-agent'
# Set the following attributes
# * `instances` (required)
# List of Kafka clusters to monitor. Each cluster is generally a dictionary with a `host`, `port` and a `name`.
-# More attributes are available. For more information, please refer to : https://github.com/DataDog/integrations-core/blob/master/kafka/conf.yaml.example
+# More attributes are available. For more information, see https://github.com/DataDog/integrations-core/blob/master/kafka/datadog_checks/kafka/data/conf.yaml.example
+# * `conf` (optional)
+# List of metrics to be collected. For more information, see https://github.com/DataDog/integrations-core/blob/master/kafka/datadog_checks/kafka/data/conf.yaml.example
# * `version` (optional)
-# Select the appropriate configuration file template. Available options are:
+# Only used if `conf` is not set, allows you to use a pre-defined conf template. Available options are:
# * `1` (Default, Kafka < 0.8.2).
# * `2` (Kafka >= 0.8.2).
@@ -35,10 +37,14 @@ include_recipe 'datadog::dd-agent'
#
#
+template_version = node['datadog']['kafka']['conf'].nil? ? node['datadog']['kafka']['version'] : 3
+
datadog_monitor 'kafka' do
instances node['datadog']['kafka']['instances']
- version node['datadog']['kafka']['version']
+ version template_version
+ init_config({ :is_jmx => true, :conf => node['datadog']['kafka']['conf'] })
logs node['datadog']['kafka']['logs']
action :add
+ use_integration_template template_version == 3
notifies :restart, 'service[datadog-agent]' if node['datadog']['agent_start']
end

MODIFY
spec/integrations/kafka_spec.rb
spec/integrations/kafka_spec.rb
@@ -622,4 +622,84 @@ describe 'datadog::kafka' do
})
end
end
+
+ context 'custom conf' do
+ expected_yaml = <<-EOF
+ logs: ~
+ instances:
+ - host: localhost
+ port: 7199
+ name: fasdfas
+ user: someuser
+ password: somepass
+ tags:
+ key: value
+
+ init_config:
+ is_jmx: true
+
+ conf:
+ - include:
+ domain: 'kafka.producer'
+ bean_regex: 'kafka\\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*'
+ attribute:
+ Count:
+ metric_type: rate
+ alias: kafka.producer.request_rate
+ EOF
+
+ cached(:chef_run) do
+ ChefSpec::SoloRunner.new(
+ platform: 'ubuntu',
+ version: '16.04',
+ step_into: ['datadog_monitor']
+ ) do |node|
+ node.automatic['languages'] = { python: { version: '2.7.2' } }
+
+ node.normal['datadog'] = {
+ api_key: 'someapikey',
+ kafka: {
+ conf: [
+ {
+ include: {
+ domain: 'kafka.producer',
+ bean_regex: 'kafka\\.producer:type=ProducerRequestMetrics,name=ProducerRequestRateAndTimeMs,clientId=.*',
+ attribute: {
+ Count: {
+ metric_type: 'rate',
+ alias: 'kafka.producer.request_rate'
+ }
+ }
+ }
+ }
+ ],
+ instances: [
+ {
+ host: 'localhost',
+ port: 7199,
+ name: 'fasdfas',
+ user: 'someuser',
+ password: 'somepass',
+ tags: { key: 'value' }
+ }
+ ]
+ }
+ }
+ end.converge(described_recipe)
+ end
+
+ subject { chef_run }
+
+ it_behaves_like 'datadog-agent'
+
+ it { is_expected.to include_recipe('datadog::dd-agent') }
+
+ it { is_expected.to add_datadog_monitor('kafka') }
+
+ it 'renders expected YAML config file' do
+ expect(chef_run).to(render_file('/etc/datadog-agent/conf.d/kafka.d/conf.yaml').with_content { |content|
+ expect(YAML.safe_load(content).to_json).to be_json_eql(YAML.safe_load(expected_yaml).to_json)
+ })
+ end
+ end
end

MODIFY
templates/default/kafka.yaml.erb
templates/default/kafka.yaml.erb
-<%= JSON.parse(({'logs' => @logs }).to_json).to_yaml %>
-
-instances:
- <% @instances.each do |i| -%>
- - host: <%= i['host'] %>
- port: <%= i['port'] %>
- <% if i['name'] -%>
- name: <%= i['name'] %>
- <% end -%>
- <% if i['user'] -%>
- user: <%= i['user'] %>
- <% end -%>
- <% if i['password'] -%>
- password: <%= i['password'] %>
- <% end -%>
- <% if i['process_name_regex'] and i['tools_jar_path'] -%>
- process_name_regex: <%= i['process_name_regex'] %> # Instead of specifying a host, and port. The agent can connect using the attach api.
- # This requires the JDK to be installed and the path to tools.jar to be set below.
- tools_jar_path: <%= i['tools_jar_path'] %>
- <% end -%>
- <% if i['java_bin_path'] -%>
- java_bin_path: <%= i['java_bin_path'] %> #Optional, should be set if the agent cannot find your java executable
- <% end -%>
- <% if i['trust_store_path'] -%>
- trust_store_path: <%= i['trust_store_path'] %> # Optional, should be set if ssl is enabled
- <% end -%>
- <% if i['trust_store_password'] -%>
- trust_store_password: <%= i['trust_store_password'] %>
- <% end -%>
- <% if i.key?('tags') -%>
- tags:
- <% i['tags'].each do |k, v| -%>
- <%= k %>: <%= v %>
- <% end -%>
- <% end -%>
- <% end -%>
+<%= JSON.parse(({'logs' => @logs, 'instances' => @instances }).to_json).to_yaml %>
<% if @version == 2 %>
init_config:

Kafka recipe: allow setting any conf (#776)