message: es index templates, handling of e2e projects
diff: @@ -100,6 +100,8 @@ objects:
  items:
  - key: FLUENT_CONF
  path: fluent.conf
  + - key: CONTAINER_LOGS_JSON
  + path: container-logs.json
  name: ${SERVICE_NAME}-config
  name: config
  - emptyDir: {}
  @@ -149,15 +151,21 @@ objects:
  <label @ELASTICSEARCH>
  <filter **>
  @type record_modifier
  + <record>
  + namespace_name "${record['kubernetes']['namespace_name']}"
  + </record>
  <replace>
  # We run automated testing which creates project names in the form of e2e-0000000000-AAAAAAAAAAAAAAAAAAAAAAAAAA
  # we just save them as with the project name 'e2'
  - key kubernetes.namespace_name
  + key namespace_name
  expression /^e2e-\d{10}-\S{26}$/
  replace e2e
  </replace>
  + </filter>
  + <filter **>
  + @type record_modifier
  <record>
  - index_name container-logs-${record['kubernetes']['namespace_name']}-${Time.at(time).strftime("%G.%V")}
  + index_name container-logs-${record['namespace_name']}-${Time.at(time).strftime("%G.%V")}
  </record>
  </filter>
  @@ -169,6 +177,8 @@ objects:
  @type elasticsearch
  host logs-db
  port 9200
  + templates { "container-logs": "/fluentd/etc/container-logs.json"}
  + template_overwrite true
  target_index_key index_name
  index_name container-logs-noproject-%G.%V
  <buffer tag,time>
  @@ -184,9 +194,57 @@ objects:
  overflow_action block
  </buffer>
  id_key viaq_msg_id
  - remove_keys viaq_msg_id, viaq_index_name, kubernetes.master_url
  + remove_keys viaq_msg_id, viaq_index_name, namespace_name, kubernetes.master_url
  type_name _doc
  user admin
  password "#{ENV['LOGSDB_ADMIN_PASSWORD']}"
  </match>
  </label>
  + CONTAINER_LOGS_JSON: |-
  + {
  + "template" : "service-logs-*",
  + "version" : 60002,
  + "settings" : {
  + "index.refresh_interval" : "5s",
  + "number_of_shards": 1,
  + "number_of_replicas": 2
  + },
  + "mappings" : {
  + "_default_" : {
  + "dynamic_templates" : [ {
  + "message_field" : {
  + "path_match" : "message",
  + "match_mapping_type" : "string",
  + "mapping" : {
  + "type" : "text",
  + "norms" : false
  + }
  + }
  + }, {
  + "string_fields" : {
  + "match" : "*",
  + "match_mapping_type" : "string",
  + "mapping" : {
  + "type" : "text", "norms" : false,
  + "fields" : {
  + "keyword" : { "type": "keyword", "ignore_above": 256 }
  + }
  + }
  + }
  + } ],
  + "properties" : {
  + "@timestamp": { "type": "date"},
  + "@version": { "type": "keyword"},
  + "geoip" : {
  + "dynamic": true,
  + "properties" : {
  + "ip": { "type": "ip" },
  + "location" : { "type" : "geo_point" },
  + "latitude" : { "type" : "half_float" },
  + "longitude" : { "type" : "half_float" }
  + }
  + }
  + }
  + }
  + }
  + }
  \ No newline at end of file
  