We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Error when using usage-example with generator.py
<!--
GitHub is reserved for bug reports and feature requests; it is not the place
for general questions. If you have a general question, please visit the
discuss forums: https://discuss.elastic.co/tag/ecs-elastic-common-schema.

Please fill in the following details to help us reproduce the bug:
-->

**Description of the problem including expected versus actual behavior**:

Trying `script/generator.py` for the first time, I stumble upon a stack trace throwing `KeyError: 'error'`. I initially though I maybe passed wrong parameters but it looks like example documented in `USAGE.md` also have the same issue.

Thanks for writing this tool, I'm eager to use it since it can greatly help me to generate/maintain configuration for custom Beats indices. Let me know if I can somehow help you further.

**Steps to reproduce**:

1. `git clone https://github.com/elastic/ecs && cd ecs`
2. Running `generator.py` as documented in `USAGE.md`:
```sh
$ python3 scripts/generator.py --ref v1.6.0 \
  --subset            usage-example/fields/subset.yml \
  --include           usage-example/fields/custom/ \
  --out               usage-example/ \
  --template-settings usage-example/fields/template-settings.json \
  --mapping-settings  usage-example/fields/mapping-settings.json
```

**Provide logs (if relevant)**:

```
Loading schemas from git ref v1.6.0
Running generator. ECS version 1.6.0
Loading user defined schemas: ['usage-example/fields/custom/']
/Users/florent/Sources/ecs/scripts/schema/cleaner.py:185: UserWarning: Example value for field `header_flags` contains an object or array which must be quoted to avoid YAML interpretation.

This will cause an exception when running in strict mode.
Warning check:
  check_example_value(field, strict=strict_mode)
/Users/florent/Sources/ecs/scripts/schema/cleaner.py:185: UserWarning: Example value for field `resolved_ip` contains an object or array which must be quoted to avoid YAML interpretation.
[...]
Traceback (most recent call last):
  File "/Users/florent/Sources/ecs/scripts/generator.py", line 106, in <module>
    main()
  File "/Users/florent/Sources/ecs/scripts/generator.py", line 60, in main
    es_template.generate_legacy(flat, ecs_version, out_dir, args.template_settings, args.mapping_settings)
  File "/Users/florent/Sources/ecs/scripts/generators/es_template.py", line 109, in generate_legacy
    generate_legacy_template_version(6, ecs_version, mappings_section, out_dir, template_settings_file)
  File "/Users/florent/Sources/ecs/scripts/generators/es_template.py", line 115, in generate_legacy_template_version
    template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)
  File "/Users/florent/Sources/ecs/scripts/generators/es_template.py", line 202, in template_settings
    error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']
KeyError: 'error'
```

**Any additional context:**

```
$ git rev-parse HEAD
83ced31c9da6b3e5599b01e319a827a7941e6439

$ python3 --version
Python 3.9.0

$ pip3 list | grep -i -E '(yaml|pep|mock|git|jinja)'
autopep8      1.4.4
gitdb         4.0.5
GitPython     3.1.2
Jinja2        2.11.2
mock          4.0.2
PyYAML        5.3b1
yamllint      1.19.0
```

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/generators/es_template.py`
Content:
```
1 import copy
2 import json
3 import sys
4 
5 from os.path import join
6 
7 from generators import ecs_helpers
8 from schema.cleaner import field_or_multi_field_datatype_defaults
9 from schema.oss import TYPE_FALLBACKS
10 
11 
12 # Composable Template
13 
14 def generate(ecs_nested, ecs_version, out_dir, mapping_settings_file):
15     """This generates all artifacts for the composable template approach"""
16     all_component_templates(ecs_nested, ecs_version, out_dir)
17     component_names = component_name_convention(ecs_version, ecs_nested)
18     save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file)
19 
20 
21 def save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file):
22     """Generate the master sample composable template"""
23     template = {
24         "index_patterns": ["try-ecs-*"],
25         "composed_of": component_names,
26         "priority": 1,  # Very low, as this is a sample template
27         "_meta": {
28             "ecs_version": ecs_version,
29             "description": "Sample composable template that includes all ECS fields"
30         },
31         "template": {
32             "settings": {
33                 "index": {
34                     "mapping": {
35                         "total_fields": {
36                             "limit": 2000
37                         }
38                     }
39                 }
40             },
41             "mappings": mapping_settings(mapping_settings_file)
42         }
43     }
44     filename = join(out_dir, "elasticsearch/template.json")
45     save_json(filename, template)
46 
47 
48 def all_component_templates(ecs_nested, ecs_version, out_dir):
49     """Generate one component template per field set"""
50     component_dir = join(out_dir, 'elasticsearch/component')
51     ecs_helpers.make_dirs(component_dir)
52 
53     for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():
54         field_mappings = {}
55         for (flat_name, field) in fieldset['fields'].items():
56             name_parts = flat_name.split('.')
57             dict_add_nested(field_mappings, name_parts, entry_for(field))
58 
59         save_component_template(fieldset_name, ecs_version, component_dir, field_mappings)
60 
61 
62 def save_component_template(template_name, ecs_version, out_dir, field_mappings):
63     filename = join(out_dir, template_name) + ".json"
64     reference_url = "https://www.elastic.co/guide/en/ecs/current/ecs-{}.html".format(template_name)
65 
66     template = {
67         'template': {'mappings': {'properties': field_mappings}},
68         '_meta': {
69             'ecs_version': ecs_version,
70             'documentation': reference_url
71         }
72     }
73     save_json(filename, template)
74 
75 
76 def component_name_convention(ecs_version, ecs_nested):
77     version = ecs_version.replace('+', '-')
78     names = []
79     for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():
80         names.append("ecs_{}_{}".format(version, fieldset_name))
81     return names
82 
83 
84 def candidate_components(ecs_nested):
85     """Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False"""
86     components = {}
87     for (fieldset_name, fieldset) in ecs_nested.items():
88         if fieldset.get('reusable', None):
89             if not fieldset['reusable']['top_level']:
90                 continue
91         components[fieldset_name] = fieldset
92     return components
93 
94 
95 # Legacy template
96 
97 
98 def generate_legacy(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file):
99     """Generate the legacy index template"""
100     field_mappings = {}
101     for flat_name in sorted(ecs_flat):
102         field = ecs_flat[flat_name]
103         name_parts = flat_name.split('.')
104         dict_add_nested(field_mappings, name_parts, entry_for(field))
105 
106     mappings_section = mapping_settings(mapping_settings_file)
107     mappings_section['properties'] = field_mappings
108 
109     generate_legacy_template_version(6, ecs_version, mappings_section, out_dir, template_settings_file)
110     generate_legacy_template_version(7, ecs_version, mappings_section, out_dir, template_settings_file)
111 
112 
113 def generate_legacy_template_version(es_version, ecs_version, mappings_section, out_dir, template_settings_file):
114     ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(es_version)))
115     template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)
116 
117     filename = join(out_dir, "elasticsearch/{}/template.json".format(es_version))
118     save_json(filename, template)
119 
120 
121 # Common helpers
122 
123 
124 def dict_add_nested(dct, name_parts, value):
125     current_nesting = name_parts[0]
126     rest_name_parts = name_parts[1:]
127     if len(rest_name_parts) > 0:
128         dct.setdefault(current_nesting, {})
129         dct[current_nesting].setdefault('properties', {})
130 
131         dict_add_nested(
132             dct[current_nesting]['properties'],
133             rest_name_parts,
134             value)
135 
136     else:
137         if current_nesting in dct and 'type' in value and 'object' == value['type']:
138             return
139         dct[current_nesting] = value
140 
141 
142 def entry_for(field):
143     field_entry = {'type': field['type']}
144     try:
145         if field['type'] == 'object' or field['type'] == 'nested':
146             if 'enabled' in field and not field['enabled']:
147                 ecs_helpers.dict_copy_existing_keys(field, field_entry, ['enabled'])
148         # the index field is only valid for field types that are not object and nested
149         elif 'index' in field and not field['index']:
150             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['index', 'doc_values'])
151 
152         if field['type'] == 'keyword':
153             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['ignore_above'])
154         elif field['type'] == 'constant_keyword':
155             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['value'])
156         elif field['type'] == 'text':
157             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['norms'])
158         elif field['type'] == 'alias':
159             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['path'])
160         elif field['type'] == 'scaled_float':
161             ecs_helpers.dict_copy_existing_keys(field, field_entry, ['scaling_factor'])
162 
163         if 'multi_fields' in field:
164             field_entry['fields'] = {}
165             for mf in field['multi_fields']:
166                 mf_type = mf['type']
167                 mf_entry = {'type': mf_type}
168                 if mf_type == 'keyword':
169                     ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['normalizer', 'ignore_above'])
170                 elif mf_type == 'text':
171                     ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms'])
172                 field_entry['fields'][mf['name']] = mf_entry
173 
174     except KeyError as ex:
175         print("Exception {} occurred for field {}".format(ex, field))
176         raise ex
177     return field_entry
178 
179 
180 def mapping_settings(mapping_settings_file):
181     if mapping_settings_file:
182         with open(mapping_settings_file) as f:
183             mappings = json.load(f)
184     else:
185         mappings = default_mapping_settings()
186     return mappings
187 
188 
189 def template_settings(es_version, ecs_version, mappings_section, template_settings_file):
190     if template_settings_file:
191         with open(template_settings_file) as f:
192             template = json.load(f)
193     else:
194         template = default_template_settings(ecs_version)
195 
196     if es_version == 6:
197         mappings_section = copy.deepcopy(mappings_section)
198         es6_type_fallback(mappings_section['properties'])
199 
200         # error.stack_trace needs special handling to set
201         # index: false and doc_values: false
202         error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']
203         error_stack_trace_mappings.setdefault('index', False)
204         error_stack_trace_mappings.setdefault('doc_values', False)
205 
206         template['mappings'] = {'_doc': mappings_section}
207     else:
208         template['mappings'] = mappings_section
209 
210     # _meta can't be at template root in legacy templates, so moving back to mappings section
211     mappings_section['_meta'] = template.pop('_meta')
212 
213     return template
214 
215 
216 def save_json(file, data):
217     open_mode = "wb"
218     if sys.version_info >= (3, 0):
219         open_mode = "w"
220     with open(file, open_mode) as jsonfile:
221         jsonfile.write(json.dumps(data, indent=2, sort_keys=True))
222 
223 
224 def default_template_settings(ecs_version):
225     return {
226         "index_patterns": ["try-ecs-*"],
227         "_meta": {"version": ecs_version},
228         "order": 1,
229         "settings": {
230             "index": {
231                 "mapping": {
232                     "total_fields": {
233                         "limit": 10000
234                     }
235                 },
236                 "refresh_interval": "5s"
237             }
238         }
239     }
240 
241 
242 def default_mapping_settings():
243     return {
244         "date_detection": False,
245         "dynamic_templates": [
246             {
247                 "strings_as_keyword": {
248                     "mapping": {
249                         "ignore_above": 1024,
250                         "type": "keyword"
251                     },
252                     "match_mapping_type": "string"
253                 }
254             }
255         ]
256     }
257 
258 
259 def es6_type_fallback(mappings):
260     '''
261     Visits each leaf in mappings object and fallback to an
262     Elasticsearch 6.x supported type.
263 
264     Since a field like `wildcard` won't have the same defaults as
265     a `keyword` field, we must add any missing defaults.
266     '''
267 
268     for (name, details) in mappings.items():
269         if 'type' in details:
270             fallback_type = TYPE_FALLBACKS.get(details['type'])
271             if fallback_type:
272                 mappings[name]['type'] = fallback_type
273                 field_or_multi_field_datatype_defaults(mappings[name])
274         if 'properties' in details:
275             es6_type_fallback(details['properties'])
276
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py
--- a/scripts/generators/es_template.py
+++ b/scripts/generators/es_template.py
@@ -198,17 +198,23 @@
         es6_type_fallback(mappings_section['properties'])
 
         # error.stack_trace needs special handling to set
-        # index: false and doc_values: false
-        error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']
-        error_stack_trace_mappings.setdefault('index', False)
-        error_stack_trace_mappings.setdefault('doc_values', False)
+        # index: false and doc_values: false if the field
+        # is present in the mappings
+        try:
+            error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']
+            error_stack_trace_mappings.setdefault('index', False)
+            error_stack_trace_mappings.setdefault('doc_values', False)
+        except KeyError:
+            pass
 
         template['mappings'] = {'_doc': mappings_section}
     else:
         template['mappings'] = mappings_section
 
     # _meta can't be at template root in legacy templates, so moving back to mappings section
-    mappings_section['_meta'] = template.pop('_meta')
+    # if present
+    if '_meta' in template:
+        mappings_section['_meta'] = template.pop('_meta')
 
     return template

{"golden_diff": "diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py\n--- a/scripts/generators/es_template.py\n+++ b/scripts/generators/es_template.py\n@@ -198,17 +198,23 @@\n         es6_type_fallback(mappings_section['properties'])\n \n         # error.stack_trace needs special handling to set\n-        # index: false and doc_values: false\n-        error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']\n-        error_stack_trace_mappings.setdefault('index', False)\n-        error_stack_trace_mappings.setdefault('doc_values', False)\n+        # index: false and doc_values: false if the field\n+        # is present in the mappings\n+        try:\n+            error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']\n+            error_stack_trace_mappings.setdefault('index', False)\n+            error_stack_trace_mappings.setdefault('doc_values', False)\n+        except KeyError:\n+            pass\n \n         template['mappings'] = {'_doc': mappings_section}\n     else:\n         template['mappings'] = mappings_section\n \n     # _meta can't be at template root in legacy templates, so moving back to mappings section\n-    mappings_section['_meta'] = template.pop('_meta')\n+    # if present\n+    if '_meta' in template:\n+        mappings_section['_meta'] = template.pop('_meta')\n \n     return template\n", "issue": "Error when using usage-example with generator.py\n<!--\r\nGitHub is reserved for bug reports and feature requests; it is not the place\r\nfor general questions. If you have a general question, please visit the\r\ndiscuss forums: https://discuss.elastic.co/tag/ecs-elastic-common-schema.\r\n\r\nPlease fill in the following details to help us reproduce the bug:\r\n-->\r\n\r\n**Description of the problem including expected versus actual behavior**:\r\n\r\nTrying `script/generator.py` for the first time, I stumble upon a stack trace throwing `KeyError: 'error'`. I initially though I maybe passed wrong parameters but it looks like example documented in `USAGE.md` also have the same issue.\r\n\r\nThanks for writing this tool, I'm eager to use it since it can greatly help me to generate/maintain configuration for custom Beats indices. Let me know if I can somehow help you further.\r\n\r\n**Steps to reproduce**:\r\n\r\n1. `git clone https://github.com/elastic/ecs && cd ecs`\r\n2. Running `generator.py` as documented in `USAGE.md`:\r\n```sh\r\n$ python3 scripts/generator.py --ref v1.6.0 \\\r\n  --subset            usage-example/fields/subset.yml \\\r\n  --include           usage-example/fields/custom/ \\\r\n  --out               usage-example/ \\\r\n  --template-settings usage-example/fields/template-settings.json \\\r\n  --mapping-settings  usage-example/fields/mapping-settings.json\r\n```\r\n\r\n**Provide logs (if relevant)**:\r\n\r\n```\r\nLoading schemas from git ref v1.6.0\r\nRunning generator. ECS version 1.6.0\r\nLoading user defined schemas: ['usage-example/fields/custom/']\r\n/Users/florent/Sources/ecs/scripts/schema/cleaner.py:185: UserWarning: Example value for field `header_flags` contains an object or array which must be quoted to avoid YAML interpretation.\r\n\r\nThis will cause an exception when running in strict mode.\r\nWarning check:\r\n  check_example_value(field, strict=strict_mode)\r\n/Users/florent/Sources/ecs/scripts/schema/cleaner.py:185: UserWarning: Example value for field `resolved_ip` contains an object or array which must be quoted to avoid YAML interpretation.\r\n[...]\r\nTraceback (most recent call last):\r\n  File \"/Users/florent/Sources/ecs/scripts/generator.py\", line 106, in <module>\r\n    main()\r\n  File \"/Users/florent/Sources/ecs/scripts/generator.py\", line 60, in main\r\n    es_template.generate_legacy(flat, ecs_version, out_dir, args.template_settings, args.mapping_settings)\r\n  File \"/Users/florent/Sources/ecs/scripts/generators/es_template.py\", line 109, in generate_legacy\r\n    generate_legacy_template_version(6, ecs_version, mappings_section, out_dir, template_settings_file)\r\n  File \"/Users/florent/Sources/ecs/scripts/generators/es_template.py\", line 115, in generate_legacy_template_version\r\n    template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)\r\n  File \"/Users/florent/Sources/ecs/scripts/generators/es_template.py\", line 202, in template_settings\r\n    error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']\r\nKeyError: 'error'\r\n```\r\n\r\n**Any additional context:**\r\n\r\n```\r\n$ git rev-parse HEAD\r\n83ced31c9da6b3e5599b01e319a827a7941e6439\r\n\r\n$ python3 --version\r\nPython 3.9.0\r\n\r\n$ pip3 list | grep -i -E '(yaml|pep|mock|git|jinja)'\r\nautopep8      1.4.4\r\ngitdb         4.0.5\r\nGitPython     3.1.2\r\nJinja2        2.11.2\r\nmock          4.0.2\r\nPyYAML        5.3b1\r\nyamllint      1.19.0\r\n```\n", "before_files": [{"content": "import copy\nimport json\nimport sys\n\nfrom os.path import join\n\nfrom generators import ecs_helpers\nfrom schema.cleaner import field_or_multi_field_datatype_defaults\nfrom schema.oss import TYPE_FALLBACKS\n\n\n# Composable Template\n\ndef generate(ecs_nested, ecs_version, out_dir, mapping_settings_file):\n    \"\"\"This generates all artifacts for the composable template approach\"\"\"\n    all_component_templates(ecs_nested, ecs_version, out_dir)\n    component_names = component_name_convention(ecs_version, ecs_nested)\n    save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file)\n\n\ndef save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file):\n    \"\"\"Generate the master sample composable template\"\"\"\n    template = {\n        \"index_patterns\": [\"try-ecs-*\"],\n        \"composed_of\": component_names,\n        \"priority\": 1,  # Very low, as this is a sample template\n        \"_meta\": {\n            \"ecs_version\": ecs_version,\n            \"description\": \"Sample composable template that includes all ECS fields\"\n        },\n        \"template\": {\n            \"settings\": {\n                \"index\": {\n                    \"mapping\": {\n                        \"total_fields\": {\n                            \"limit\": 2000\n                        }\n                    }\n                }\n            },\n            \"mappings\": mapping_settings(mapping_settings_file)\n        }\n    }\n    filename = join(out_dir, \"elasticsearch/template.json\")\n    save_json(filename, template)\n\n\ndef all_component_templates(ecs_nested, ecs_version, out_dir):\n    \"\"\"Generate one component template per field set\"\"\"\n    component_dir = join(out_dir, 'elasticsearch/component')\n    ecs_helpers.make_dirs(component_dir)\n\n    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():\n        field_mappings = {}\n        for (flat_name, field) in fieldset['fields'].items():\n            name_parts = flat_name.split('.')\n            dict_add_nested(field_mappings, name_parts, entry_for(field))\n\n        save_component_template(fieldset_name, ecs_version, component_dir, field_mappings)\n\n\ndef save_component_template(template_name, ecs_version, out_dir, field_mappings):\n    filename = join(out_dir, template_name) + \".json\"\n    reference_url = \"https://www.elastic.co/guide/en/ecs/current/ecs-{}.html\".format(template_name)\n\n    template = {\n        'template': {'mappings': {'properties': field_mappings}},\n        '_meta': {\n            'ecs_version': ecs_version,\n            'documentation': reference_url\n        }\n    }\n    save_json(filename, template)\n\n\ndef component_name_convention(ecs_version, ecs_nested):\n    version = ecs_version.replace('+', '-')\n    names = []\n    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():\n        names.append(\"ecs_{}_{}\".format(version, fieldset_name))\n    return names\n\n\ndef candidate_components(ecs_nested):\n    \"\"\"Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False\"\"\"\n    components = {}\n    for (fieldset_name, fieldset) in ecs_nested.items():\n        if fieldset.get('reusable', None):\n            if not fieldset['reusable']['top_level']:\n                continue\n        components[fieldset_name] = fieldset\n    return components\n\n\n# Legacy template\n\n\ndef generate_legacy(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file):\n    \"\"\"Generate the legacy index template\"\"\"\n    field_mappings = {}\n    for flat_name in sorted(ecs_flat):\n        field = ecs_flat[flat_name]\n        name_parts = flat_name.split('.')\n        dict_add_nested(field_mappings, name_parts, entry_for(field))\n\n    mappings_section = mapping_settings(mapping_settings_file)\n    mappings_section['properties'] = field_mappings\n\n    generate_legacy_template_version(6, ecs_version, mappings_section, out_dir, template_settings_file)\n    generate_legacy_template_version(7, ecs_version, mappings_section, out_dir, template_settings_file)\n\n\ndef generate_legacy_template_version(es_version, ecs_version, mappings_section, out_dir, template_settings_file):\n    ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(es_version)))\n    template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)\n\n    filename = join(out_dir, \"elasticsearch/{}/template.json\".format(es_version))\n    save_json(filename, template)\n\n\n# Common helpers\n\n\ndef dict_add_nested(dct, name_parts, value):\n    current_nesting = name_parts[0]\n    rest_name_parts = name_parts[1:]\n    if len(rest_name_parts) > 0:\n        dct.setdefault(current_nesting, {})\n        dct[current_nesting].setdefault('properties', {})\n\n        dict_add_nested(\n            dct[current_nesting]['properties'],\n            rest_name_parts,\n            value)\n\n    else:\n        if current_nesting in dct and 'type' in value and 'object' == value['type']:\n            return\n        dct[current_nesting] = value\n\n\ndef entry_for(field):\n    field_entry = {'type': field['type']}\n    try:\n        if field['type'] == 'object' or field['type'] == 'nested':\n            if 'enabled' in field and not field['enabled']:\n                ecs_helpers.dict_copy_existing_keys(field, field_entry, ['enabled'])\n        # the index field is only valid for field types that are not object and nested\n        elif 'index' in field and not field['index']:\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['index', 'doc_values'])\n\n        if field['type'] == 'keyword':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['ignore_above'])\n        elif field['type'] == 'constant_keyword':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['value'])\n        elif field['type'] == 'text':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['norms'])\n        elif field['type'] == 'alias':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['path'])\n        elif field['type'] == 'scaled_float':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['scaling_factor'])\n\n        if 'multi_fields' in field:\n            field_entry['fields'] = {}\n            for mf in field['multi_fields']:\n                mf_type = mf['type']\n                mf_entry = {'type': mf_type}\n                if mf_type == 'keyword':\n                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['normalizer', 'ignore_above'])\n                elif mf_type == 'text':\n                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms'])\n                field_entry['fields'][mf['name']] = mf_entry\n\n    except KeyError as ex:\n        print(\"Exception {} occurred for field {}\".format(ex, field))\n        raise ex\n    return field_entry\n\n\ndef mapping_settings(mapping_settings_file):\n    if mapping_settings_file:\n        with open(mapping_settings_file) as f:\n            mappings = json.load(f)\n    else:\n        mappings = default_mapping_settings()\n    return mappings\n\n\ndef template_settings(es_version, ecs_version, mappings_section, template_settings_file):\n    if template_settings_file:\n        with open(template_settings_file) as f:\n            template = json.load(f)\n    else:\n        template = default_template_settings(ecs_version)\n\n    if es_version == 6:\n        mappings_section = copy.deepcopy(mappings_section)\n        es6_type_fallback(mappings_section['properties'])\n\n        # error.stack_trace needs special handling to set\n        # index: false and doc_values: false\n        error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']\n        error_stack_trace_mappings.setdefault('index', False)\n        error_stack_trace_mappings.setdefault('doc_values', False)\n\n        template['mappings'] = {'_doc': mappings_section}\n    else:\n        template['mappings'] = mappings_section\n\n    # _meta can't be at template root in legacy templates, so moving back to mappings section\n    mappings_section['_meta'] = template.pop('_meta')\n\n    return template\n\n\ndef save_json(file, data):\n    open_mode = \"wb\"\n    if sys.version_info >= (3, 0):\n        open_mode = \"w\"\n    with open(file, open_mode) as jsonfile:\n        jsonfile.write(json.dumps(data, indent=2, sort_keys=True))\n\n\ndef default_template_settings(ecs_version):\n    return {\n        \"index_patterns\": [\"try-ecs-*\"],\n        \"_meta\": {\"version\": ecs_version},\n        \"order\": 1,\n        \"settings\": {\n            \"index\": {\n                \"mapping\": {\n                    \"total_fields\": {\n                        \"limit\": 10000\n                    }\n                },\n                \"refresh_interval\": \"5s\"\n            }\n        }\n    }\n\n\ndef default_mapping_settings():\n    return {\n        \"date_detection\": False,\n        \"dynamic_templates\": [\n            {\n                \"strings_as_keyword\": {\n                    \"mapping\": {\n                        \"ignore_above\": 1024,\n                        \"type\": \"keyword\"\n                    },\n                    \"match_mapping_type\": \"string\"\n                }\n            }\n        ]\n    }\n\n\ndef es6_type_fallback(mappings):\n    '''\n    Visits each leaf in mappings object and fallback to an\n    Elasticsearch 6.x supported type.\n\n    Since a field like `wildcard` won't have the same defaults as\n    a `keyword` field, we must add any missing defaults.\n    '''\n\n    for (name, details) in mappings.items():\n        if 'type' in details:\n            fallback_type = TYPE_FALLBACKS.get(details['type'])\n            if fallback_type:\n                mappings[name]['type'] = fallback_type\n                field_or_multi_field_datatype_defaults(mappings[name])\n        if 'properties' in details:\n            es6_type_fallback(details['properties'])\n", "path": "scripts/generators/es_template.py"}], "after_files": [{"content": "import copy\nimport json\nimport sys\n\nfrom os.path import join\n\nfrom generators import ecs_helpers\nfrom schema.cleaner import field_or_multi_field_datatype_defaults\nfrom schema.oss import TYPE_FALLBACKS\n\n\n# Composable Template\n\ndef generate(ecs_nested, ecs_version, out_dir, mapping_settings_file):\n    \"\"\"This generates all artifacts for the composable template approach\"\"\"\n    all_component_templates(ecs_nested, ecs_version, out_dir)\n    component_names = component_name_convention(ecs_version, ecs_nested)\n    save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file)\n\n\ndef save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file):\n    \"\"\"Generate the master sample composable template\"\"\"\n    template = {\n        \"index_patterns\": [\"try-ecs-*\"],\n        \"composed_of\": component_names,\n        \"priority\": 1,  # Very low, as this is a sample template\n        \"_meta\": {\n            \"ecs_version\": ecs_version,\n            \"description\": \"Sample composable template that includes all ECS fields\"\n        },\n        \"template\": {\n            \"settings\": {\n                \"index\": {\n                    \"mapping\": {\n                        \"total_fields\": {\n                            \"limit\": 2000\n                        }\n                    }\n                }\n            },\n            \"mappings\": mapping_settings(mapping_settings_file)\n        }\n    }\n    filename = join(out_dir, \"elasticsearch/template.json\")\n    save_json(filename, template)\n\n\ndef all_component_templates(ecs_nested, ecs_version, out_dir):\n    \"\"\"Generate one component template per field set\"\"\"\n    component_dir = join(out_dir, 'elasticsearch/component')\n    ecs_helpers.make_dirs(component_dir)\n\n    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():\n        field_mappings = {}\n        for (flat_name, field) in fieldset['fields'].items():\n            name_parts = flat_name.split('.')\n            dict_add_nested(field_mappings, name_parts, entry_for(field))\n\n        save_component_template(fieldset_name, ecs_version, component_dir, field_mappings)\n\n\ndef save_component_template(template_name, ecs_version, out_dir, field_mappings):\n    filename = join(out_dir, template_name) + \".json\"\n    reference_url = \"https://www.elastic.co/guide/en/ecs/current/ecs-{}.html\".format(template_name)\n\n    template = {\n        'template': {'mappings': {'properties': field_mappings}},\n        '_meta': {\n            'ecs_version': ecs_version,\n            'documentation': reference_url\n        }\n    }\n    save_json(filename, template)\n\n\ndef component_name_convention(ecs_version, ecs_nested):\n    version = ecs_version.replace('+', '-')\n    names = []\n    for (fieldset_name, fieldset) in candidate_components(ecs_nested).items():\n        names.append(\"ecs_{}_{}\".format(version, fieldset_name))\n    return names\n\n\ndef candidate_components(ecs_nested):\n    \"\"\"Returns same structure as ecs_nested, but skips all field sets with reusable.top_level: False\"\"\"\n    components = {}\n    for (fieldset_name, fieldset) in ecs_nested.items():\n        if fieldset.get('reusable', None):\n            if not fieldset['reusable']['top_level']:\n                continue\n        components[fieldset_name] = fieldset\n    return components\n\n\n# Legacy template\n\n\ndef generate_legacy(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file):\n    \"\"\"Generate the legacy index template\"\"\"\n    field_mappings = {}\n    for flat_name in sorted(ecs_flat):\n        field = ecs_flat[flat_name]\n        name_parts = flat_name.split('.')\n        dict_add_nested(field_mappings, name_parts, entry_for(field))\n\n    mappings_section = mapping_settings(mapping_settings_file)\n    mappings_section['properties'] = field_mappings\n\n    generate_legacy_template_version(6, ecs_version, mappings_section, out_dir, template_settings_file)\n    generate_legacy_template_version(7, ecs_version, mappings_section, out_dir, template_settings_file)\n\n\ndef generate_legacy_template_version(es_version, ecs_version, mappings_section, out_dir, template_settings_file):\n    ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(es_version)))\n    template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)\n\n    filename = join(out_dir, \"elasticsearch/{}/template.json\".format(es_version))\n    save_json(filename, template)\n\n\n# Common helpers\n\n\ndef dict_add_nested(dct, name_parts, value):\n    current_nesting = name_parts[0]\n    rest_name_parts = name_parts[1:]\n    if len(rest_name_parts) > 0:\n        dct.setdefault(current_nesting, {})\n        dct[current_nesting].setdefault('properties', {})\n\n        dict_add_nested(\n            dct[current_nesting]['properties'],\n            rest_name_parts,\n            value)\n\n    else:\n        if current_nesting in dct and 'type' in value and 'object' == value['type']:\n            return\n        dct[current_nesting] = value\n\n\ndef entry_for(field):\n    field_entry = {'type': field['type']}\n    try:\n        if field['type'] == 'object' or field['type'] == 'nested':\n            if 'enabled' in field and not field['enabled']:\n                ecs_helpers.dict_copy_existing_keys(field, field_entry, ['enabled'])\n        # the index field is only valid for field types that are not object and nested\n        elif 'index' in field and not field['index']:\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['index', 'doc_values'])\n\n        if field['type'] == 'keyword':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['ignore_above'])\n        elif field['type'] == 'constant_keyword':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['value'])\n        elif field['type'] == 'text':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['norms'])\n        elif field['type'] == 'alias':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['path'])\n        elif field['type'] == 'scaled_float':\n            ecs_helpers.dict_copy_existing_keys(field, field_entry, ['scaling_factor'])\n\n        if 'multi_fields' in field:\n            field_entry['fields'] = {}\n            for mf in field['multi_fields']:\n                mf_type = mf['type']\n                mf_entry = {'type': mf_type}\n                if mf_type == 'keyword':\n                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['normalizer', 'ignore_above'])\n                elif mf_type == 'text':\n                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms'])\n                field_entry['fields'][mf['name']] = mf_entry\n\n    except KeyError as ex:\n        print(\"Exception {} occurred for field {}\".format(ex, field))\n        raise ex\n    return field_entry\n\n\ndef mapping_settings(mapping_settings_file):\n    if mapping_settings_file:\n        with open(mapping_settings_file) as f:\n            mappings = json.load(f)\n    else:\n        mappings = default_mapping_settings()\n    return mappings\n\n\ndef template_settings(es_version, ecs_version, mappings_section, template_settings_file):\n    if template_settings_file:\n        with open(template_settings_file) as f:\n            template = json.load(f)\n    else:\n        template = default_template_settings(ecs_version)\n\n    if es_version == 6:\n        mappings_section = copy.deepcopy(mappings_section)\n        es6_type_fallback(mappings_section['properties'])\n\n        # error.stack_trace needs special handling to set\n        # index: false and doc_values: false if the field\n        # is present in the mappings\n        try:\n            error_stack_trace_mappings = mappings_section['properties']['error']['properties']['stack_trace']\n            error_stack_trace_mappings.setdefault('index', False)\n            error_stack_trace_mappings.setdefault('doc_values', False)\n        except KeyError:\n            pass\n\n        template['mappings'] = {'_doc': mappings_section}\n    else:\n        template['mappings'] = mappings_section\n\n    # _meta can't be at template root in legacy templates, so moving back to mappings section\n    # if present\n    if '_meta' in template:\n        mappings_section['_meta'] = template.pop('_meta')\n\n    return template\n\n\ndef save_json(file, data):\n    open_mode = \"wb\"\n    if sys.version_info >= (3, 0):\n        open_mode = \"w\"\n    with open(file, open_mode) as jsonfile:\n        jsonfile.write(json.dumps(data, indent=2, sort_keys=True))\n\n\ndef default_template_settings(ecs_version):\n    return {\n        \"index_patterns\": [\"try-ecs-*\"],\n        \"_meta\": {\"version\": ecs_version},\n        \"order\": 1,\n        \"settings\": {\n            \"index\": {\n                \"mapping\": {\n                    \"total_fields\": {\n                        \"limit\": 10000\n                    }\n                },\n                \"refresh_interval\": \"5s\"\n            }\n        }\n    }\n\n\ndef default_mapping_settings():\n    return {\n        \"date_detection\": False,\n        \"dynamic_templates\": [\n            {\n                \"strings_as_keyword\": {\n                    \"mapping\": {\n                        \"ignore_above\": 1024,\n                        \"type\": \"keyword\"\n                    },\n                    \"match_mapping_type\": \"string\"\n                }\n            }\n        ]\n    }\n\n\ndef es6_type_fallback(mappings):\n    '''\n    Visits each leaf in mappings object and fallback to an\n    Elasticsearch 6.x supported type.\n\n    Since a field like `wildcard` won't have the same defaults as\n    a `keyword` field, we must add any missing defaults.\n    '''\n\n    for (name, details) in mappings.items():\n        if 'type' in details:\n            fallback_type = TYPE_FALLBACKS.get(details['type'])\n            if fallback_type:\n                mappings[name]['type'] = fallback_type\n                field_or_multi_field_datatype_defaults(mappings[name])\n        if 'properties' in details:\n            es6_type_fallback(details['properties'])\n", "path": "scripts/generators/es_template.py"}]}