We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Remove `ignore_above` setting when not indexing field
Certain ECS fields are type `keyword` but are also not:

* searchable (`index: false`)
* used in aggregations, sorting, or scripting (`doc_values: false`)

The ECS tooling's behavior is to set `ignore_above: 1024` for all `keyword` fields.

However, `ignore_above` should not be present when `index: false` and `doc_values: false` are set on a field.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scripts/schema/cleaner.py`
Content:
```
1 import copy
2 
3 from generators import ecs_helpers
4 from schema import visitor
5 
6 # This script performs a few cleanup functions in place, within the deeply nested
7 # 'fields' structure passed to `clean(fields)`.
8 #
9 # What happens here:
10 #
11 # - check that mandatory attributes are present, without which we can't do much.
12 # - cleans things up, like stripping spaces, sorting arrays
13 # - makes lots of defaults explicit
14 # - pre-calculate a few additional helpful fields
15 # - converts shorthands into full representation (e.g. reuse locations)
16 #
17 # This script only deals with field sets themselves and the fields defined
18 # inside them. It doesn't perform field reuse, and therefore doesn't
19 # deal with final field names either.
20 
21 
22 def clean(fields, strict=False):
23     global strict_mode
24     strict_mode = strict
25     visitor.visit_fields(fields, fieldset_func=schema_cleanup, field_func=field_cleanup)
26 
27 
28 # Schema level cleanup
29 
30 
31 def schema_cleanup(schema):
32     # Sanity check first
33     schema_mandatory_attributes(schema)
34     # trailing space cleanup
35     ecs_helpers.dict_clean_string_values(schema['schema_details'])
36     ecs_helpers.dict_clean_string_values(schema['field_details'])
37     # Some defaults
38     schema['schema_details'].setdefault('group', 2)
39     schema['schema_details'].setdefault('root', False)
40     schema['field_details'].setdefault('type', 'group')
41     schema['field_details'].setdefault('short', schema['field_details']['description'])
42     if 'reusable' in schema['schema_details']:
43         # order to perform chained reuses. Set to 1 if it needs to happen earlier.
44         schema['schema_details']['reusable'].setdefault('order', 2)
45     # Precalculate stuff. Those can't be set in the YAML.
46     if schema['schema_details']['root']:
47         schema['schema_details']['prefix'] = ''
48     else:
49         schema['schema_details']['prefix'] = schema['field_details']['name'] + '.'
50     normalize_reuse_notation(schema)
51     # Final validity check if in strict mode
52     schema_assertions_and_warnings(schema)
53 
54 
55 SCHEMA_MANDATORY_ATTRIBUTES = ['name', 'title', 'description']
56 
57 
58 def schema_mandatory_attributes(schema):
59     """Ensures for the presence of the mandatory schema attributes and raises if any are missing"""
60     current_schema_attributes = sorted(list(schema['field_details'].keys()) +
61                                        list(schema['schema_details'].keys()))
62     missing_attributes = ecs_helpers.list_subtract(SCHEMA_MANDATORY_ATTRIBUTES, current_schema_attributes)
63     if len(missing_attributes) > 0:
64         msg = "Schema {} is missing the following mandatory attributes: {}.\nFound these: {}".format(
65             schema['field_details']['name'], ', '.join(missing_attributes), current_schema_attributes)
66         raise ValueError(msg)
67     if 'reusable' in schema['schema_details']:
68         reuse_attributes = sorted(schema['schema_details']['reusable'].keys())
69         missing_reuse_attributes = ecs_helpers.list_subtract(['expected', 'top_level'], reuse_attributes)
70         if len(missing_reuse_attributes) > 0:
71             msg = "Reusable schema {} is missing the following reuse attributes: {}.\nFound these: {}".format(
72                 schema['field_details']['name'], ', '.join(missing_reuse_attributes), reuse_attributes)
73             raise ValueError(msg)
74 
75 
76 def schema_assertions_and_warnings(schema):
77     """Additional checks on a fleshed out schema"""
78     single_line_short_description(schema, strict=strict_mode)
79     if 'beta' in schema['field_details']:
80         single_line_beta_description(schema, strict=strict_mode)
81     if 'reusable' in schema['schema_details']:
82         single_line_short_override_description(schema, strict=strict_mode)
83 
84 
85 def normalize_reuse_notation(schema):
86     """
87     Replace single word reuse shorthands from the schema YAMLs with the explicit {at: , as:} notation.
88 
89     When marking "user" as reusable under "destination" with the shorthand entry
90     `- destination`, this is expanded to the complete entry
91     `- { "at": "destination", "as": "user" }`.
92     The field set is thus nested at `destination.user.*`, with fields such as `destination.user.name`.
93 
94     The dictionary notation enables nesting a field set as a different name.
95     An example is nesting "process" fields to capture parent process details
96     at `process.parent.*`.
97     The dictionary notation `- { "at": "process", "as": "parent" }` will yield
98     fields such as `process.parent.pid`.
99     """
100     if 'reusable' not in schema['schema_details']:
101         return
102     schema_name = schema['field_details']['name']
103     reuse_entries = []
104     for reuse_entry in schema['schema_details']['reusable']['expected']:
105         if type(reuse_entry) is dict:  # Already explicit
106             if 'at' in reuse_entry and 'as' in reuse_entry:
107                 explicit_entry = reuse_entry
108             else:
109                 raise ValueError("When specifying reusable expected locations for {} " +
110                                  "with the dictionary notation, keys 'as' and 'at' are required. " +
111                                  "Got {}.".format(schema_name, reuse_entry))
112         else:  # Make it explicit
113             explicit_entry = {'at': reuse_entry, 'as': schema_name}
114         explicit_entry['full'] = explicit_entry['at'] + '.' + explicit_entry['as']
115         reuse_entries.append(explicit_entry)
116     schema['schema_details']['reusable']['expected'] = reuse_entries
117 
118 
119 # Field level cleanup
120 
121 
122 def field_cleanup(field):
123     field_mandatory_attributes(field)
124     if ecs_helpers.is_intermediate(field):
125         return
126     ecs_helpers.dict_clean_string_values(field['field_details'])
127     if 'allowed_values' in field['field_details']:
128         for allowed_value in field['field_details']['allowed_values']:
129             ecs_helpers.dict_clean_string_values(allowed_value)
130     field_defaults(field)
131     field_assertions_and_warnings(field)
132 
133 
134 def field_defaults(field):
135     field['field_details'].setdefault('short', field['field_details']['description'])
136     field['field_details'].setdefault('normalize', [])
137     field_or_multi_field_datatype_defaults(field['field_details'])
138     if 'multi_fields' in field['field_details']:
139         for mf in field['field_details']['multi_fields']:
140             field_or_multi_field_datatype_defaults(mf)
141             if 'name' not in mf:
142                 mf['name'] = mf['type']
143 
144 
145 def field_or_multi_field_datatype_defaults(field_details):
146     """Sets datatype-related defaults on a canonical field or multi-field entries."""
147     if field_details['type'] == 'keyword':
148         field_details.setdefault('ignore_above', 1024)
149     if field_details['type'] == 'text':
150         field_details.setdefault('norms', False)
151     # wildcard needs the index param stripped
152     if field_details['type'] == 'wildcard':
153         field_details.pop('index', None)
154     if 'index' in field_details and not field_details['index']:
155         field_details.setdefault('doc_values', False)
156 
157 
158 FIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']
159 ACCEPTABLE_FIELD_LEVELS = ['core', 'extended', 'custom']
160 
161 
162 def field_mandatory_attributes(field):
163     """Ensures for the presence of the mandatory field attributes and raises if any are missing"""
164     if ecs_helpers.is_intermediate(field):
165         return
166     current_field_attributes = sorted(field['field_details'].keys())
167     missing_attributes = ecs_helpers.list_subtract(FIELD_MANDATORY_ATTRIBUTES, current_field_attributes)
168 
169     # `alias` fields require a target `path` attribute.
170     if field['field_details'].get('type') == 'alias' and 'path' not in current_field_attributes:
171         missing_attributes.append('path')
172     # `scaled_float` fields require a `scaling_factor` attribute.
173     if field['field_details'].get('type') == 'scaled_float' and 'scaling_factor' not in current_field_attributes:
174         missing_attributes.append('scaling_factor')
175 
176     if len(missing_attributes) > 0:
177         msg = "Field is missing the following mandatory attributes: {}.\nFound these: {}.\nField details: {}"
178         raise ValueError(msg.format(', '.join(missing_attributes),
179                                     current_field_attributes, field))
180 
181 
182 def field_assertions_and_warnings(field):
183     """Additional checks on a fleshed out field"""
184     if not ecs_helpers.is_intermediate(field):
185         # check short description length if in strict mode
186         single_line_short_description(field, strict=strict_mode)
187         check_example_value(field, strict=strict_mode)
188         if 'beta' in field['field_details']:
189             single_line_beta_description(field, strict=strict_mode)
190         if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:
191             msg = "Invalid level for field '{}'.\nValue: {}\nAcceptable values: {}".format(
192                 field['field_details']['name'], field['field_details']['level'],
193                 ACCEPTABLE_FIELD_LEVELS)
194             raise ValueError(msg)
195 
196 # Common
197 
198 
199 SHORT_LIMIT = 120
200 
201 
202 def single_line_short_check(short_to_check, short_name):
203     short_length = len(short_to_check)
204     if "\n" in short_to_check or short_length > SHORT_LIMIT:
205         msg = "Short descriptions must be single line, and under {} characters (current length: {}).\n".format(
206             SHORT_LIMIT, short_length)
207         msg += "Offending field or field set: {}\nShort description:\n  {}".format(
208             short_name,
209             short_to_check)
210         return msg
211     return None
212 
213 
214 def single_line_short_description(schema_or_field, strict=True):
215     error = single_line_short_check(schema_or_field['field_details']['short'], schema_or_field['field_details']['name'])
216     if error:
217         if strict:
218             raise ValueError(error)
219         else:
220             ecs_helpers.strict_warning(error)
221 
222 
223 def single_line_short_override_description(schema_or_field, strict=True):
224     for field in schema_or_field['schema_details']['reusable']['expected']:
225         if not 'short_override' in field:
226             continue
227         error = single_line_short_check(field['short_override'], field['full'])
228         if error:
229             if strict:
230                 raise ValueError(error)
231             else:
232                 ecs_helpers.strict_warning(error)
233 
234 
235 def check_example_value(field, strict=True):
236     """
237     Checks if value of the example field is of type list or dict.
238     Fails or warns (depending on strict mode) if so.
239     """
240     example_value = field['field_details'].get('example', None)
241     if isinstance(example_value, (list, dict)):
242         name = field['field_details']['name']
243         msg = f"Example value for field `{name}` contains an object or array which must be quoted to avoid YAML interpretation."
244         if strict:
245             raise ValueError(msg)
246         else:
247             ecs_helpers.strict_warning(msg)
248 
249 
250 def single_line_beta_description(schema_or_field, strict=True):
251     if "\n" in schema_or_field['field_details']['beta']:
252         msg = "Beta descriptions must be single line.\n"
253         msg += f"Offending field or field set: {schema_or_field['field_details']['name']}"
254         if strict:
255             raise ValueError(msg)
256         else:
257             ecs_helpers.strict_warning(msg)
258
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py
--- a/scripts/schema/cleaner.py
+++ b/scripts/schema/cleaner.py
@@ -153,6 +153,7 @@
         field_details.pop('index', None)
     if 'index' in field_details and not field_details['index']:
         field_details.setdefault('doc_values', False)
+        field_details.pop('ignore_above', None)
 
 
 FIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']

{"golden_diff": "diff --git a/scripts/schema/cleaner.py b/scripts/schema/cleaner.py\n--- a/scripts/schema/cleaner.py\n+++ b/scripts/schema/cleaner.py\n@@ -153,6 +153,7 @@\n         field_details.pop('index', None)\n     if 'index' in field_details and not field_details['index']:\n         field_details.setdefault('doc_values', False)\n+        field_details.pop('ignore_above', None)\n \n \n FIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']\n", "issue": "Remove `ignore_above` setting when not indexing field\nCertain ECS fields are type `keyword` but are also not:\r\n\r\n* searchable (`index: false`)\r\n* used in aggregations, sorting, or scripting (`doc_values: false`)\r\n\r\nThe ECS tooling's behavior is to set `ignore_above: 1024` for all `keyword` fields.\r\n\r\nHowever, `ignore_above` should not be present when `index: false` and `doc_values: false` are set on a field.\r\n\n", "before_files": [{"content": "import copy\n\nfrom generators import ecs_helpers\nfrom schema import visitor\n\n# This script performs a few cleanup functions in place, within the deeply nested\n# 'fields' structure passed to `clean(fields)`.\n#\n# What happens here:\n#\n# - check that mandatory attributes are present, without which we can't do much.\n# - cleans things up, like stripping spaces, sorting arrays\n# - makes lots of defaults explicit\n# - pre-calculate a few additional helpful fields\n# - converts shorthands into full representation (e.g. reuse locations)\n#\n# This script only deals with field sets themselves and the fields defined\n# inside them. It doesn't perform field reuse, and therefore doesn't\n# deal with final field names either.\n\n\ndef clean(fields, strict=False):\n    global strict_mode\n    strict_mode = strict\n    visitor.visit_fields(fields, fieldset_func=schema_cleanup, field_func=field_cleanup)\n\n\n# Schema level cleanup\n\n\ndef schema_cleanup(schema):\n    # Sanity check first\n    schema_mandatory_attributes(schema)\n    # trailing space cleanup\n    ecs_helpers.dict_clean_string_values(schema['schema_details'])\n    ecs_helpers.dict_clean_string_values(schema['field_details'])\n    # Some defaults\n    schema['schema_details'].setdefault('group', 2)\n    schema['schema_details'].setdefault('root', False)\n    schema['field_details'].setdefault('type', 'group')\n    schema['field_details'].setdefault('short', schema['field_details']['description'])\n    if 'reusable' in schema['schema_details']:\n        # order to perform chained reuses. Set to 1 if it needs to happen earlier.\n        schema['schema_details']['reusable'].setdefault('order', 2)\n    # Precalculate stuff. Those can't be set in the YAML.\n    if schema['schema_details']['root']:\n        schema['schema_details']['prefix'] = ''\n    else:\n        schema['schema_details']['prefix'] = schema['field_details']['name'] + '.'\n    normalize_reuse_notation(schema)\n    # Final validity check if in strict mode\n    schema_assertions_and_warnings(schema)\n\n\nSCHEMA_MANDATORY_ATTRIBUTES = ['name', 'title', 'description']\n\n\ndef schema_mandatory_attributes(schema):\n    \"\"\"Ensures for the presence of the mandatory schema attributes and raises if any are missing\"\"\"\n    current_schema_attributes = sorted(list(schema['field_details'].keys()) +\n                                       list(schema['schema_details'].keys()))\n    missing_attributes = ecs_helpers.list_subtract(SCHEMA_MANDATORY_ATTRIBUTES, current_schema_attributes)\n    if len(missing_attributes) > 0:\n        msg = \"Schema {} is missing the following mandatory attributes: {}.\\nFound these: {}\".format(\n            schema['field_details']['name'], ', '.join(missing_attributes), current_schema_attributes)\n        raise ValueError(msg)\n    if 'reusable' in schema['schema_details']:\n        reuse_attributes = sorted(schema['schema_details']['reusable'].keys())\n        missing_reuse_attributes = ecs_helpers.list_subtract(['expected', 'top_level'], reuse_attributes)\n        if len(missing_reuse_attributes) > 0:\n            msg = \"Reusable schema {} is missing the following reuse attributes: {}.\\nFound these: {}\".format(\n                schema['field_details']['name'], ', '.join(missing_reuse_attributes), reuse_attributes)\n            raise ValueError(msg)\n\n\ndef schema_assertions_and_warnings(schema):\n    \"\"\"Additional checks on a fleshed out schema\"\"\"\n    single_line_short_description(schema, strict=strict_mode)\n    if 'beta' in schema['field_details']:\n        single_line_beta_description(schema, strict=strict_mode)\n    if 'reusable' in schema['schema_details']:\n        single_line_short_override_description(schema, strict=strict_mode)\n\n\ndef normalize_reuse_notation(schema):\n    \"\"\"\n    Replace single word reuse shorthands from the schema YAMLs with the explicit {at: , as:} notation.\n\n    When marking \"user\" as reusable under \"destination\" with the shorthand entry\n    `- destination`, this is expanded to the complete entry\n    `- { \"at\": \"destination\", \"as\": \"user\" }`.\n    The field set is thus nested at `destination.user.*`, with fields such as `destination.user.name`.\n\n    The dictionary notation enables nesting a field set as a different name.\n    An example is nesting \"process\" fields to capture parent process details\n    at `process.parent.*`.\n    The dictionary notation `- { \"at\": \"process\", \"as\": \"parent\" }` will yield\n    fields such as `process.parent.pid`.\n    \"\"\"\n    if 'reusable' not in schema['schema_details']:\n        return\n    schema_name = schema['field_details']['name']\n    reuse_entries = []\n    for reuse_entry in schema['schema_details']['reusable']['expected']:\n        if type(reuse_entry) is dict:  # Already explicit\n            if 'at' in reuse_entry and 'as' in reuse_entry:\n                explicit_entry = reuse_entry\n            else:\n                raise ValueError(\"When specifying reusable expected locations for {} \" +\n                                 \"with the dictionary notation, keys 'as' and 'at' are required. \" +\n                                 \"Got {}.\".format(schema_name, reuse_entry))\n        else:  # Make it explicit\n            explicit_entry = {'at': reuse_entry, 'as': schema_name}\n        explicit_entry['full'] = explicit_entry['at'] + '.' + explicit_entry['as']\n        reuse_entries.append(explicit_entry)\n    schema['schema_details']['reusable']['expected'] = reuse_entries\n\n\n# Field level cleanup\n\n\ndef field_cleanup(field):\n    field_mandatory_attributes(field)\n    if ecs_helpers.is_intermediate(field):\n        return\n    ecs_helpers.dict_clean_string_values(field['field_details'])\n    if 'allowed_values' in field['field_details']:\n        for allowed_value in field['field_details']['allowed_values']:\n            ecs_helpers.dict_clean_string_values(allowed_value)\n    field_defaults(field)\n    field_assertions_and_warnings(field)\n\n\ndef field_defaults(field):\n    field['field_details'].setdefault('short', field['field_details']['description'])\n    field['field_details'].setdefault('normalize', [])\n    field_or_multi_field_datatype_defaults(field['field_details'])\n    if 'multi_fields' in field['field_details']:\n        for mf in field['field_details']['multi_fields']:\n            field_or_multi_field_datatype_defaults(mf)\n            if 'name' not in mf:\n                mf['name'] = mf['type']\n\n\ndef field_or_multi_field_datatype_defaults(field_details):\n    \"\"\"Sets datatype-related defaults on a canonical field or multi-field entries.\"\"\"\n    if field_details['type'] == 'keyword':\n        field_details.setdefault('ignore_above', 1024)\n    if field_details['type'] == 'text':\n        field_details.setdefault('norms', False)\n    # wildcard needs the index param stripped\n    if field_details['type'] == 'wildcard':\n        field_details.pop('index', None)\n    if 'index' in field_details and not field_details['index']:\n        field_details.setdefault('doc_values', False)\n\n\nFIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']\nACCEPTABLE_FIELD_LEVELS = ['core', 'extended', 'custom']\n\n\ndef field_mandatory_attributes(field):\n    \"\"\"Ensures for the presence of the mandatory field attributes and raises if any are missing\"\"\"\n    if ecs_helpers.is_intermediate(field):\n        return\n    current_field_attributes = sorted(field['field_details'].keys())\n    missing_attributes = ecs_helpers.list_subtract(FIELD_MANDATORY_ATTRIBUTES, current_field_attributes)\n\n    # `alias` fields require a target `path` attribute.\n    if field['field_details'].get('type') == 'alias' and 'path' not in current_field_attributes:\n        missing_attributes.append('path')\n    # `scaled_float` fields require a `scaling_factor` attribute.\n    if field['field_details'].get('type') == 'scaled_float' and 'scaling_factor' not in current_field_attributes:\n        missing_attributes.append('scaling_factor')\n\n    if len(missing_attributes) > 0:\n        msg = \"Field is missing the following mandatory attributes: {}.\\nFound these: {}.\\nField details: {}\"\n        raise ValueError(msg.format(', '.join(missing_attributes),\n                                    current_field_attributes, field))\n\n\ndef field_assertions_and_warnings(field):\n    \"\"\"Additional checks on a fleshed out field\"\"\"\n    if not ecs_helpers.is_intermediate(field):\n        # check short description length if in strict mode\n        single_line_short_description(field, strict=strict_mode)\n        check_example_value(field, strict=strict_mode)\n        if 'beta' in field['field_details']:\n            single_line_beta_description(field, strict=strict_mode)\n        if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:\n            msg = \"Invalid level for field '{}'.\\nValue: {}\\nAcceptable values: {}\".format(\n                field['field_details']['name'], field['field_details']['level'],\n                ACCEPTABLE_FIELD_LEVELS)\n            raise ValueError(msg)\n\n# Common\n\n\nSHORT_LIMIT = 120\n\n\ndef single_line_short_check(short_to_check, short_name):\n    short_length = len(short_to_check)\n    if \"\\n\" in short_to_check or short_length > SHORT_LIMIT:\n        msg = \"Short descriptions must be single line, and under {} characters (current length: {}).\\n\".format(\n            SHORT_LIMIT, short_length)\n        msg += \"Offending field or field set: {}\\nShort description:\\n  {}\".format(\n            short_name,\n            short_to_check)\n        return msg\n    return None\n\n\ndef single_line_short_description(schema_or_field, strict=True):\n    error = single_line_short_check(schema_or_field['field_details']['short'], schema_or_field['field_details']['name'])\n    if error:\n        if strict:\n            raise ValueError(error)\n        else:\n            ecs_helpers.strict_warning(error)\n\n\ndef single_line_short_override_description(schema_or_field, strict=True):\n    for field in schema_or_field['schema_details']['reusable']['expected']:\n        if not 'short_override' in field:\n            continue\n        error = single_line_short_check(field['short_override'], field['full'])\n        if error:\n            if strict:\n                raise ValueError(error)\n            else:\n                ecs_helpers.strict_warning(error)\n\n\ndef check_example_value(field, strict=True):\n    \"\"\"\n    Checks if value of the example field is of type list or dict.\n    Fails or warns (depending on strict mode) if so.\n    \"\"\"\n    example_value = field['field_details'].get('example', None)\n    if isinstance(example_value, (list, dict)):\n        name = field['field_details']['name']\n        msg = f\"Example value for field `{name}` contains an object or array which must be quoted to avoid YAML interpretation.\"\n        if strict:\n            raise ValueError(msg)\n        else:\n            ecs_helpers.strict_warning(msg)\n\n\ndef single_line_beta_description(schema_or_field, strict=True):\n    if \"\\n\" in schema_or_field['field_details']['beta']:\n        msg = \"Beta descriptions must be single line.\\n\"\n        msg += f\"Offending field or field set: {schema_or_field['field_details']['name']}\"\n        if strict:\n            raise ValueError(msg)\n        else:\n            ecs_helpers.strict_warning(msg)\n", "path": "scripts/schema/cleaner.py"}], "after_files": [{"content": "import copy\n\nfrom generators import ecs_helpers\nfrom schema import visitor\n\n# This script performs a few cleanup functions in place, within the deeply nested\n# 'fields' structure passed to `clean(fields)`.\n#\n# What happens here:\n#\n# - check that mandatory attributes are present, without which we can't do much.\n# - cleans things up, like stripping spaces, sorting arrays\n# - makes lots of defaults explicit\n# - pre-calculate a few additional helpful fields\n# - converts shorthands into full representation (e.g. reuse locations)\n#\n# This script only deals with field sets themselves and the fields defined\n# inside them. It doesn't perform field reuse, and therefore doesn't\n# deal with final field names either.\n\n\ndef clean(fields, strict=False):\n    global strict_mode\n    strict_mode = strict\n    visitor.visit_fields(fields, fieldset_func=schema_cleanup, field_func=field_cleanup)\n\n\n# Schema level cleanup\n\n\ndef schema_cleanup(schema):\n    # Sanity check first\n    schema_mandatory_attributes(schema)\n    # trailing space cleanup\n    ecs_helpers.dict_clean_string_values(schema['schema_details'])\n    ecs_helpers.dict_clean_string_values(schema['field_details'])\n    # Some defaults\n    schema['schema_details'].setdefault('group', 2)\n    schema['schema_details'].setdefault('root', False)\n    schema['field_details'].setdefault('type', 'group')\n    schema['field_details'].setdefault('short', schema['field_details']['description'])\n    if 'reusable' in schema['schema_details']:\n        # order to perform chained reuses. Set to 1 if it needs to happen earlier.\n        schema['schema_details']['reusable'].setdefault('order', 2)\n    # Precalculate stuff. Those can't be set in the YAML.\n    if schema['schema_details']['root']:\n        schema['schema_details']['prefix'] = ''\n    else:\n        schema['schema_details']['prefix'] = schema['field_details']['name'] + '.'\n    normalize_reuse_notation(schema)\n    # Final validity check if in strict mode\n    schema_assertions_and_warnings(schema)\n\n\nSCHEMA_MANDATORY_ATTRIBUTES = ['name', 'title', 'description']\n\n\ndef schema_mandatory_attributes(schema):\n    \"\"\"Ensures for the presence of the mandatory schema attributes and raises if any are missing\"\"\"\n    current_schema_attributes = sorted(list(schema['field_details'].keys()) +\n                                       list(schema['schema_details'].keys()))\n    missing_attributes = ecs_helpers.list_subtract(SCHEMA_MANDATORY_ATTRIBUTES, current_schema_attributes)\n    if len(missing_attributes) > 0:\n        msg = \"Schema {} is missing the following mandatory attributes: {}.\\nFound these: {}\".format(\n            schema['field_details']['name'], ', '.join(missing_attributes), current_schema_attributes)\n        raise ValueError(msg)\n    if 'reusable' in schema['schema_details']:\n        reuse_attributes = sorted(schema['schema_details']['reusable'].keys())\n        missing_reuse_attributes = ecs_helpers.list_subtract(['expected', 'top_level'], reuse_attributes)\n        if len(missing_reuse_attributes) > 0:\n            msg = \"Reusable schema {} is missing the following reuse attributes: {}.\\nFound these: {}\".format(\n                schema['field_details']['name'], ', '.join(missing_reuse_attributes), reuse_attributes)\n            raise ValueError(msg)\n\n\ndef schema_assertions_and_warnings(schema):\n    \"\"\"Additional checks on a fleshed out schema\"\"\"\n    single_line_short_description(schema, strict=strict_mode)\n    if 'beta' in schema['field_details']:\n        single_line_beta_description(schema, strict=strict_mode)\n    if 'reusable' in schema['schema_details']:\n        single_line_short_override_description(schema, strict=strict_mode)\n\n\ndef normalize_reuse_notation(schema):\n    \"\"\"\n    Replace single word reuse shorthands from the schema YAMLs with the explicit {at: , as:} notation.\n\n    When marking \"user\" as reusable under \"destination\" with the shorthand entry\n    `- destination`, this is expanded to the complete entry\n    `- { \"at\": \"destination\", \"as\": \"user\" }`.\n    The field set is thus nested at `destination.user.*`, with fields such as `destination.user.name`.\n\n    The dictionary notation enables nesting a field set as a different name.\n    An example is nesting \"process\" fields to capture parent process details\n    at `process.parent.*`.\n    The dictionary notation `- { \"at\": \"process\", \"as\": \"parent\" }` will yield\n    fields such as `process.parent.pid`.\n    \"\"\"\n    if 'reusable' not in schema['schema_details']:\n        return\n    schema_name = schema['field_details']['name']\n    reuse_entries = []\n    for reuse_entry in schema['schema_details']['reusable']['expected']:\n        if type(reuse_entry) is dict:  # Already explicit\n            if 'at' in reuse_entry and 'as' in reuse_entry:\n                explicit_entry = reuse_entry\n            else:\n                raise ValueError(\"When specifying reusable expected locations for {} \" +\n                                 \"with the dictionary notation, keys 'as' and 'at' are required. \" +\n                                 \"Got {}.\".format(schema_name, reuse_entry))\n        else:  # Make it explicit\n            explicit_entry = {'at': reuse_entry, 'as': schema_name}\n        explicit_entry['full'] = explicit_entry['at'] + '.' + explicit_entry['as']\n        reuse_entries.append(explicit_entry)\n    schema['schema_details']['reusable']['expected'] = reuse_entries\n\n\n# Field level cleanup\n\n\ndef field_cleanup(field):\n    field_mandatory_attributes(field)\n    if ecs_helpers.is_intermediate(field):\n        return\n    ecs_helpers.dict_clean_string_values(field['field_details'])\n    if 'allowed_values' in field['field_details']:\n        for allowed_value in field['field_details']['allowed_values']:\n            ecs_helpers.dict_clean_string_values(allowed_value)\n    field_defaults(field)\n    field_assertions_and_warnings(field)\n\n\ndef field_defaults(field):\n    field['field_details'].setdefault('short', field['field_details']['description'])\n    field['field_details'].setdefault('normalize', [])\n    field_or_multi_field_datatype_defaults(field['field_details'])\n    if 'multi_fields' in field['field_details']:\n        for mf in field['field_details']['multi_fields']:\n            field_or_multi_field_datatype_defaults(mf)\n            if 'name' not in mf:\n                mf['name'] = mf['type']\n\n\ndef field_or_multi_field_datatype_defaults(field_details):\n    \"\"\"Sets datatype-related defaults on a canonical field or multi-field entries.\"\"\"\n    if field_details['type'] == 'keyword':\n        field_details.setdefault('ignore_above', 1024)\n    if field_details['type'] == 'text':\n        field_details.setdefault('norms', False)\n    # wildcard needs the index param stripped\n    if field_details['type'] == 'wildcard':\n        field_details.pop('index', None)\n    if 'index' in field_details and not field_details['index']:\n        field_details.setdefault('doc_values', False)\n        field_details.pop('ignore_above', None)\n\n\nFIELD_MANDATORY_ATTRIBUTES = ['name', 'description', 'type', 'level']\nACCEPTABLE_FIELD_LEVELS = ['core', 'extended', 'custom']\n\n\ndef field_mandatory_attributes(field):\n    \"\"\"Ensures for the presence of the mandatory field attributes and raises if any are missing\"\"\"\n    if ecs_helpers.is_intermediate(field):\n        return\n    current_field_attributes = sorted(field['field_details'].keys())\n    missing_attributes = ecs_helpers.list_subtract(FIELD_MANDATORY_ATTRIBUTES, current_field_attributes)\n\n    # `alias` fields require a target `path` attribute.\n    if field['field_details'].get('type') == 'alias' and 'path' not in current_field_attributes:\n        missing_attributes.append('path')\n    # `scaled_float` fields require a `scaling_factor` attribute.\n    if field['field_details'].get('type') == 'scaled_float' and 'scaling_factor' not in current_field_attributes:\n        missing_attributes.append('scaling_factor')\n\n    if len(missing_attributes) > 0:\n        msg = \"Field is missing the following mandatory attributes: {}.\\nFound these: {}.\\nField details: {}\"\n        raise ValueError(msg.format(', '.join(missing_attributes),\n                                    current_field_attributes, field))\n\n\ndef field_assertions_and_warnings(field):\n    \"\"\"Additional checks on a fleshed out field\"\"\"\n    if not ecs_helpers.is_intermediate(field):\n        # check short description length if in strict mode\n        single_line_short_description(field, strict=strict_mode)\n        check_example_value(field, strict=strict_mode)\n        if 'beta' in field['field_details']:\n            single_line_beta_description(field, strict=strict_mode)\n        if field['field_details']['level'] not in ACCEPTABLE_FIELD_LEVELS:\n            msg = \"Invalid level for field '{}'.\\nValue: {}\\nAcceptable values: {}\".format(\n                field['field_details']['name'], field['field_details']['level'],\n                ACCEPTABLE_FIELD_LEVELS)\n            raise ValueError(msg)\n\n# Common\n\n\nSHORT_LIMIT = 120\n\n\ndef single_line_short_check(short_to_check, short_name):\n    short_length = len(short_to_check)\n    if \"\\n\" in short_to_check or short_length > SHORT_LIMIT:\n        msg = \"Short descriptions must be single line, and under {} characters (current length: {}).\\n\".format(\n            SHORT_LIMIT, short_length)\n        msg += \"Offending field or field set: {}\\nShort description:\\n  {}\".format(\n            short_name,\n            short_to_check)\n        return msg\n    return None\n\n\ndef single_line_short_description(schema_or_field, strict=True):\n    error = single_line_short_check(schema_or_field['field_details']['short'], schema_or_field['field_details']['name'])\n    if error:\n        if strict:\n            raise ValueError(error)\n        else:\n            ecs_helpers.strict_warning(error)\n\n\ndef single_line_short_override_description(schema_or_field, strict=True):\n    for field in schema_or_field['schema_details']['reusable']['expected']:\n        if not 'short_override' in field:\n            continue\n        error = single_line_short_check(field['short_override'], field['full'])\n        if error:\n            if strict:\n                raise ValueError(error)\n            else:\n                ecs_helpers.strict_warning(error)\n\n\ndef check_example_value(field, strict=True):\n    \"\"\"\n    Checks if value of the example field is of type list or dict.\n    Fails or warns (depending on strict mode) if so.\n    \"\"\"\n    example_value = field['field_details'].get('example', None)\n    if isinstance(example_value, (list, dict)):\n        name = field['field_details']['name']\n        msg = f\"Example value for field `{name}` contains an object or array which must be quoted to avoid YAML interpretation.\"\n        if strict:\n            raise ValueError(msg)\n        else:\n            ecs_helpers.strict_warning(msg)\n\n\ndef single_line_beta_description(schema_or_field, strict=True):\n    if \"\\n\" in schema_or_field['field_details']['beta']:\n        msg = \"Beta descriptions must be single line.\\n\"\n        msg += f\"Offending field or field set: {schema_or_field['field_details']['name']}\"\n        if strict:\n            raise ValueError(msg)\n        else:\n            ecs_helpers.strict_warning(msg)\n", "path": "scripts/schema/cleaner.py"}]}