prompt: dbt postgres fix command errors with UnicodeEncodeError and also wipes the .sql file
  _If this is a parsing or linting issue, please include a minimal SQL example which reproduces the issue, along with the `sqlfluff parse` output, `sqlfluff lint` output and `sqlfluff fix` output when relevant._
  
  ## Expected Behaviour
  Violation failure notice at a minimum, without wiping the file. Would like a way to ignore the known error at a minimum as --noqa is not getting past this. Actually would expect --noqa to totally ignore this.
  
  ## Observed Behaviour
  Reported error: `UnicodeEncodeError: 'charmap' codec can't encode character '\u2192' in position 120: character maps to <undefined>`
  
  ## Steps to Reproduce
  SQL file:
  ```sql
  SELECT
      reacted_table_name_right.descendant_id AS category_id,
      string_agg(redacted_table_name_left.name, ' → ' ORDER BY reacted_table_name_right.generations DESC) AS breadcrumbs -- noqa
  FROM {{ ref2('redacted_schema_name', 'redacted_table_name_left') }} AS redacted_table_name_left
  INNER JOIN {{ ref2('redacted_schema_name', 'reacted_table_name_right') }} AS reacted_table_name_right
      ON redacted_table_name_left.id = order_issue_category_hierarchies.ancestor_id
  GROUP BY reacted_table_name_right.descendant_id
  ```
  Running `sqlfluff fix --ignore templating,parsing,lexing -vvvv` and accepting proposed fixes for linting violations.
  
  ## Dialect
  `postgres`, with `dbt` templater
  
  ## Version
  `python 3.7.12`
  `sqlfluff 0.7.0`
  `sqlfluff-templater-dbt 0.7.0`
  
  ## Configuration
  I've tried a few, here's one:
  ```
  [sqlfluff]
  verbose = 2
  dialect = postgres
  templater = dbt
  exclude_rules = None
  output_line_length = 80
  runaway_limit = 10
  ignore_templated_areas = True
  processes = 3
  # Comma separated list of file extensions to lint.
  
  # NB: This config will only apply in the root folder.
  sql_file_exts = .sql
  
  [sqlfluff:indentation]
  indented_joins = False
  indented_using_on = True
  template_blocks_indent = True
  
  [sqlfluff:templater]
  unwrap_wrapped_queries = True
  
  [sqlfluff:templater:jinja]
  apply_dbt_builtins = True
  
  [sqlfluff:templater:jinja:macros]
  # Macros provided as builtins for dbt projects
  dbt_ref = {% macro ref(model_ref) %}{{model_ref}}{% endmacro %}
  dbt_source = {% macro source(source_name, table) %}{{source_name}}_{{table}}{% endmacro %}
  dbt_config = {% macro config() %}{% for k in kwargs %}{% endfor %}{% endmacro %}
  dbt_var = {% macro var(variable, default='') %}item{% endmacro %}
  dbt_is_incremental = {% macro is_incremental() %}True{% endmacro %}
  
  # Common config across rules
  [sqlfluff:rules]
  tab_space_size = 4
  indent_unit = space
  single_table_references = consistent
  unquoted_identifiers_policy = all
  
  # L001 - Remove trailing whitespace (fix)
  # L002 - Single section of whitespace should not contain both tabs and spaces (fix)
  # L003 - Keep consistent indentation (fix)
  # L004 - We use 4 spaces for indentation just for completeness (fix)
  # L005 - Remove space before commas (fix)
  # L006 - Operators (+, -, *, /) will be wrapped by a single space each side (fix)
  
  # L007 - Operators should not be at the end of a line
  [sqlfluff:rules:L007]  # Keywords
  operator_new_lines = after
  
  # L008 - Always use a single whitespace after a comma (fix)
  # L009 - Files will always end with a trailing newline
  
  # L010 - All keywords will use full upper case (fix)
  [sqlfluff:rules:L010]  # Keywords
  capitalisation_policy = upper
  
  # L011 - Always explicitly alias tables (fix)
  [sqlfluff:rules:L011]  # Aliasing
  aliasing = explicit
  
  # L012 - Do not have to explicitly alias all columns
  [sqlfluff:rules:L012]  # Aliasing
  aliasing = explicit
  
  # L013 - Always explicitly alias a column with an expression in it (fix)
  [sqlfluff:rules:L013]  # Aliasing
  allow_scalar = False
  
  # L014 - Always user full lower case for 'quoted identifiers' -> column refs. without an alias (fix)
  [sqlfluff:rules:L014]  # Unquoted identifiers
  extended_capitalisation_policy = lower
  
  # L015 - Always remove parenthesis when using DISTINCT to be clear that DISTINCT applies to all columns (fix)
  
  # L016 - Lines should be 120 characters of less. Comment lines should not be ignored (fix)
  [sqlfluff:rules:L016]
  ignore_comment_lines = False
  max_line_length = 120
  
  # L017 - There should not be whitespace between function name and brackets (fix)
  # L018 - Always align closing bracket of WITH to the WITH keyword (fix)
  
  # L019 - Always use trailing commas / commas at the end of the line (fix)
  [sqlfluff:rules:L019]
  comma_style = trailing
  
  # L020 - Table aliases will always be unique per statement
  # L021 - Remove any use of ambiguous DISTINCT and GROUP BY combinations. Lean on removing the GROUP BY.
  # L022 - Add blank lines after common table expressions (CTE) / WITH.
  # L023 - Always add a single whitespace after AS in a WITH clause (fix)
  
  [sqlfluff:rules:L026]
  force_enable = False
  
  # L027 - Always add references if more than one referenced table or view is used
  
  [sqlfluff:rules:L028]
  force_enable = False
  
  [sqlfluff:rules:L029]  # Keyword identifiers
  unquoted_identifiers_policy = aliases
  
  [sqlfluff:rules:L030]  # Function names
  capitalisation_policy = upper
  
  # L032 - We prefer use of join keys rather than USING
  # L034 - We prefer ordering of columns in select statements as (fix):
  # 1. wildcards
  # 2. single identifiers
  # 3. calculations and aggregates
  
  # L035 - Omit 'else NULL'; it is redundant (fix)
  # L036 - Move select targets / identifiers onto new lines each (fix)
  # L037 - When using ORDER BY, make the direction explicit (fix)
  
  # L038 - Never use trailing commas at the end of the SELECT clause
  [sqlfluff:rules:L038]
  select_clause_trailing_comma = forbid
  
  # L039 - Remove unnecessary whitespace (fix)
  
  [sqlfluff:rules:L040]  # Null & Boolean Literals
  capitalisation_policy = upper
  
  # L042 - Join clauses should not contain subqueries. Use common tables expressions (CTE) instead.
  [sqlfluff:rules:L042]
  # By default, allow subqueries in from clauses, but not join clauses.
  forbid_subquery_in = join
  
  # L043 - Reduce CASE WHEN conditions to COALESCE (fix)
  # L044 - Prefer a known number of columns along the path to the source data
  # L045 - Remove unused common tables expressions (CTE) / WITH statements (fix)
  # L046 - Jinja tags should have a single whitespace on both sides
  
  # L047 - Use COUNT(*) instead of COUNT(0) or COUNT(1) alternatives (fix)
  [sqlfluff:rules:L047]  # Consistent syntax to count all rows
  prefer_count_1 = False
  prefer_count_0 = False
  
  # L048 - Quoted literals should be surrounded by a single whitespace (fix)
  # L049 - Always use IS or IS NOT for comparisons with NULL (fix)
  ```
  
  
instance: {"FAIL_TO_PASS": "[\"test/core/linter_test.py::test_safe_create_replace_file[utf8_create]\", \"test/core/linter_test.py::test_safe_create_replace_file[utf8_update]\", \"test/core/linter_test.py::test_safe_create_replace_file[utf8_special_char]\"]", "PASS_TO_PASS": "[\"test/core/linter_test.py::test__linter__path_from_paths__dir\", \"test/core/linter_test.py::test__linter__path_from_paths__default\", \"test/core/linter_test.py::test__linter__path_from_paths__exts\", \"test/core/linter_test.py::test__linter__path_from_paths__file\", \"test/core/linter_test.py::test__linter__path_from_paths__not_exist\", \"test/core/linter_test.py::test__linter__path_from_paths__not_exist_ignore\", \"test/core/linter_test.py::test__linter__path_from_paths__explicit_ignore\", \"test/core/linter_test.py::test__linter__path_from_paths__dot\", \"test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore]\", \"test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/]\", \"test/core/linter_test.py::test__linter__path_from_paths__ignore[test/fixtures/linter/sqlfluffignore/.]\", \"test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/indentation_errors.sql]\", \"test/core/linter_test.py::test__linter__lint_string_vs_file[test/fixtures/linter/whitespace_errors.sql]\", \"test/core/linter_test.py::test__linter__get_violations_filter_rules[None-7]\", \"test/core/linter_test.py::test__linter__get_violations_filter_rules[L010-2]\", \"test/core/linter_test.py::test__linter__get_violations_filter_rules[rules2-2]\", \"test/core/linter_test.py::test__linter__linting_result__sum_dicts\", \"test/core/linter_test.py::test__linter__linting_result__combine_dicts\", \"test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[False-list]\", \"test/core/linter_test.py::test__linter__linting_result_check_tuples_by_path[True-dict]\", \"test/core/linter_test.py::test__linter__linting_result_get_violations[1]\", \"test/core/linter_test.py::test__linter__linting_result_get_violations[2]\", \"test/core/linter_test.py::test__linter__linting_parallel_thread[False]\", \"test/core/linter_test.py::test__linter__linting_parallel_thread[True]\", \"test/core/linter_test.py::test_lint_path_parallel_wrapper_exception\", \"test/core/linter_test.py::test__linter__linting_unexpected_error_handled_gracefully\", \"test/core/linter_test.py::test__linter__raises_malformed_noqa\", \"test/core/linter_test.py::test__linter__empty_file\", \"test/core/linter_test.py::test__linter__mask_templated_violations[True-check_tuples0]\", \"test/core/linter_test.py::test__linter__mask_templated_violations[False-check_tuples1]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-autodetect-False]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-autodetect-False]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-False]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-True]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8.sql-utf-8-sig-False]\", \"test/core/linter_test.py::test__linter__encoding[test/fixtures/linter/encoding-utf-8-sig.sql-utf-8-sig-False]\", \"test/core/linter_test.py::test_parse_noqa[-None]\", \"test/core/linter_test.py::test_parse_noqa[noqa-expected1]\", \"test/core/linter_test.py::test_parse_noqa[noqa?-SQLParseError]\", \"test/core/linter_test.py::test_parse_noqa[noqa:-expected3]\", \"test/core/linter_test.py::test_parse_noqa[noqa:L001,L002-expected4]\", \"test/core/linter_test.py::test_parse_noqa[noqa:\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_no_ignore]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_specific_line]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_line]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_different_specific_rule]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_enable_this_range]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_ignore_disable_this_range]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_specific_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_specific_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_specific_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_specific_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_1_ignore_disable_all_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_2_ignore_disable_all_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_3_ignore_disable_all_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[1_violation_line_4_ignore_disable_all_2_3]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_specific_enable_all]\", \"test/core/linter_test.py::test_linted_file_ignore_masked_violations[4_violations_two_types_disable_all_enable_specific]\", \"test/core/linter_test.py::test_linter_noqa\", \"test/core/linter_test.py::test_linter_noqa_with_templating\", \"test/core/linter_test.py::test_delayed_exception\", \"test/core/linter_test.py::test__attempt_to_change_templater_warning\", \"test/core/linter_test.py::test_safe_create_replace_file[incorrect_encoding]\"]", "base_commit": "a10057635e5b2559293a676486f0b730981f037a", "created_at": "2021-10-26T17:28:28Z", "environment_setup_commit": "67023b85c41d23d6c6d69812a41b207c4f8a9331", "hints_text": "I get a dbt-related error -- can you provide your project file as well? Also, what operating system are you running this on? I tested a simplified (non-dbt) version of your file on my Mac, and it worked okay.\r\n\r\n```\r\ndbt.exceptions.DbtProjectError: Runtime Error\r\n  no dbt_project.yml found at expected path /Users/bhart/dev/sqlfluff/dbt_project.yml\r\n```\nNever mind the questions above -- I managed to reproduce the error in a sample dbt project. Taking a look now...\n@Tumble17: Have you tried setting the `encoding` parameter in `.sqlfluff`? Do you know what encoding you're using? The default is `autodetect`, and SQLFluff \"thinks\" the file uses \"Windows-1252\" encoding, which I assume is incorrect -- that's why SQLFluff is unable to write out the updated file.\nI added this line to the first section of your `.sqlfluff`, and now it seems to work. I'll look into changing the behavior of `sqlfluff fix` so it doesn't erase the file when it fails.\r\n\r\n```\r\nencoding = utf-8\r\n```", "instance_id": "sqlfluff__sqlfluff-1763", "patch": "diff --git a/src/sqlfluff/core/linter/linted_file.py b/src/sqlfluff/core/linter/linted_file.py\n--- a/src/sqlfluff/core/linter/linted_file.py\n+++ b/src/sqlfluff/core/linter/linted_file.py\n@@ -7,6 +7,8 @@\n \n import os\n import logging\n+import shutil\n+import tempfile\n from typing import (\n     Any,\n     Iterable,\n@@ -493,7 +495,24 @@ def persist_tree(self, suffix: str = \"\") -> bool:\n             if suffix:\n                 root, ext = os.path.splitext(fname)\n                 fname = root + suffix + ext\n-            # Actually write the file.\n-            with open(fname, \"w\", encoding=self.encoding) as f:\n-                f.write(write_buff)\n+            self._safe_create_replace_file(fname, write_buff, self.encoding)\n         return success\n+\n+    @staticmethod\n+    def _safe_create_replace_file(fname, write_buff, encoding):\n+        # Write to a temporary file first, so in case of encoding or other\n+        # issues, we don't delete or corrupt the user's existing file.\n+        dirname, basename = os.path.split(fname)\n+        with tempfile.NamedTemporaryFile(\n+            mode=\"w\",\n+            encoding=encoding,\n+            prefix=basename,\n+            dir=dirname,\n+            suffix=os.path.splitext(fname)[1],\n+            delete=False,\n+        ) as tmp:\n+            tmp.file.write(write_buff)\n+            tmp.flush()\n+            os.fsync(tmp.fileno())\n+        # Once the temp file is safely written, replace the existing file.\n+        shutil.move(tmp.name, fname)\n", "problem_statement": "dbt postgres fix command errors with UnicodeEncodeError and also wipes the .sql file\n_If this is a parsing or linting issue, please include a minimal SQL example which reproduces the issue, along with the `sqlfluff parse` output, `sqlfluff lint` output and `sqlfluff fix` output when relevant._\r\n\r\n## Expected Behaviour\r\nViolation failure notice at a minimum, without wiping the file. Would like a way to ignore the known error at a minimum as --noqa is not getting past this. Actually would expect --noqa to totally ignore this.\r\n\r\n## Observed Behaviour\r\nReported error: `UnicodeEncodeError: 'charmap' codec can't encode character '\\u2192' in position 120: character maps to <undefined>`\r\n\r\n## Steps to Reproduce\r\nSQL file:\r\n```sql\r\nSELECT\r\n    reacted_table_name_right.descendant_id AS category_id,\r\n    string_agg(redacted_table_name_left.name, ' \u2192 ' ORDER BY reacted_table_name_right.generations DESC) AS breadcrumbs -- noqa\r\nFROM {{ ref2('redacted_schema_name', 'redacted_table_name_left') }} AS redacted_table_name_left\r\nINNER JOIN {{ ref2('redacted_schema_name', 'reacted_table_name_right') }} AS reacted_table_name_right\r\n    ON redacted_table_name_left.id = order_issue_category_hierarchies.ancestor_id\r\nGROUP BY reacted_table_name_right.descendant_id\r\n```\r\nRunning `sqlfluff fix --ignore templating,parsing,lexing -vvvv` and accepting proposed fixes for linting violations.\r\n\r\n## Dialect\r\n`postgres`, with `dbt` templater\r\n\r\n## Version\r\n`python 3.7.12`\r\n`sqlfluff 0.7.0`\r\n`sqlfluff-templater-dbt 0.7.0`\r\n\r\n## Configuration\r\nI've tried a few, here's one:\r\n```\r\n[sqlfluff]\r\nverbose = 2\r\ndialect = postgres\r\ntemplater = dbt\r\nexclude_rules = None\r\noutput_line_length = 80\r\nrunaway_limit = 10\r\nignore_templated_areas = True\r\nprocesses = 3\r\n# Comma separated list of file extensions to lint.\r\n\r\n# NB: This config will only apply in the root folder.\r\nsql_file_exts = .sql\r\n\r\n[sqlfluff:indentation]\r\nindented_joins = False\r\nindented_using_on = True\r\ntemplate_blocks_indent = True\r\n\r\n[sqlfluff:templater]\r\nunwrap_wrapped_queries = True\r\n\r\n[sqlfluff:templater:jinja]\r\napply_dbt_builtins = True\r\n\r\n[sqlfluff:templater:jinja:macros]\r\n# Macros provided as builtins for dbt projects\r\ndbt_ref = {% macro ref(model_ref) %}{{model_ref}}{% endmacro %}\r\ndbt_source = {% macro source(source_name, table) %}{{source_name}}_{{table}}{% endmacro %}\r\ndbt_config = {% macro config() %}{% for k in kwargs %}{% endfor %}{% endmacro %}\r\ndbt_var = {% macro var(variable, default='') %}item{% endmacro %}\r\ndbt_is_incremental = {% macro is_incremental() %}True{% endmacro %}\r\n\r\n# Common config across rules\r\n[sqlfluff:rules]\r\ntab_space_size = 4\r\nindent_unit = space\r\nsingle_table_references = consistent\r\nunquoted_identifiers_policy = all\r\n\r\n# L001 - Remove trailing whitespace (fix)\r\n# L002 - Single section of whitespace should not contain both tabs and spaces (fix)\r\n# L003 - Keep consistent indentation (fix)\r\n# L004 - We use 4 spaces for indentation just for completeness (fix)\r\n# L005 - Remove space before commas (fix)\r\n# L006 - Operators (+, -, *, /) will be wrapped by a single space each side (fix)\r\n\r\n# L007 - Operators should not be at the end of a line\r\n[sqlfluff:rules:L007]  # Keywords\r\noperator_new_lines = after\r\n\r\n# L008 - Always use a single whitespace after a comma (fix)\r\n# L009 - Files will always end with a trailing newline\r\n\r\n# L010 - All keywords will use full upper case (fix)\r\n[sqlfluff:rules:L010]  # Keywords\r\ncapitalisation_policy = upper\r\n\r\n# L011 - Always explicitly alias tables (fix)\r\n[sqlfluff:rules:L011]  # Aliasing\r\naliasing = explicit\r\n\r\n# L012 - Do not have to explicitly alias all columns\r\n[sqlfluff:rules:L012]  # Aliasing\r\naliasing = explicit\r\n\r\n# L013 - Always explicitly alias a column with an expression in it (fix)\r\n[sqlfluff:rules:L013]  # Aliasing\r\nallow_scalar = False\r\n\r\n# L014 - Always user full lower case for 'quoted identifiers' -> column refs. without an alias (fix)\r\n[sqlfluff:rules:L014]  # Unquoted identifiers\r\nextended_capitalisation_policy = lower\r\n\r\n# L015 - Always remove parenthesis when using DISTINCT to be clear that DISTINCT applies to all columns (fix)\r\n\r\n# L016 - Lines should be 120 characters of less. Comment lines should not be ignored (fix)\r\n[sqlfluff:rules:L016]\r\nignore_comment_lines = False\r\nmax_line_length = 120\r\n\r\n# L017 - There should not be whitespace between function name and brackets (fix)\r\n# L018 - Always align closing bracket of WITH to the WITH keyword (fix)\r\n\r\n# L019 - Always use trailing commas / commas at the end of the line (fix)\r\n[sqlfluff:rules:L019]\r\ncomma_style = trailing\r\n\r\n# L020 - Table aliases will always be unique per statement\r\n# L021 - Remove any use of ambiguous DISTINCT and GROUP BY combinations. Lean on removing the GROUP BY.\r\n# L022 - Add blank lines after common table expressions (CTE) / WITH.\r\n# L023 - Always add a single whitespace after AS in a WITH clause (fix)\r\n\r\n[sqlfluff:rules:L026]\r\nforce_enable = False\r\n\r\n# L027 - Always add references if more than one referenced table or view is used\r\n\r\n[sqlfluff:rules:L028]\r\nforce_enable = False\r\n\r\n[sqlfluff:rules:L029]  # Keyword identifiers\r\nunquoted_identifiers_policy = aliases\r\n\r\n[sqlfluff:rules:L030]  # Function names\r\ncapitalisation_policy = upper\r\n\r\n# L032 - We prefer use of join keys rather than USING\r\n# L034 - We prefer ordering of columns in select statements as (fix):\r\n# 1. wildcards\r\n# 2. single identifiers\r\n# 3. calculations and aggregates\r\n\r\n# L035 - Omit 'else NULL'; it is redundant (fix)\r\n# L036 - Move select targets / identifiers onto new lines each (fix)\r\n# L037 - When using ORDER BY, make the direction explicit (fix)\r\n\r\n# L038 - Never use trailing commas at the end of the SELECT clause\r\n[sqlfluff:rules:L038]\r\nselect_clause_trailing_comma = forbid\r\n\r\n# L039 - Remove unnecessary whitespace (fix)\r\n\r\n[sqlfluff:rules:L040]  # Null & Boolean Literals\r\ncapitalisation_policy = upper\r\n\r\n# L042 - Join clauses should not contain subqueries. Use common tables expressions (CTE) instead.\r\n[sqlfluff:rules:L042]\r\n# By default, allow subqueries in from clauses, but not join clauses.\r\nforbid_subquery_in = join\r\n\r\n# L043 - Reduce CASE WHEN conditions to COALESCE (fix)\r\n# L044 - Prefer a known number of columns along the path to the source data\r\n# L045 - Remove unused common tables expressions (CTE) / WITH statements (fix)\r\n# L046 - Jinja tags should have a single whitespace on both sides\r\n\r\n# L047 - Use COUNT(*) instead of COUNT(0) or COUNT(1) alternatives (fix)\r\n[sqlfluff:rules:L047]  # Consistent syntax to count all rows\r\nprefer_count_1 = False\r\nprefer_count_0 = False\r\n\r\n# L048 - Quoted literals should be surrounded by a single whitespace (fix)\r\n# L049 - Always use IS or IS NOT for comparisons with NULL (fix)\r\n```\r\n\n", "repo": "sqlfluff/sqlfluff", "test_patch": "diff --git a/test/core/linter_test.py b/test/core/linter_test.py\n--- a/test/core/linter_test.py\n+++ b/test/core/linter_test.py\n@@ -641,3 +641,56 @@ def test__attempt_to_change_templater_warning(caplog):\n         assert \"Attempt to set templater to \" in caplog.text\n     finally:\n         logger.propagate = original_propagate_value\n+\n+\n+@pytest.mark.parametrize(\n+    \"case\",\n+    [\n+        dict(\n+            name=\"utf8_create\",\n+            fname=\"test.sql\",\n+            encoding=\"utf-8\",\n+            existing=None,\n+            update=\"def\",\n+            expected=\"def\",\n+        ),\n+        dict(\n+            name=\"utf8_update\",\n+            fname=\"test.sql\",\n+            encoding=\"utf-8\",\n+            existing=\"abc\",\n+            update=\"def\",\n+            expected=\"def\",\n+        ),\n+        dict(\n+            name=\"utf8_special_char\",\n+            fname=\"test.sql\",\n+            encoding=\"utf-8\",\n+            existing=\"abc\",\n+            update=\"\u2192\",  # Special utf-8 character\n+            expected=\"\u2192\",\n+        ),\n+        dict(\n+            name=\"incorrect_encoding\",\n+            fname=\"test.sql\",\n+            encoding=\"Windows-1252\",\n+            existing=\"abc\",\n+            update=\"\u2192\",  # Not valid in Windows-1252\n+            expected=\"abc\",  # File should be unchanged\n+        ),\n+    ],\n+    ids=lambda case: case[\"name\"],\n+)\n+def test_safe_create_replace_file(case, tmp_path):\n+    \"\"\"Test creating or updating .sql files, various content and encoding.\"\"\"\n+    p = tmp_path / case[\"fname\"]\n+    if case[\"existing\"]:\n+        p.write_text(case[\"existing\"])\n+    try:\n+        linter.LintedFile._safe_create_replace_file(\n+            str(p), case[\"update\"], case[\"encoding\"]\n+        )\n+    except:  # noqa: E722\n+        pass\n+    actual = p.read_text(encoding=case[\"encoding\"])\n+    assert case[\"expected\"] == actual\n", "version": "0.6"}
parsed_commit_content: {"file_diffs":[{"old_file_content":"\"\"\"Defines the LintedFile class.\n\nThis holds linting results for a single file, and also\ncontains all of the routines to apply fixes to that file\npost linting.\n\"\"\"\n\nimport os\nimport logging\nfrom typing import (\n    Any,\n    Iterable,\n    List,\n    NamedTuple,\n    Optional,\n    Tuple,\n    Union,\n    cast,\n    Type,\n)\n\nfrom sqlfluff.core.errors import (\n    SQLBaseError,\n    SQLLintError,\n    CheckTuple,\n)\nfrom sqlfluff.core.string_helpers import findall\nfrom sqlfluff.core.templaters import TemplatedFile\n\n# Classes needed only for type checking\nfrom sqlfluff.core.parser.segments.base import BaseSegment, FixPatch\n\nfrom sqlfluff.core.linter.common import NoQaDirective, EnrichedFixPatch\n\n# Instantiate the linter logger\nlinter_logger: logging.Logger = logging.getLogger(\"sqlfluff.linter\")\n\n\nclass LintedFile(NamedTuple):\n    \"\"\"A class to store the idea of a linted file.\"\"\"\n\n    path: str\n    violations: List[SQLBaseError]\n    time_dict: dict\n    tree: Optional[BaseSegment]\n    ignore_mask: List[NoQaDirective]\n    templated_file: TemplatedFile\n    encoding: str\n\n    def check_tuples(self, raise_on_non_linting_violations=True) -> List[CheckTuple]:\n        \"\"\"Make a list of check_tuples.\n\n        This assumes that all the violations found are\n        linting violations (and therefore implement `check_tuple()`).\n        If they don't then this function raises that error.\n        \"\"\"\n        vs: List[CheckTuple] = []\n        v: SQLLintError\n        for v in self.get_violations():\n            if hasattr(v, \"check_tuple\"):\n                vs.append(v.check_tuple())\n            elif raise_on_non_linting_violations:\n                raise v\n        return vs\n\n    def get_violations(\n        self,\n        rules: Optional[Union[str, Tuple[str, ...]]] = None,\n        types: Optional[Union[Type[SQLBaseError], Iterable[Type[SQLBaseError]]]] = None,\n        filter_ignore: bool = True,\n        fixable: bool = None,\n    ) -> list:\n        \"\"\"Get a list of violations, respecting filters and ignore options.\n\n        Optionally now with filters.\n        \"\"\"\n        violations = self.violations\n        # Filter types\n        if types:\n            # If it's a singular type, make it a single item in a tuple\n            # otherwise coerce to tuple normally so that we can use it with\n            # isinstance.\n            if isinstance(types, type) and issubclass(types, SQLBaseError):\n                types = (types,)\n            else:\n                types = tuple(types)  # pragma: no cover TODO?\n            violations = [v for v in violations if isinstance(v, types)]\n        # Filter rules\n        if rules:\n            if isinstance(rules, str):\n                rules = (rules,)\n            else:\n                rules = tuple(rules)\n            violations = [v for v in violations if v.rule_code() in rules]\n        # Filter fixable\n        if fixable is not None:\n            # Assume that fixable is true or false if not None\n            violations = [v for v in violations if v.fixable is fixable]\n        # Filter ignorable violations\n        if filter_ignore:\n            violations = [v for v in violations if not v.ignore]\n            # Ignore any rules in the ignore mask\n            if self.ignore_mask:\n                violations = self.ignore_masked_violations(violations, self.ignore_mask)\n        return violations\n\n    @staticmethod\n    def _ignore_masked_violations_single_line(\n        violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore error for line-specific directives.\n\n        The \"ignore\" list is assumed to ONLY contain NoQaDirectives with\n        action=None.\n        \"\"\"\n        for ignore in ignore_mask:\n            violations = [\n                v\n                for v in violations\n                if not (\n                    v.line_no == ignore.line_no\n                    and (ignore.rules is None or v.rule_code() in ignore.rules)\n                )\n            ]\n        return violations\n\n    @staticmethod\n    def _should_ignore_violation_line_range(\n        line_no: int, ignore_rule: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore a violation at line_no.\"\"\"\n        # Loop through the NoQaDirectives to find the state of things at\n        # line_no. Assumptions about \"ignore_rule\":\n        # - Contains directives for only ONE RULE, i.e. the rule that was\n        #   violated at line_no\n        # - Sorted in ascending order by line number\n        disable = False\n        for ignore in ignore_rule:\n            if ignore.line_no > line_no:\n                break\n            disable = ignore.action == \"disable\"\n        return disable\n\n    @classmethod\n    def _ignore_masked_violations_line_range(\n        cls, violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore error for line-range directives.\n\n        The \"ignore\" list is assumed to ONLY contain NoQaDirectives where\n        action is \"enable\" or \"disable\".\n        \"\"\"\n        result = []\n        for v in violations:\n            # Find the directives that affect the violated rule \"v\", either\n            # because they specifically reference it or because they don't\n            # specify a list of rules, thus affecting ALL rules.\n            ignore_rule = sorted(\n                (\n                    ignore\n                    for ignore in ignore_mask\n                    if not ignore.rules\n                    or (v.rule_code() in cast(Tuple[str, ...], ignore.rules))\n                ),\n                key=lambda ignore: ignore.line_no,\n            )\n            # Determine whether to ignore the violation, based on the relevant\n            # enable/disable directives.\n            if not cls._should_ignore_violation_line_range(v.line_no, ignore_rule):\n                result.append(v)\n        return result\n\n    @classmethod\n    def ignore_masked_violations(\n        cls, violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ) -> List[SQLBaseError]:\n        \"\"\"Remove any violations specified by ignore_mask.\n\n        This involves two steps:\n        1. Filter out violations affected by single-line \"noqa\" directives.\n        2. Filter out violations affected by disable/enable \"noqa\" directives.\n        \"\"\"\n        ignore_specific = [ignore for ignore in ignore_mask if not ignore.action]\n        ignore_range = [ignore for ignore in ignore_mask if ignore.action]\n        violations = cls._ignore_masked_violations_single_line(\n            violations, ignore_specific\n        )\n        violations = cls._ignore_masked_violations_line_range(violations, ignore_range)\n        return violations\n\n    def num_violations(self, **kwargs) -> int:\n        \"\"\"Count the number of violations.\n\n        Optionally now with filters.\n        \"\"\"\n        violations = self.get_violations(**kwargs)\n        return len(violations)\n\n    def is_clean(self) -> bool:\n        \"\"\"Return True if there are no ignorable violations.\"\"\"\n        return not any(self.get_violations(filter_ignore=True))\n\n    @staticmethod\n    def _log_hints(\n        patch: Union[EnrichedFixPatch, FixPatch], templated_file: TemplatedFile\n    ):\n        \"\"\"Log hints for debugging during patch generation.\"\"\"\n        # This next bit is ALL FOR LOGGING AND DEBUGGING\n        max_log_length = 10\n        if patch.templated_slice.start >= max_log_length:\n            pre_hint = templated_file.templated_str[\n                patch.templated_slice.start\n                - max_log_length : patch.templated_slice.start\n            ]\n        else:\n            pre_hint = templated_file.templated_str[: patch.templated_slice.start]\n        if patch.templated_slice.stop + max_log_length < len(\n            templated_file.templated_str\n        ):\n            post_hint = templated_file.templated_str[\n                patch.templated_slice.stop : patch.templated_slice.stop + max_log_length\n            ]\n        else:\n            post_hint = templated_file.templated_str[patch.templated_slice.stop :]\n        linter_logger.debug(\n            \"        Templated Hint: ...%r <> %r...\", pre_hint, post_hint\n        )\n\n    def fix_string(self) -> Tuple[Any, bool]:\n        \"\"\"Obtain the changes to a path as a string.\n\n        We use the source mapping features of TemplatedFile\n        to generate a list of \"patches\" which cover the non\n        templated parts of the file and refer back to the locations\n        in the original file.\n\n        NB: This is MUCH FASTER than the original approach\n        using difflib in pre 0.4.0.\n\n        There is an important distinction here between Slices and\n        Segments. A Slice is a portion of a file which is determined\n        by the templater based on which portions of the source file\n        are templated or not, and therefore before Lexing and so is\n        completely dialect agnostic. A Segment is determined by the\n        Lexer from portions of strings after templating.\n        \"\"\"\n        linter_logger.debug(\"Original Tree: %r\", self.templated_file.templated_str)\n        assert self.tree\n        linter_logger.debug(\"Fixed Tree: %r\", self.tree.raw)\n\n        # The sliced file is contiguous in the TEMPLATED space.\n        # NB: It has gaps and repeats in the source space.\n        # It's also not the FIXED file either.\n        linter_logger.debug(\"### Templated File.\")\n        for idx, file_slice in enumerate(self.templated_file.sliced_file):\n            t_str = self.templated_file.templated_str[file_slice.templated_slice]\n            s_str = self.templated_file.source_str[file_slice.source_slice]\n            if t_str == s_str:\n                linter_logger.debug(\n                    \"    File slice: %s %r [invariant]\", idx, file_slice\n                )\n            else:\n                linter_logger.debug(\"    File slice: %s %r\", idx, file_slice)\n                linter_logger.debug(\"    \\t\\t\\ttemplated: %r\\tsource: %r\", t_str, s_str)\n\n        original_source = self.templated_file.source_str\n\n        # Make sure no patches overlap and divide up the source file into slices.\n        # Any Template tags in the source file are off limits.\n        source_only_slices = self.templated_file.source_only_slices()\n\n        linter_logger.debug(\"Source-only slices: %s\", source_only_slices)\n\n        # Iterate patches, filtering and translating as we go:\n        linter_logger.debug(\"### Beginning Patch Iteration.\")\n        filtered_source_patches = []\n        dedupe_buffer = []\n        # We use enumerate so that we get an index for each patch. This is entirely\n        # so when debugging logs we can find a given patch again!\n        patch: Union[EnrichedFixPatch, FixPatch]\n        for idx, patch in enumerate(\n            self.tree.iter_patches(templated_str=self.templated_file.templated_str)\n        ):\n            linter_logger.debug(\"  %s Yielded patch: %s\", idx, patch)\n            self._log_hints(patch, self.templated_file)\n\n            # Attempt to convert to source space.\n            try:\n                source_slice = self.templated_file.templated_slice_to_source_slice(\n                    patch.templated_slice,\n                )\n            except ValueError:\n                linter_logger.info(\n                    \"      - Skipping. Source space Value Error. i.e. attempted insertion within templated section.\"\n                )\n                # If we try and slice within a templated section, then we may fail\n                # in which case, we should skip this patch.\n                continue\n\n            # Check for duplicates\n            dedupe_tuple = (source_slice, patch.fixed_raw)\n            if dedupe_tuple in dedupe_buffer:\n                linter_logger.info(\n                    \"      - Skipping. Source space Duplicate: %s\", dedupe_tuple\n                )\n                continue\n\n            # We now evaluate patches in the source-space for whether they overlap\n            # or disrupt any templated sections.\n            # The intent here is that unless explicitly stated, a fix should never\n            # disrupt a templated section.\n            # NOTE: We rely here on the patches being sorted.\n            # TODO: Implement a mechanism for doing templated section fixes. For\n            # now it's just not allowed.\n\n            # Get the affected raw slices.\n            local_raw_slices = self.templated_file.raw_slices_spanning_source_slice(\n                source_slice\n            )\n            local_type_list = [slc.slice_type for slc in local_raw_slices]\n\n            enriched_patch = EnrichedFixPatch(\n                source_slice=source_slice,\n                templated_slice=patch.templated_slice,\n                patch_category=patch.patch_category,\n                fixed_raw=patch.fixed_raw,\n                templated_str=self.templated_file.templated_str[patch.templated_slice],\n                source_str=self.templated_file.source_str[source_slice],\n            )\n\n            # Deal with the easy case of only literals\n            if set(local_type_list) == {\"literal\"}:\n                linter_logger.info(\n                    \"      * Keeping patch on literal-only section: %s\", enriched_patch\n                )\n                filtered_source_patches.append(enriched_patch)\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())\n            # Is it a zero length patch.\n            elif (\n                enriched_patch.source_slice.start == enriched_patch.source_slice.stop\n                and enriched_patch.source_slice.start == local_raw_slices[0].source_idx\n            ):\n                linter_logger.info(\n                    \"      * Keeping insertion patch on slice boundary: %s\",\n                    enriched_patch,\n                )\n                filtered_source_patches.append(enriched_patch)\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())\n            # If it's ONLY templated then we should skip it.\n            elif \"literal\" not in local_type_list:\n                linter_logger.info(\n                    \"      - Skipping patch over templated section: %s\", enriched_patch\n                )\n            # If we span more than two slices then we should just skip it. Too Hard.\n            elif len(local_raw_slices) > 2:\n                linter_logger.info(\n                    \"      - Skipping patch over more than two raw slices: %s\",\n                    enriched_patch,\n                )\n            # If it's an insertion (i.e. the string in the pre-fix template is '') then we\n            # won't be able to place it, so skip.\n            elif not enriched_patch.templated_str:  # pragma: no cover TODO?\n                linter_logger.info(\n                    \"      - Skipping insertion patch in templated section: %s\",\n                    enriched_patch,\n                )\n            # If the string from the templated version isn't in the source, then we can't fix it.\n            elif (\n                enriched_patch.templated_str not in enriched_patch.source_str\n            ):  # pragma: no cover TODO?\n                linter_logger.info(\n                    \"      - Skipping edit patch on templated content: %s\",\n                    enriched_patch,\n                )\n            else:\n                # Identify all the places the string appears in the source content.\n                positions = list(\n                    findall(enriched_patch.templated_str, enriched_patch.source_str)\n                )\n                if len(positions) != 1:\n                    linter_logger.debug(\n                        \"        - Skipping edit patch on non-unique templated content: %s\",\n                        enriched_patch,\n                    )\n                    continue\n                # We have a single occurrence of the thing we want to patch. This\n                # means we can use its position to place our patch.\n                new_source_slice = slice(  # pragma: no cover\n                    enriched_patch.source_slice.start + positions[0],\n                    enriched_patch.source_slice.start\n                    + positions[0]\n                    + len(enriched_patch.templated_str),\n                )\n                enriched_patch = EnrichedFixPatch(  # pragma: no cover\n                    source_slice=new_source_slice,\n                    templated_slice=enriched_patch.templated_slice,\n                    patch_category=enriched_patch.patch_category,\n                    fixed_raw=enriched_patch.fixed_raw,\n                    templated_str=enriched_patch.templated_str,\n                    source_str=enriched_patch.source_str,\n                )\n                linter_logger.debug(  # pragma: no cover\n                    \"      * Keeping Tricky Case. Positions: %s, New Slice: %s, Patch: %s\",\n                    positions,\n                    new_source_slice,\n                    enriched_patch,\n                )\n                filtered_source_patches.append(enriched_patch)  # pragma: no cover\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())  # pragma: no cover\n                continue  # pragma: no cover\n\n        # Sort the patches before building up the file.\n        filtered_source_patches = sorted(\n            filtered_source_patches, key=lambda x: x.source_slice.start\n        )\n        # We now slice up the file using the patches and any source only slices.\n        # This gives us regions to apply changes to.\n        slice_buff = []\n        source_idx = 0\n        for patch in filtered_source_patches:\n            # Are there templated slices at or before the start of this patch?\n            while (\n                source_only_slices\n                and source_only_slices[0].source_idx < patch.source_slice.start\n            ):\n                next_so_slice = source_only_slices.pop(0).source_slice()\n                # Add a pre-slice before the next templated slices if needed.\n                if next_so_slice.start > source_idx:\n                    slice_buff.append(slice(source_idx, next_so_slice.start))\n                # Add the templated slice.\n                slice_buff.append(next_so_slice)\n                source_idx = next_so_slice.stop\n\n            # Is there a gap between current position and this patch?\n            if patch.source_slice.start > source_idx:\n                # Add a slice up to this patch.\n                slice_buff.append(slice(source_idx, patch.source_slice.start))\n\n            # Is this patch covering an area we've already covered?\n            if patch.source_slice.start < source_idx:\n                linter_logger.info(\n                    \"Skipping overlapping patch at Index %s, Patch: %s\",\n                    source_idx,\n                    patch,\n                )\n                # Ignore the patch for now...\n                continue\n\n            # Add this patch.\n            slice_buff.append(patch.source_slice)\n            source_idx = patch.source_slice.stop\n        # Add a tail slice.\n        if source_idx < len(self.templated_file.source_str):\n            slice_buff.append(slice(source_idx, len(self.templated_file.source_str)))\n\n        linter_logger.debug(\"Final slice buffer: %s\", slice_buff)\n\n        # Iterate through the patches, building up the new string.\n        str_buff = \"\"\n        for source_slice in slice_buff:\n            # Is it one in the patch buffer:\n            for patch in filtered_source_patches:\n                if patch.source_slice == source_slice:\n                    # Use the patched version\n                    linter_logger.debug(\n                        \"%-30s    %s    %r > %r\",\n                        f\"Appending {patch.patch_category} Patch:\",\n                        patch.source_slice,\n                        patch.source_str,\n                        patch.fixed_raw,\n                    )\n                    str_buff += patch.fixed_raw\n                    break\n            else:\n                # Use the raw string\n                linter_logger.debug(\n                    \"Appending Raw:                    %s     %r\",\n                    source_slice,\n                    self.templated_file.source_str[source_slice],\n                )\n                str_buff += self.templated_file.source_str[source_slice]\n\n        # The success metric here is whether anything ACTUALLY changed.\n        return str_buff, str_buff != original_source\n\n    def persist_tree(self, suffix: str = \"\") -> bool:\n        \"\"\"Persist changes to the given path.\"\"\"\n        write_buff, success = self.fix_string()\n\n        if success:\n            fname = self.path\n            # If there is a suffix specified, then use it.s\n            if suffix:\n                root, ext = os.path.splitext(fname)\n                fname = root + suffix + ext\n            # Actually write the file.\n            with open(fname, \"w\", encoding=self.encoding) as f:\n                f.write(write_buff)\n        return success\n","new_file_content":"\"\"\"Defines the LintedFile class.\n\nThis holds linting results for a single file, and also\ncontains all of the routines to apply fixes to that file\npost linting.\n\"\"\"\n\nimport os\nimport logging\nimport shutil\nimport tempfile\nfrom typing import (\n    Any,\n    Iterable,\n    List,\n    NamedTuple,\n    Optional,\n    Tuple,\n    Union,\n    cast,\n    Type,\n)\n\nfrom sqlfluff.core.errors import (\n    SQLBaseError,\n    SQLLintError,\n    CheckTuple,\n)\nfrom sqlfluff.core.string_helpers import findall\nfrom sqlfluff.core.templaters import TemplatedFile\n\n# Classes needed only for type checking\nfrom sqlfluff.core.parser.segments.base import BaseSegment, FixPatch\n\nfrom sqlfluff.core.linter.common import NoQaDirective, EnrichedFixPatch\n\n# Instantiate the linter logger\nlinter_logger: logging.Logger = logging.getLogger(\"sqlfluff.linter\")\n\n\nclass LintedFile(NamedTuple):\n    \"\"\"A class to store the idea of a linted file.\"\"\"\n\n    path: str\n    violations: List[SQLBaseError]\n    time_dict: dict\n    tree: Optional[BaseSegment]\n    ignore_mask: List[NoQaDirective]\n    templated_file: TemplatedFile\n    encoding: str\n\n    def check_tuples(self, raise_on_non_linting_violations=True) -> List[CheckTuple]:\n        \"\"\"Make a list of check_tuples.\n\n        This assumes that all the violations found are\n        linting violations (and therefore implement `check_tuple()`).\n        If they don't then this function raises that error.\n        \"\"\"\n        vs: List[CheckTuple] = []\n        v: SQLLintError\n        for v in self.get_violations():\n            if hasattr(v, \"check_tuple\"):\n                vs.append(v.check_tuple())\n            elif raise_on_non_linting_violations:\n                raise v\n        return vs\n\n    def get_violations(\n        self,\n        rules: Optional[Union[str, Tuple[str, ...]]] = None,\n        types: Optional[Union[Type[SQLBaseError], Iterable[Type[SQLBaseError]]]] = None,\n        filter_ignore: bool = True,\n        fixable: bool = None,\n    ) -> list:\n        \"\"\"Get a list of violations, respecting filters and ignore options.\n\n        Optionally now with filters.\n        \"\"\"\n        violations = self.violations\n        # Filter types\n        if types:\n            # If it's a singular type, make it a single item in a tuple\n            # otherwise coerce to tuple normally so that we can use it with\n            # isinstance.\n            if isinstance(types, type) and issubclass(types, SQLBaseError):\n                types = (types,)\n            else:\n                types = tuple(types)  # pragma: no cover TODO?\n            violations = [v for v in violations if isinstance(v, types)]\n        # Filter rules\n        if rules:\n            if isinstance(rules, str):\n                rules = (rules,)\n            else:\n                rules = tuple(rules)\n            violations = [v for v in violations if v.rule_code() in rules]\n        # Filter fixable\n        if fixable is not None:\n            # Assume that fixable is true or false if not None\n            violations = [v for v in violations if v.fixable is fixable]\n        # Filter ignorable violations\n        if filter_ignore:\n            violations = [v for v in violations if not v.ignore]\n            # Ignore any rules in the ignore mask\n            if self.ignore_mask:\n                violations = self.ignore_masked_violations(violations, self.ignore_mask)\n        return violations\n\n    @staticmethod\n    def _ignore_masked_violations_single_line(\n        violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore error for line-specific directives.\n\n        The \"ignore\" list is assumed to ONLY contain NoQaDirectives with\n        action=None.\n        \"\"\"\n        for ignore in ignore_mask:\n            violations = [\n                v\n                for v in violations\n                if not (\n                    v.line_no == ignore.line_no\n                    and (ignore.rules is None or v.rule_code() in ignore.rules)\n                )\n            ]\n        return violations\n\n    @staticmethod\n    def _should_ignore_violation_line_range(\n        line_no: int, ignore_rule: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore a violation at line_no.\"\"\"\n        # Loop through the NoQaDirectives to find the state of things at\n        # line_no. Assumptions about \"ignore_rule\":\n        # - Contains directives for only ONE RULE, i.e. the rule that was\n        #   violated at line_no\n        # - Sorted in ascending order by line number\n        disable = False\n        for ignore in ignore_rule:\n            if ignore.line_no > line_no:\n                break\n            disable = ignore.action == \"disable\"\n        return disable\n\n    @classmethod\n    def _ignore_masked_violations_line_range(\n        cls, violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ):\n        \"\"\"Returns whether to ignore error for line-range directives.\n\n        The \"ignore\" list is assumed to ONLY contain NoQaDirectives where\n        action is \"enable\" or \"disable\".\n        \"\"\"\n        result = []\n        for v in violations:\n            # Find the directives that affect the violated rule \"v\", either\n            # because they specifically reference it or because they don't\n            # specify a list of rules, thus affecting ALL rules.\n            ignore_rule = sorted(\n                (\n                    ignore\n                    for ignore in ignore_mask\n                    if not ignore.rules\n                    or (v.rule_code() in cast(Tuple[str, ...], ignore.rules))\n                ),\n                key=lambda ignore: ignore.line_no,\n            )\n            # Determine whether to ignore the violation, based on the relevant\n            # enable/disable directives.\n            if not cls._should_ignore_violation_line_range(v.line_no, ignore_rule):\n                result.append(v)\n        return result\n\n    @classmethod\n    def ignore_masked_violations(\n        cls, violations: List[SQLBaseError], ignore_mask: List[NoQaDirective]\n    ) -> List[SQLBaseError]:\n        \"\"\"Remove any violations specified by ignore_mask.\n\n        This involves two steps:\n        1. Filter out violations affected by single-line \"noqa\" directives.\n        2. Filter out violations affected by disable/enable \"noqa\" directives.\n        \"\"\"\n        ignore_specific = [ignore for ignore in ignore_mask if not ignore.action]\n        ignore_range = [ignore for ignore in ignore_mask if ignore.action]\n        violations = cls._ignore_masked_violations_single_line(\n            violations, ignore_specific\n        )\n        violations = cls._ignore_masked_violations_line_range(violations, ignore_range)\n        return violations\n\n    def num_violations(self, **kwargs) -> int:\n        \"\"\"Count the number of violations.\n\n        Optionally now with filters.\n        \"\"\"\n        violations = self.get_violations(**kwargs)\n        return len(violations)\n\n    def is_clean(self) -> bool:\n        \"\"\"Return True if there are no ignorable violations.\"\"\"\n        return not any(self.get_violations(filter_ignore=True))\n\n    @staticmethod\n    def _log_hints(\n        patch: Union[EnrichedFixPatch, FixPatch], templated_file: TemplatedFile\n    ):\n        \"\"\"Log hints for debugging during patch generation.\"\"\"\n        # This next bit is ALL FOR LOGGING AND DEBUGGING\n        max_log_length = 10\n        if patch.templated_slice.start >= max_log_length:\n            pre_hint = templated_file.templated_str[\n                patch.templated_slice.start\n                - max_log_length : patch.templated_slice.start\n            ]\n        else:\n            pre_hint = templated_file.templated_str[: patch.templated_slice.start]\n        if patch.templated_slice.stop + max_log_length < len(\n            templated_file.templated_str\n        ):\n            post_hint = templated_file.templated_str[\n                patch.templated_slice.stop : patch.templated_slice.stop + max_log_length\n            ]\n        else:\n            post_hint = templated_file.templated_str[patch.templated_slice.stop :]\n        linter_logger.debug(\n            \"        Templated Hint: ...%r <> %r...\", pre_hint, post_hint\n        )\n\n    def fix_string(self) -> Tuple[Any, bool]:\n        \"\"\"Obtain the changes to a path as a string.\n\n        We use the source mapping features of TemplatedFile\n        to generate a list of \"patches\" which cover the non\n        templated parts of the file and refer back to the locations\n        in the original file.\n\n        NB: This is MUCH FASTER than the original approach\n        using difflib in pre 0.4.0.\n\n        There is an important distinction here between Slices and\n        Segments. A Slice is a portion of a file which is determined\n        by the templater based on which portions of the source file\n        are templated or not, and therefore before Lexing and so is\n        completely dialect agnostic. A Segment is determined by the\n        Lexer from portions of strings after templating.\n        \"\"\"\n        linter_logger.debug(\"Original Tree: %r\", self.templated_file.templated_str)\n        assert self.tree\n        linter_logger.debug(\"Fixed Tree: %r\", self.tree.raw)\n\n        # The sliced file is contiguous in the TEMPLATED space.\n        # NB: It has gaps and repeats in the source space.\n        # It's also not the FIXED file either.\n        linter_logger.debug(\"### Templated File.\")\n        for idx, file_slice in enumerate(self.templated_file.sliced_file):\n            t_str = self.templated_file.templated_str[file_slice.templated_slice]\n            s_str = self.templated_file.source_str[file_slice.source_slice]\n            if t_str == s_str:\n                linter_logger.debug(\n                    \"    File slice: %s %r [invariant]\", idx, file_slice\n                )\n            else:\n                linter_logger.debug(\"    File slice: %s %r\", idx, file_slice)\n                linter_logger.debug(\"    \\t\\t\\ttemplated: %r\\tsource: %r\", t_str, s_str)\n\n        original_source = self.templated_file.source_str\n\n        # Make sure no patches overlap and divide up the source file into slices.\n        # Any Template tags in the source file are off limits.\n        source_only_slices = self.templated_file.source_only_slices()\n\n        linter_logger.debug(\"Source-only slices: %s\", source_only_slices)\n\n        # Iterate patches, filtering and translating as we go:\n        linter_logger.debug(\"### Beginning Patch Iteration.\")\n        filtered_source_patches = []\n        dedupe_buffer = []\n        # We use enumerate so that we get an index for each patch. This is entirely\n        # so when debugging logs we can find a given patch again!\n        patch: Union[EnrichedFixPatch, FixPatch]\n        for idx, patch in enumerate(\n            self.tree.iter_patches(templated_str=self.templated_file.templated_str)\n        ):\n            linter_logger.debug(\"  %s Yielded patch: %s\", idx, patch)\n            self._log_hints(patch, self.templated_file)\n\n            # Attempt to convert to source space.\n            try:\n                source_slice = self.templated_file.templated_slice_to_source_slice(\n                    patch.templated_slice,\n                )\n            except ValueError:\n                linter_logger.info(\n                    \"      - Skipping. Source space Value Error. i.e. attempted insertion within templated section.\"\n                )\n                # If we try and slice within a templated section, then we may fail\n                # in which case, we should skip this patch.\n                continue\n\n            # Check for duplicates\n            dedupe_tuple = (source_slice, patch.fixed_raw)\n            if dedupe_tuple in dedupe_buffer:\n                linter_logger.info(\n                    \"      - Skipping. Source space Duplicate: %s\", dedupe_tuple\n                )\n                continue\n\n            # We now evaluate patches in the source-space for whether they overlap\n            # or disrupt any templated sections.\n            # The intent here is that unless explicitly stated, a fix should never\n            # disrupt a templated section.\n            # NOTE: We rely here on the patches being sorted.\n            # TODO: Implement a mechanism for doing templated section fixes. For\n            # now it's just not allowed.\n\n            # Get the affected raw slices.\n            local_raw_slices = self.templated_file.raw_slices_spanning_source_slice(\n                source_slice\n            )\n            local_type_list = [slc.slice_type for slc in local_raw_slices]\n\n            enriched_patch = EnrichedFixPatch(\n                source_slice=source_slice,\n                templated_slice=patch.templated_slice,\n                patch_category=patch.patch_category,\n                fixed_raw=patch.fixed_raw,\n                templated_str=self.templated_file.templated_str[patch.templated_slice],\n                source_str=self.templated_file.source_str[source_slice],\n            )\n\n            # Deal with the easy case of only literals\n            if set(local_type_list) == {\"literal\"}:\n                linter_logger.info(\n                    \"      * Keeping patch on literal-only section: %s\", enriched_patch\n                )\n                filtered_source_patches.append(enriched_patch)\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())\n            # Is it a zero length patch.\n            elif (\n                enriched_patch.source_slice.start == enriched_patch.source_slice.stop\n                and enriched_patch.source_slice.start == local_raw_slices[0].source_idx\n            ):\n                linter_logger.info(\n                    \"      * Keeping insertion patch on slice boundary: %s\",\n                    enriched_patch,\n                )\n                filtered_source_patches.append(enriched_patch)\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())\n            # If it's ONLY templated then we should skip it.\n            elif \"literal\" not in local_type_list:\n                linter_logger.info(\n                    \"      - Skipping patch over templated section: %s\", enriched_patch\n                )\n            # If we span more than two slices then we should just skip it. Too Hard.\n            elif len(local_raw_slices) > 2:\n                linter_logger.info(\n                    \"      - Skipping patch over more than two raw slices: %s\",\n                    enriched_patch,\n                )\n            # If it's an insertion (i.e. the string in the pre-fix template is '') then we\n            # won't be able to place it, so skip.\n            elif not enriched_patch.templated_str:  # pragma: no cover TODO?\n                linter_logger.info(\n                    \"      - Skipping insertion patch in templated section: %s\",\n                    enriched_patch,\n                )\n            # If the string from the templated version isn't in the source, then we can't fix it.\n            elif (\n                enriched_patch.templated_str not in enriched_patch.source_str\n            ):  # pragma: no cover TODO?\n                linter_logger.info(\n                    \"      - Skipping edit patch on templated content: %s\",\n                    enriched_patch,\n                )\n            else:\n                # Identify all the places the string appears in the source content.\n                positions = list(\n                    findall(enriched_patch.templated_str, enriched_patch.source_str)\n                )\n                if len(positions) != 1:\n                    linter_logger.debug(\n                        \"        - Skipping edit patch on non-unique templated content: %s\",\n                        enriched_patch,\n                    )\n                    continue\n                # We have a single occurrence of the thing we want to patch. This\n                # means we can use its position to place our patch.\n                new_source_slice = slice(  # pragma: no cover\n                    enriched_patch.source_slice.start + positions[0],\n                    enriched_patch.source_slice.start\n                    + positions[0]\n                    + len(enriched_patch.templated_str),\n                )\n                enriched_patch = EnrichedFixPatch(  # pragma: no cover\n                    source_slice=new_source_slice,\n                    templated_slice=enriched_patch.templated_slice,\n                    patch_category=enriched_patch.patch_category,\n                    fixed_raw=enriched_patch.fixed_raw,\n                    templated_str=enriched_patch.templated_str,\n                    source_str=enriched_patch.source_str,\n                )\n                linter_logger.debug(  # pragma: no cover\n                    \"      * Keeping Tricky Case. Positions: %s, New Slice: %s, Patch: %s\",\n                    positions,\n                    new_source_slice,\n                    enriched_patch,\n                )\n                filtered_source_patches.append(enriched_patch)  # pragma: no cover\n                dedupe_buffer.append(enriched_patch.dedupe_tuple())  # pragma: no cover\n                continue  # pragma: no cover\n\n        # Sort the patches before building up the file.\n        filtered_source_patches = sorted(\n            filtered_source_patches, key=lambda x: x.source_slice.start\n        )\n        # We now slice up the file using the patches and any source only slices.\n        # This gives us regions to apply changes to.\n        slice_buff = []\n        source_idx = 0\n        for patch in filtered_source_patches:\n            # Are there templated slices at or before the start of this patch?\n            while (\n                source_only_slices\n                and source_only_slices[0].source_idx < patch.source_slice.start\n            ):\n                next_so_slice = source_only_slices.pop(0).source_slice()\n                # Add a pre-slice before the next templated slices if needed.\n                if next_so_slice.start > source_idx:\n                    slice_buff.append(slice(source_idx, next_so_slice.start))\n                # Add the templated slice.\n                slice_buff.append(next_so_slice)\n                source_idx = next_so_slice.stop\n\n            # Is there a gap between current position and this patch?\n            if patch.source_slice.start > source_idx:\n                # Add a slice up to this patch.\n                slice_buff.append(slice(source_idx, patch.source_slice.start))\n\n            # Is this patch covering an area we've already covered?\n            if patch.source_slice.start < source_idx:\n                linter_logger.info(\n                    \"Skipping overlapping patch at Index %s, Patch: %s\",\n                    source_idx,\n                    patch,\n                )\n                # Ignore the patch for now...\n                continue\n\n            # Add this patch.\n            slice_buff.append(patch.source_slice)\n            source_idx = patch.source_slice.stop\n        # Add a tail slice.\n        if source_idx < len(self.templated_file.source_str):\n            slice_buff.append(slice(source_idx, len(self.templated_file.source_str)))\n\n        linter_logger.debug(\"Final slice buffer: %s\", slice_buff)\n\n        # Iterate through the patches, building up the new string.\n        str_buff = \"\"\n        for source_slice in slice_buff:\n            # Is it one in the patch buffer:\n            for patch in filtered_source_patches:\n                if patch.source_slice == source_slice:\n                    # Use the patched version\n                    linter_logger.debug(\n                        \"%-30s    %s    %r > %r\",\n                        f\"Appending {patch.patch_category} Patch:\",\n                        patch.source_slice,\n                        patch.source_str,\n                        patch.fixed_raw,\n                    )\n                    str_buff += patch.fixed_raw\n                    break\n            else:\n                # Use the raw string\n                linter_logger.debug(\n                    \"Appending Raw:                    %s     %r\",\n                    source_slice,\n                    self.templated_file.source_str[source_slice],\n                )\n                str_buff += self.templated_file.source_str[source_slice]\n\n        # The success metric here is whether anything ACTUALLY changed.\n        return str_buff, str_buff != original_source\n\n    def persist_tree(self, suffix: str = \"\") -> bool:\n        \"\"\"Persist changes to the given path.\"\"\"\n        write_buff, success = self.fix_string()\n\n        if success:\n            fname = self.path\n            # If there is a suffix specified, then use it.s\n            if suffix:\n                root, ext = os.path.splitext(fname)\n                fname = root + suffix + ext\n            self._safe_create_replace_file(fname, write_buff, self.encoding)\n        return success\n\n    @staticmethod\n    def _safe_create_replace_file(fname, write_buff, encoding):\n        # Write to a temporary file first, so in case of encoding or other\n        # issues, we don't delete or corrupt the user's existing file.\n        dirname, basename = os.path.split(fname)\n        with tempfile.NamedTemporaryFile(\n            mode=\"w\",\n            encoding=encoding,\n            prefix=basename,\n            dir=dirname,\n            suffix=os.path.splitext(fname)[1],\n            delete=False,\n        ) as tmp:\n            tmp.file.write(write_buff)\n            tmp.flush()\n            os.fsync(tmp.fileno())\n        # Once the temp file is safely written, replace the existing file.\n        shutil.move(tmp.name, fname)\n","header":{"file":{"path":"src/sqlfluff/core/linter/linted_file.py"},"misc_line":null},"index_line":null,"is_binary_file":false,"binary_line":null,"minus_file":{"path":"a/src/sqlfluff/core/linter/linted_file.py"},"plus_file":{"path":"b/src/sqlfluff/core/linter/linted_file.py"},"hunks":[{"descriptor":{"old_range":{"start":7,"length":6},"new_range":{"start":7,"length":8},"section":""},"line_group":{"all_lines":[{"content":"","type":"context"},{"content":"import os","type":"context"},{"content":"import logging","type":"context"},{"content":"import shutil","type":"added"},{"content":"import tempfile","type":"added"},{"content":"from typing import (","type":"context"},{"content":"    Any,","type":"context"},{"content":"    Iterable,","type":"context"}]},"modified_entities":[],"added_entities":[],"deleted_entities":[]},{"descriptor":{"old_range":{"start":493,"length":7},"new_range":{"start":495,"length":24},"section":"def persist_tree(self, suffix: str = \"\") -> bool:"},"line_group":{"all_lines":[{"content":"            if suffix:","type":"context"},{"content":"                root, ext = os.path.splitext(fname)","type":"context"},{"content":"                fname = root + suffix + ext","type":"context"},{"content":"            # Actually write the file.","type":"deleted"},{"content":"            with open(fname, \"w\", encoding=self.encoding) as f:","type":"deleted"},{"content":"                f.write(write_buff)","type":"deleted"},{"content":"            self._safe_create_replace_file(fname, write_buff, self.encoding)","type":"added"},{"content":"        return success","type":"context"},{"content":"","type":"added"},{"content":"    @staticmethod","type":"added"},{"content":"    def _safe_create_replace_file(fname, write_buff, encoding):","type":"added"},{"content":"        # Write to a temporary file first, so in case of encoding or other","type":"added"},{"content":"        # issues, we don't delete or corrupt the user's existing file.","type":"added"},{"content":"        dirname, basename = os.path.split(fname)","type":"added"},{"content":"        with tempfile.NamedTemporaryFile(","type":"added"},{"content":"            mode=\"w\",","type":"added"},{"content":"            encoding=encoding,","type":"added"},{"content":"            prefix=basename,","type":"added"},{"content":"            dir=dirname,","type":"added"},{"content":"            suffix=os.path.splitext(fname)[1],","type":"added"},{"content":"            delete=False,","type":"added"},{"content":"        ) as tmp:","type":"added"},{"content":"            tmp.file.write(write_buff)","type":"added"},{"content":"            tmp.flush()","type":"added"},{"content":"            os.fsync(tmp.fileno())","type":"added"},{"content":"        # Once the temp file is safely written, replace the existing file.","type":"added"},{"content":"        shutil.move(tmp.name, fname)","type":"added"},{"content":"","type":"context"}]},"modified_entities":[],"added_entities":[],"deleted_entities":[]}]}],"old_commit_hash":"a10057635e5b2559293a676486f0b730981f037a","new_commit_hash":"e3fb8535acf3cdded95476fbfc723a8359550d94","commit_message":"Ensure the temporary file is written to disk before using it","commit_date":"2021-10-26T18:45:50-04:00","metadata":{}}