diff --git a/concentration/settings.py b/concentration/settings.py
index <HASH>..<HASH> 100644
--- a/concentration/settings.py
+++ b/concentration/settings.py
@@ -28,7 +28,9 @@ for config_file_path in ('/etc/concentration.distractors', os.path.expanduser('~
 for config_file_path in ('/etc/concentration.safe', os.path.expanduser('~/.concentration.safe')):
     if os.path.isfile(config_file_path):
         with open(config_file_path) as config_file:
-            DISTRACTORS.update(config_file.read().splitlines())
+            # Convert the whitelist into a set of domains and then remove that
+            # set from DISTRACTORS
+            DISTRACTORS -= set(config_file.read().splitlines())
 
 DISTRACTORS.discard('')
Fixed a bug where the whitelist domains were being added to the list of DISTRACTORS instead of being removed