wathen: > Ok here is my review:
> 
>     1. Run fine but take a while to do so, maybe use a smaller/easier test set? Also some strange warnings (hard to tell where these are coming from):
> 
> 
> ```
>  <string>:1: RuntimeWarning: invalid value encountered in power 
>  'NoneType' object has no attribute 'T'
> ```
> 
>     1. Run fine with the expected warning for mantid problems:
> 
> 
> ```
> Missing dependencies for fit.
> Details: Requirements are missing for mantid parser: No module named 'mantid'
> ```
> 
>     1. Raises the expected error along with an informative message (but with weird spaces between words):
> 
> 
> ```
> Error while running FitBenchmarking. Exiting. See below for more information.
> FitBenchmarking ran with no results
> Details: The user chosen options and/or problem setup resulted in  all minimizers and/or parsers raising an exception.  This is likely due to the way `algorithm_type` was set  in the options or the selected problem set requires  additional software to be installed. Please review your  options setup and/or problem set then re-run FitBenmarking.
> ```
1) I changed the default to NIST low diff data set however this still produces the warning you have seen. Maybe it would be better to run with a smaller number of minimizers as a default. I will create an issue for this.

3) Fixed spacing issues.  

diff --git a/docs/source/users/running.rst b/docs/source/users/running.rst
index 20b9cb21..387cef6f 100644
--- a/docs/source/users/running.rst
+++ b/docs/source/users/running.rst
@@ -10,7 +10,7 @@ directory along with a template for passing in options.
 To run with any of these problem sets, you should pass the path to the
 `fitbenchmarking` command.
 
-  e.g ``fitbenchmarking examples/benchmarking_problems/simple_tests``
+  e.g ``fitbenchmarking examples/benchmark_problems/NIST/low_difficulty``
 
 An options file can also be passed with the ``-o`` argument.
 The template in `examples` is heavily documented and serves as a good
diff --git a/fitbenchmarking/core/fitting_benchmarking.py b/fitbenchmarking/core/fitting_benchmarking.py
index c27a7a64..2b5f69a4 100644
--- a/fitbenchmarking/core/fitting_benchmarking.py
+++ b/fitbenchmarking/core/fitting_benchmarking.py
@@ -9,13 +9,14 @@ from __future__ import absolute_import, division, print_function
 from collections import defaultdict
 import timeit
 import warnings
+import os
 
 import numpy as np
 
 from fitbenchmarking.controllers.controller_factory import ControllerFactory
 from fitbenchmarking.parsing.parser_factory import parse_problem_file
 from fitbenchmarking.utils.exceptions import NoResultsError, \
-    UnknownMinimizerError, UnsupportedMinimizerError
+    UnknownMinimizerError, UnsupportedMinimizerError, MissingSoftwareError
 from fitbenchmarking.jacobian.jacobian_factory import create_jacobian
 from fitbenchmarking.utils import fitbm_result, misc, output_grabber
 
@@ -61,15 +62,6 @@ def benchmark(options, data_dir):
 
     options.minimizers = minimizer_dict
 
-    # If the results are an empty list then this means that all minimizers
-    # raise an exception and the tables will produce errors if they run.
-    if results == []:
-        message = "The user chosen options setup resulted in all minimizers " \
-                  "raising an exception. This is likely due to the way " \
-                  "`algorithm_type` was set in the options. Please review " \
-                  "your options setup and re-run FitBenmarking."
-        raise NoResultsError(message)
-
     # Used to group elements in list by name
     results_dict = defaultdict(list)
     for problem_result in results:
@@ -92,32 +84,49 @@ def loop_over_benchmark_problems(problem_group, options):
              list of failed problems and dictionary of unselected minimizers
     :rtype: tuple(list, list, dict)
     """
-
     grabbed_output = output_grabber.OutputGrabber(options)
     results = []
     failed_problems = []
     for i, p in enumerate(problem_group):
-        with grabbed_output:
-            parsed_problem = parse_problem_file(p, options)
-        parsed_problem.correct_data()
-
-        name = parsed_problem.name
-
+        problem_passed = True
         info_str = " Running data from: {} {}/{}".format(
-            name, i + 1, len(problem_group))
+            os.path.basename(p), i + 1, len(problem_group))
         LOGGER.info('\n' + '#' * (len(info_str) + 1))
         LOGGER.info(info_str)
         LOGGER.info('#' * (len(info_str) + 1))
+        try:
+            with grabbed_output:
+                parsed_problem = parse_problem_file(p, options)
+                parsed_problem.correct_data()
+        except Exception as e:
+            info_str = " Could not run data from: {} {}/{}".format(
+                p, i + 1, len(problem_group))
+            LOGGER.warn(e)
+            problem_passed = False
+
+        if problem_passed:
+            ##############################
+            # Loops over starting values #
+            ##############################
+            problem_results, problem_fails, \
+                unselected_minimzers, minimizer_dict = \
+                loop_over_starting_values(
+                    parsed_problem, options, grabbed_output)
+            results.extend(problem_results)
+            failed_problems.extend(problem_fails)
 
-        ##############################
-        # Loops over starting values #
-        ##############################
-        problem_results, problem_fails, \
-            unselected_minimzers, minimizer_dict = \
-            loop_over_starting_values(parsed_problem, options, grabbed_output)
+    # If the results are an empty list then this means that all minimizers
+    # raise an exception and the tables will produce errors if they run.
+    if results == []:
+        message = "The user chosen options and/or problem setup resulted in " \
+                  "all minimizers and/or parsers raising an exception. " \
+                  "This is likely due to the way `algorithm_type` was set " \
+                  "in the options or the selected problem set requires " \
+                  "additional software to be installed. Please review your " \
+                  "options setup and/or problem set then re-run " \
+                  "FitBenmarking."
+        raise NoResultsError(message)
 
-        results.extend(problem_results)
-        failed_problems.extend(problem_fails)
     return results, failed_problems, unselected_minimzers, minimizer_dict
 
 
diff --git a/fitbenchmarking/parsing/nist_parser.py b/fitbenchmarking/parsing/nist_parser.py
index d29573e7..29bc41c7 100644
--- a/fitbenchmarking/parsing/nist_parser.py
+++ b/fitbenchmarking/parsing/nist_parser.py
@@ -57,11 +57,15 @@ class NISTParser(Parser):
             nist_func_definition(function=fitting_problem.equation,
                                  param_names=starting_values[0].keys())
         fitting_problem.format = "nist"
+        try:
+            jacobian = self._parse_jacobian(name)
+            fitting_problem.jacobian = \
+                nist_jacobian_definition(jacobian=jacobian,
+                                         param_names=starting_values[0].keys())
+        except NoJacobianError:
+            LOGGER.warn("WARNING: Could not find analytic Jacobian "
+                        "information for {} problem".format(name))
 
-        jacobian = self._parse_jacobian(name)
-        fitting_problem.jacobian = \
-            nist_jacobian_definition(jacobian=jacobian,
-                                     param_names=starting_values[0].keys())
         return fitting_problem
 
     def _parse_jacobian(self, name):

Use a better first example 
**Description of the documentation**
The suggested example is to use the `simple_tests` problem set.

This includes Mantid data, and so will fail if Mantid isn't installed.  Change the default to be, e.g., a set of NIST problems, so that it can be run independently of third party software.


diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
index 0a0b37b6..43743a80 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py
@@ -160,23 +160,6 @@ class BenchmarkTests(unittest.TestCase):
         self.shared_tests(expected_names, expected_unselected_minimzers,
                           expected_minimzers)
 
-    @mock.patch('{}.loop_over_benchmark_problems'.format(FITTING_DIR))
-    def test_check_no_results_produced(self, loop_over_benchmark_problems):
-        """
-        Checks benchmarking raises an error when no results are produced
-        """
-
-        results = []
-        problem_fails = []
-        expected_unselected_minimzers = {"scipy": []}
-        expected_minimzers = copy.copy(self.all_minimzers)
-        loop_over_benchmark_problems.return_value = \
-            (results, problem_fails, expected_unselected_minimzers,
-             expected_minimzers)
-        with self.assertRaises(NoResultsError):
-            _, _, _ = \
-                benchmark(self.options, self.default_parsers_dir)
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
index 1c2d2f5b..e56d758b 100644
--- a/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
+++ b/fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py
@@ -9,7 +9,7 @@ import unittest
 import mock
 
 from fitbenchmarking import mock_problems
-from fitbenchmarking.utils import fitbm_result
+from fitbenchmarking.utils import fitbm_result, exceptions
 from fitbenchmarking.core.fitting_benchmarking import \
     loop_over_benchmark_problems
 from fitbenchmarking.parsing.parser_factory import parse_problem_file
@@ -134,7 +134,6 @@ class LoopOverBenchmarkProblemsTests(unittest.TestCase):
         self.problem_fails = ['Random_failed_problem_1',
                               'Random_failed_problem_2']
         loop_over_starting_values.side_effect = self.mock_func_call
-        self.problem_group = []
         self.problem_group = [os.path.join(self.default_parsers_dir,
                                            "cubic.dat")]
 
@@ -142,6 +141,20 @@ class LoopOverBenchmarkProblemsTests(unittest.TestCase):
         expected_list_length = len(self.list_results)
         self.shared_tests(expected_list_length, expected_problem_fails)
 
+    @mock.patch('{}.loop_over_starting_values'.format(FITTING_DIR))
+    def test_check_no_results_produced(self, loop_over_starting_values):
+        """
+        Checks that multiple failed problems are reported correctly
+        """
+        loop_over_starting_values.side_effect = self.mock_func_call
+        problem_group = [os.path.join(self.default_parsers_dir,
+                                      "META.txt")]
+
+        with self.assertRaises(exceptions.NoResultsError):
+            _, _, _ = \
+                loop_over_benchmark_problems(problem_group,
+                                             self.options)
+
 
 if __name__ == "__main__":
     unittest.main()

pip install -e .[dev]
parse_log_pytest
requirements.txt
pytest>=3.6 pytest-cov coveralls python-coveralls urllib3==1.23 mock pytest
apt-get update apt-get install -y gcc
3.9
requirements.txt
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
bumps==0.9.3
certifi==2025.1.31
charset-normalizer==3.4.1
configparser==7.2.0
contourpy==1.3.0
coverage==4.5.4
coveralls==3.3.1
cycler==0.12.1
DFO-LS==1.5.4
DFOGN==1.0.2
docopt==0.6.2
docutils==0.21.2
exceptiongroup==1.2.2
-e git+https://github.com/fitbenchmarking/fitbenchmarking.git@eddaae19cee2e5dc1c8d15519a2ad07bb5c58ed0#egg=FitBenchmarking
fonttools==4.56.0
h5py==3.13.0
idna==3.10
iminuit==2.30.1
importlib_resources==6.5.2
iniconfig==2.1.0
Jinja2==3.1.6
kiwisolver==1.4.7
lxml==5.3.1
MarkupSafe==3.0.2
matplotlib==3.9.4
mock==5.2.0
numpy==2.0.2
packaging==24.2
pandas==2.2.3
pillow==11.1.0
pluggy==1.5.0
pyparsing==3.2.3
pytest==8.3.5
pytest-cov==2.10.1
python-coveralls==2.9.3
python-dateutil==2.9.0.post0
pytz==2025.2
PyYAML==6.0.2
requests==2.32.3
sasmodels==1.0.9
scipy==1.13.1
six==1.17.0
tinycc==1.1
tomli==2.2.1
tzdata==2025.2
urllib3==1.23
zipp==3.21.0

name: fitbenchmarking
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2025.2.25=h06a4308_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=3.0.16=h5eee18b_0
  - pip=25.0=py39h06a4308_0
  - python=3.9.21=he870216_1
  - readline=8.2=h5eee18b_0
  - setuptools=75.8.0=py39h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - wheel=0.45.1=py39h06a4308_0
  - xz=5.6.4=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - bumps==0.9.3
      - certifi==2025.1.31
      - charset-normalizer==3.4.1
      - configparser==7.2.0
      - contourpy==1.3.0
      - coverage==4.5.4
      - coveralls==3.3.1
      - cycler==0.12.1
      - dfo-ls==1.5.4
      - dfogn==1.0.2
      - docopt==0.6.2
      - docutils==0.21.2
      - exceptiongroup==1.2.2
      - fonttools==4.56.0
      - h5py==3.13.0
      - idna==3.10
      - iminuit==2.30.1
      - importlib-resources==6.5.2
      - iniconfig==2.1.0
      - jinja2==3.1.6
      - kiwisolver==1.4.7
      - lxml==5.3.1
      - markupsafe==3.0.2
      - matplotlib==3.9.4
      - mock==5.2.0
      - numpy==2.0.2
      - packaging==24.2
      - pandas==2.2.3
      - pillow==11.1.0
      - pluggy==1.5.0
      - pyparsing==3.2.3
      - pytest==8.3.5
      - pytest-cov==2.10.1
      - python-coveralls==2.9.3
      - python-dateutil==2.9.0.post0
      - pytz==2025.2
      - pyyaml==6.0.2
      - requests==2.32.3
      - sasmodels==1.0.9
      - scipy==1.13.1
      - six==1.17.0
      - tinycc==1.1
      - tomli==2.2.1
      - tzdata==2025.2
      - urllib3==1.23
      - zipp==3.21.0
prefix: /opt/conda/envs/fitbenchmarking

fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py::LoopOverBenchmarkProblemsTests::test_check_no_results_produced
fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py::BenchmarkTests::test_check_no_unselected_minimizers fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark.py::BenchmarkTests::test_check_unselected_minimizers fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py::LoopOverBenchmarkProblemsTests::test_run_multiple_benchmark_problems fitbenchmarking/core/tests/test_fitting_benchmarking_benchmark_problems.py::LoopOverBenchmarkProblemsTests::test_run_multiple_failed_problems
fitbenchmarking/parsing/nist_parser.py fitbenchmarking/core/fitting_benchmarking.py docs/source/users/running.rst
fitbenchmarking/parsing/nist_parser.py fitbenchmarking/core/fitting_benchmarking.py docs/source/users/running.rst