void (@NotNull final MyTestProcessRunner runner, @NotNull final String stdout, @NotNull final String stderr, @NotNull final String all, int exitCode) { final Set<String> expectedInterpreters = myInterpreters.entrySet().stream() .filter(intAndExp -> intAndExp.getValue() != null) .filter(o -> o.getValue().myUntilStep > runner.getCurrentRerunStep()) // Remove interp. which shouldn't be launched on this step .map(intAndExp -> intAndExp.getKey()) .collect(Collectors.toSet()); // Interpreters are used in tox.ini, so there should be such text for (final String interpreterName : expectedInterpreters) { assertThat(all) .describedAs(String.format("No %s used from tox.ini", interpreterName)) .contains(interpreterName); } if (!stderr.isEmpty()) { Logger.getInstance(PyToxTest.class).warn(stderr); } final Set<String> checkedInterpreters = new HashSet<>(); final Collection<String> skippedMissingInterpreters = new HashSet<>(); // Interpreter should either run tests or mentioned as NotFound for (final SMTestProxy interpreterSuite : runner.getTestProxy().getChildren()) { final String interpreterName = interpreterSuite.getName(); assert interpreterName.startsWith("py") : String .format("Bad interpreter name: %s. Tree is %s \n", interpreterName, getTestTree(interpreterSuite, 0)); checkedInterpreters.add(interpreterName); final InterpreterExpectations expectations = myInterpreters.get(interpreterName); if (expectations == null) { LOGGER.warn(String.format("Launched %s, but no expectation provided, skipping", interpreterName)); continue; } if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("ERROR")) { // Interpreter failed to run final String testOutput = BaseSMTRunnerTestCase.getTestOutput(interpreterSuite.getChildren().get(0)); if (testOutput.contains("InterpreterNotFound")) { // Skipped with out of "skip_missing_interpreters = True" Logger.getInstance(PyToxTest.class).warn(String.format("Interpreter %s does not exit", interpreterName)); skippedMissingInterpreters.add(interpreterName); // Interpreter does not exit continue; } // Some other error? Assert .assertFalse(String.format("Interpreter %s should not fail, but failed: %s", interpreterName, BaseSMTRunnerTestCase .getTestOutput(interpreterSuite)), expectations.myExpectedSuccess); continue; } if (interpreterSuite.getChildren().size() == 1 && interpreterSuite.getChildren().get(0).getName().endsWith("SKIP")) { // The only reason it may be skipped is it does not exist and skip_missing_interpreters = True final String output = BaseSMTRunnerTestCase.getTestOutput(interpreterSuite); assertThat(output) .describedAs("Test marked skipped but not because interpreter not found") .contains("InterpreterNotFound"); } // Interpretr run success, //At least one interpreter tests should passed final int numberOfTests = new SMRootTestsCounter(interpreterSuite.getRoot()).getPassedTestsCount(); assertThat(numberOfTests) .describedAs(String.format("Not enough test passed, should %s at least", myMinimumSuccessTestCount)) .isGreaterThanOrEqualTo(myMinimumSuccessTestCount); assertThat(numberOfTests) .describedAs(String.format("Too many tests passed, should %s maximum", myMaximumSuccessTestCount)) .isLessThanOrEqualTo(myMaximumSuccessTestCount); // Check expected output final String message = String.format("Interpreter %s does not have expected string in output. \n ", interpreterName) + String.format("All: %s \n", all) + String.format("Test tree: %s \n", getTestTree(interpreterSuite, 0)) + String.format("Error: %s \n", stderr); assertThat(message) .describedAs(BaseSMTRunnerTestCase.getTestOutput(interpreterSuite)) .contains(expectations.myExpectedOutput); } // Skipped interpreters should not be checked since we do not know which interpreters used on environemnt // But if all interpreters are skipped, we can't say we tested something. assert !skippedMissingInterpreters.equals(checkedInterpreters) : "All interpreters skipped (they do not exist on platform), " + "we test nothing"; expectedInterpreters.removeAll(skippedMissingInterpreters); checkedInterpreters.removeAll(skippedMissingInterpreters); assertThat(checkedInterpreters) .describedAs(String.format("No all interpreters from tox.ini used (test tree \n%s\n )", getTestTree(runner.getTestProxy(), 0))) .are(new Condition<>() { @Override public boolean matches(String value) { return expectedInterpreters.contains(value); } }); }