diff --git a/test/ExerciseRunner/CgiRunnerTest.php b/test/ExerciseRunner/CgiRunnerTest.php
index <HASH>..<HASH> 100644
--- a/test/ExerciseRunner/CgiRunnerTest.php
+++ b/test/ExerciseRunner/CgiRunnerTest.php
@@ -351,7 +351,7 @@ class CgiRunnerTest extends PHPUnit_Framework_TestCase
         $this->expectOutputString($exp);
 
         $success = $this->runner->run(
-            new Input('app', ['program' => '']),
+            new Input('app', ['program' => 'not-existing-file.php']),
             $output
         );
         $this->assertFalse($success);
Some PHP versions are printing warnings if empty filename passed to cgi