anchor.diff:    ''
     start_all()
   
  -  # Set up authentification across the cluster
  -  for node in [submit, control, dbd, node1, node2, node3]:
  -
  -      node.wait_for_unit("default.target")
  -
  -      node.succeed("mkdir /etc/munge")
  -      node.succeed(
  -          "echo '${mungekey}' > /etc/munge/munge.key"
  -      )
  -      node.succeed("chmod 0400 /etc/munge/munge.key")
  -      node.succeed("chown munge:munge /etc/munge/munge.key")
  -      node.succeed("systemctl restart munged")
  -
  -      node.wait_for_unit("munged")
  -
  -
  -  # Restart the services since they have probably failed due to the munge init
  -  # failure
  +  # Make sure DBD is up after DB initialzation
     with subtest("can_start_slurmdbd"):
         dbd.succeed("systemctl restart slurmdbd")
         dbd.wait_for_unit("slurmdbd.service")
  
positive.code:           # find the srun job from above in the database
            control.succeed("sleep 5")
            control.succeed("sacct | grep hostname")
    '';
  })
  
negative.code: import ./make-test-python.nix ({ lib, pkgs, ... }:
  let
      slurmconfig = {
        services.slurm = {
          controlMachine = "control";
          nodeName = [ "node[1-3] CPUs=1 State=UNKNOWN" ];
          partitionName = [ "debug Nodes=node[1-3] Default=YES MaxTime=INFINITE State=UP" ];
          extraConfig = ''
            AccountingStorageHost=dbd
            AccountingStorageType=accounting_storage/slurmdbd
          '';
        };
        environment.systemPackages = [ mpitest ];
        networking.firewall.enable = false;
        systemd.tmpfiles.rules = [
          "f /etc/munge/munge.key 0400 munge munge - mungeverryweakkeybuteasytointegratoinatest"
        ];
      };
  
      mpitest = let
        mpitestC = pkgs.writeText "mpitest.c" ''
          #include <stdio.h>
          #include <stdlib.h>
          #include <mpi.h>
  
          int
          main (int argc, char *argv[])
          {
            int rank, size, length;
            char name[512];
  
            MPI_Init (&argc, &argv);
            MPI_Comm_rank (MPI_COMM_WORLD, &rank);
            MPI_Comm_size (MPI_COMM_WORLD, &size);
            MPI_Get_processor_name (name, &length);
  
            if ( rank == 0 ) printf("size=%d\n", size);
  
            printf ("%s: hello world from process %d of %d\n", name, rank, size);
  
            MPI_Finalize ();
  
            return EXIT_SUCCESS;
          }
        '';
      in pkgs.runCommandNoCC "mpitest" {} ''
        mkdir -p $out/bin
        ${pkgs.openmpi}/bin/mpicc ${mpitestC} -o $out/bin/mpitest
      '';
  in {
    name = "slurm";
  
  