package com.orange.clara.cloud.servicedbdumper.integrations;

import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
import com.orange.clara.cloud.servicedbdumper.dbdumper.DatabaseRefManager;
import com.orange.clara.cloud.servicedbdumper.dbdumper.core.dbdrivers.DatabaseDriver;
import com.orange.clara.cloud.servicedbdumper.dbdumper.core.dbdrivers.DbDumpersFactory;
import com.orange.clara.cloud.servicedbdumper.exception.CannotFindDatabaseDumperException;
import com.orange.clara.cloud.servicedbdumper.exception.DatabaseExtractionException;
import com.orange.clara.cloud.servicedbdumper.exception.ServiceKeyException;
import com.orange.clara.cloud.servicedbdumper.filer.Filer;
import com.orange.clara.cloud.servicedbdumper.integrations.model.DatabaseAccess;
import com.orange.clara.cloud.servicedbdumper.model.*;
import com.orange.clara.cloud.servicedbdumper.repo.DatabaseDumpFileRepo;
import com.orange.clara.cloud.servicedbdumper.repo.DbDumperServiceInstanceRepo;
import com.orange.clara.cloud.servicedbdumper.service.DbDumperServiceInstanceBindingService;
import com.orange.clara.cloud.servicedbdumper.utiltest.ReportIntegration;
import com.orange.clara.cloud.servicedbdumper.utiltest.ReportManager;
import org.cloudfoundry.community.servicebroker.exception.*;
import org.cloudfoundry.community.servicebroker.model.ServiceInstance;
import org.cloudfoundry.community.servicebroker.model.ServiceInstanceBinding;
import org.cloudfoundry.community.servicebroker.model.ServiceInstanceLastOperation;
import org.cloudfoundry.community.servicebroker.service.ServiceInstanceBindingService;
import org.cloudfoundry.community.servicebroker.service.ServiceInstanceService;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;

import java.io.*;
import java.net.Socket;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

import static org.fest.assertions.Assertions.assertThat;
import static org.fest.assertions.Fail.fail;
import static org.junit.Assume.assumeTrue;

/**
 * Copyright (C) 2016 Orange
 * <p>
 * This software is distributed under the terms and conditions of the 'Apache-2.0'
 * license which can be found in the file 'LICENSE' in this package distribution
 * or at 'https://opensource.org/licenses/Apache-2.0'.
 * <p>
 * Author: <NAME>
 * Date: 25/03/2016
 */
abstract public class AbstractIntegrationTest {
    protected final static String DATABASE_SOURCE_NAME = "dbdumpertestsource";
    protected final static String DATABASE_TARGET_NAME = "dbdumpertesttarget";
    protected final static String BINDING_ID = "db-dumper-service-binding";
    protected ReportIntegration reportIntegration;
    protected Map<DatabaseType, DatabaseAccess> databaseAccessMap = Maps.newHashMap();

    protected Logger logger = LoggerFactory.getLogger(AbstractIntegrationTest.class);
    @Autowired
    protected ServiceBrokerRequestForge requestForge;
    @Autowired
    protected DbDumpersFactory dbDumpersFactory;

    @Autowired
    protected ServiceInstanceService dbDumperServiceInstanceService;

    @Autowired
    protected ServiceInstanceBindingService serviceInstanceBindingService;

    @Value("${int.mysql.server:mysql://root@localhost/dbdumpertestsource}")
    protected String mysqlServer;

    @Value("${int.postgres.server:postgres://postgres@localhost/dbdumpertestsource}")
    protected String postgresServer;

    @Value("${int.redis.server:redis://localhost}")
    protected String redisServer;

    @Value("${int.mongodb.server:mongodb://localhost/dbdumpertestsource}")
    protected String mongoServer;
    @Value("${test.populate.data.retry:5}")
    protected int populateDataRetry;
    @Autowired
    protected Filer filer;

    @Autowired
    protected DatabaseRefManager databaseRefManager;
    @Autowired
    protected DbDumperServiceInstanceRepo serviceInstanceRepo;
    protected DatabaseType currentDatabaseType;
    protected String serviceIdSource;
    protected String bindingId;
    protected String serviceIdTarget;
    protected boolean skipCleaning;
    @Value("${test.timeout.action:3}")
    protected int timeoutAction;
    protected String prefixReportName = "";
    @Value("${mongodb.fake.data.file:classpath:data/fake-data-mongodb.bin}")
    private File mongodbFakeData;
    @Value("${mysql.fake.data.file:classpath:data/fake-data-mysql.sql}")
    private File mysqlFakeData;
    @Value("${redis.fake.data.file:classpath:data/fake-data-redis.rdmp}")
    private File redisFakeData;
    @Value("${postgres.fake.data.file:classpath:data/fake-data-postgres.sql}")
    private File postgresFakeData;
    @Value("${binaries.db.folder:classpath:binaries}")
    private File binariesFolder;
    @Autowired
    @Qualifier("postgresBinaryRestore")
    private File psqlBinary;
    @Value("${int.check.binaries:true}")
    private boolean checkBinariesValid;
    @Autowired
    @Qualifier("mysqlBinaryRestore")
    private File mysqlBinary;
    @Autowired
    private DatabaseDumpFileRepo dumpFileRepo;
    @Value("${test.chunk.size.diff:2097152}")
    private int chunkSizeDiff;

    @Before
    public void init() throws DatabaseExtractionException {
        this.prefixReportName = humanize.Humanize.decamelize(this.getClass().getSimpleName()) + " ";
        skipCleaning = false;
        currentDatabaseType = null;
        bindingId = null;
        serviceIdSource = null;
        serviceIdTarget = null;
        this.requestForge.setMetadata(null);
        this.populateDatabaseAccessMap();
    }

    public void doBeforeTest(DatabaseType databaseType) throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException {
        boolean isBinariesValid = this.isBinariesValid(databaseType);
        boolean isServerListening = isServerListening(databaseType);
        String skipMessage = "";
        if (!isBinariesValid) {
            this.skipCleaning = true;
            skipMessage = String.format("Binaries for database %s use for integrations test are not correct, you're not running on linux 64, please set %s.dump.bin.path and %s.restore.bin.path .\nSkipping test.",
                    databaseType.toString().toLowerCase(),
                    databaseType.toString().toLowerCase(),
                    databaseType.toString().toLowerCase()
            );
            this.reportIntegration.setSkipped(true);
            this.reportIntegration.setSkippedReason(skipMessage);
            assumeTrue(skipMessage, false);
        }
        if (!isServerListening) {
            skipMessage = String.format("Server(s) for '%s' not accessible, skipping test.", databaseType.toString().toLowerCase());
            this.skipCleaning = true;
            this.reportIntegration.setSkipped(true);
            this.reportIntegration.setSkippedReason(skipMessage);
            assumeTrue(skipMessage, false);
        }
        this.populateData(databaseType);
    }

    public void cleanDatabase(DatabaseType databaseType) throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException {
        this.dropDatabase(databaseType);
    }

    @After
    public void cleanAfterTest() throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException, ServiceBrokerAsyncRequiredException, ServiceBrokerException {
        if (this.skipCleaning) {
            return;
        }
        if (bindingId != null) {
            try {
                this.deleteBinding();
            } catch (ServiceBrokerAsyncRequiredException | ServiceBrokerException e) {
            }
        }
        if (serviceIdSource != null && !serviceIdSource.isEmpty()) {
            this.deleteServiceInstance(serviceIdSource);
        }
        if (serviceIdTarget != null && !serviceIdTarget.isEmpty()) {
            this.deleteServiceInstance(serviceIdTarget);
        }
        Iterable<DatabaseDumpFile> databaseDumpFiles = this.dumpFileRepo.findAll();
        for (DatabaseDumpFile databaseDumpFile : databaseDumpFiles) {
            String filePath = databaseDumpFile.getDbDumperServiceInstance().getDatabaseRef().getName() + "/" + databaseDumpFile.getFileName();
            try {
                this.filer.delete(filePath);
                this.dumpFileRepo.delete(databaseDumpFile);
            } catch (Exception e) {

            }

        }
        if (currentDatabaseType == null) {
            return;
        }
        this.cleanDatabase(currentDatabaseType);
    }

    protected void deleteServiceInstance(String instanceId) throws ServiceBrokerAsyncRequiredException, ServiceBrokerException {
        this.dbDumperServiceInstanceService.deleteServiceInstance(this.requestForge.createDeleteServiceRequest(instanceId));
    }

    abstract public String getDbParamsForDump(DatabaseType databaseType);

    abstract public String getDbParamsForRestore(DatabaseType databaseType);

    protected void createSourceDatabaseDump(DatabaseType databaseType) throws ServiceBrokerException, ServiceInstanceExistsException, ServiceBrokerAsyncRequiredException {
        this.dbDumperServiceInstanceService.createServiceInstance(this.requestForge.createNewDumpRequest(this.getDbParamsForDump(databaseType), serviceIdSource));
        if (!this.isFinishedAction(serviceIdSource)) {
            fail("Creating dump for source database failed");
        }
    }

    protected void createSourceDatabaseDumpFromExistingService(DatabaseType databaseType) throws ServiceInstanceDoesNotExistException, ServiceInstanceUpdateNotSupportedException, ServiceBrokerException, ServiceBrokerAsyncRequiredException {
        this.dbDumperServiceInstanceService.updateServiceInstance(this.requestForge.createDumpFromExistingServiceRequest(this.getDbParamsForDump(databaseType), serviceIdSource));
        if (!this.isFinishedAction(serviceIdSource)) {
            fail("Creating new dump for source database failed");
        }
    }

    protected ServiceInstanceBinding getBinding(Map<String, Object> params) throws ServiceInstanceBindingExistsException, ServiceBrokerException {
        this.loadBeforeAction();
        return this.serviceInstanceBindingService.createServiceInstanceBinding(this.requestForge.createBindingCreationRequest(serviceIdSource, bindingId, params));
    }

    protected ServiceInstanceBinding deleteBinding() throws ServiceBrokerAsyncRequiredException, ServiceBrokerException {
        this.loadBeforeAction();
        return this.serviceInstanceBindingService.deleteServiceInstanceBinding(this.requestForge.createBindingDeletionRequest(bindingId));
    }

    protected void createTargetDatabaseDump(DatabaseType databaseType) throws ServiceBrokerException, ServiceInstanceExistsException, ServiceBrokerAsyncRequiredException {
        this.dbDumperServiceInstanceService.createServiceInstance(this.requestForge.createNewDumpRequest(this.getDbParamsForRestore(databaseType), serviceIdTarget));
        if (!this.isFinishedAction(serviceIdTarget)) {
            fail("Creating dump for target database failed");
        }
    }

    protected void restoreSourceDatabaseDump(DatabaseType databaseType) throws ServiceBrokerException, ServiceInstanceExistsException, ServiceBrokerAsyncRequiredException, ServiceInstanceUpdateNotSupportedException, ServiceInstanceDoesNotExistException {
        this.dbDumperServiceInstanceService.updateServiceInstance(this.requestForge.createRestoreRequest(this.getDbParamsForRestore(databaseType), serviceIdSource));
        if (!this.isFinishedAction(serviceIdSource)) {
            fail("Restoring dump failed");
        }
    }

    protected void loadServiceIds(DatabaseType databaseType) {
        serviceIdSource = databaseType.toString() + "-service-source";
        serviceIdTarget = databaseType.toString() + "-service-target";
    }

    protected void loadBindingId() {
        bindingId = BINDING_ID;
    }

    protected void dumpAndRestoreTest(DatabaseType databaseType) throws ServiceBrokerException, InterruptedException, ServiceBrokerAsyncRequiredException, IOException, DatabaseExtractionException, CannotFindDatabaseDumperException, ServiceKeyException, ServiceInstanceExistsException, ServiceInstanceUpdateNotSupportedException, ServiceInstanceDoesNotExistException {
        this.loadServiceIds(databaseType);
        this.currentDatabaseType = databaseType;
        this.doBeforeTest(databaseType);

        this.loadBeforeAction();
        long currentTime = System.currentTimeMillis();
        createSourceDatabaseDump(databaseType);
        this.reportIntegration.setDumpDatabaseSourceTime((System.currentTimeMillis() - currentTime) / 1000);
        logger.info("Dump database source finished after {}", humanize.Humanize.duration(this.reportIntegration.getDumpDatabaseSourceTime()));

        this.loadBeforeAction();
        currentTime = System.currentTimeMillis();
        this.restoreSourceDatabaseDump(databaseType);
        this.reportIntegration.setRestoreDatabaseSourceToTargetTime((System.currentTimeMillis() - currentTime) / 1000);
        logger.info("Restore database source to database target finished after {}", humanize.Humanize.duration(this.reportIntegration.getRestoreDatabaseSourceToTargetTime()));


        this.loadBeforeAction();
        currentTime = System.currentTimeMillis();
        this.createTargetDatabaseDump(databaseType);
        this.reportIntegration.setDumpDatabaseTargetTime((System.currentTimeMillis() - currentTime) / 1000);
        logger.info("Dump database target finished after {}", humanize.Humanize.duration(this.reportIntegration.getDumpDatabaseTargetTime()));
        this.diffSourceAndTargetDatabase(databaseType);
        this.reportIntegration.setFailed(false);
    }

    protected void loadBeforeAction() {

    }

    public boolean isBinariesValid(DatabaseType databaseType) {
        if (!this.checkBinariesValid) {
            return true;
        }

        String OS = System.getProperty("os.name").toLowerCase();
        String arch = System.getProperty("os.arch");
        for (File fileToCheck : this.databaseAccessMap.get(databaseType).getBinaries()) {
            if (fileToCheck.getAbsolutePath().contains(binariesFolder.getAbsolutePath()) && (!OS.contains("nux") || !arch.contains("64"))) {
                return false;
            }
        }
        return true;
    }

    @Test
    public void when_binding_to_a_db_dumper_i_should_have_correct_information_about_my_dumps() throws InterruptedException, CannotFindDatabaseDumperException, DatabaseExtractionException, IOException, ServiceBrokerException, ServiceInstanceExistsException, ServiceBrokerAsyncRequiredException, ServiceInstanceDoesNotExistException, ServiceInstanceUpdateNotSupportedException, ServiceInstanceBindingExistsException {
        this.reportIntegration = ReportManager.createReportIntegration(prefixReportName + " - when binding to a db dumper i should have correct information about my dumps");
        DatabaseType databaseType = DatabaseType.MYSQL;
        Metadata metadata = new Metadata();
        metadata.setTags(Arrays.asList("mytag"));

        this.requestForge.setMetadata(metadata);

        this.loadServiceIds(databaseType);
        this.loadBindingId();

        this.currentDatabaseType = databaseType;
        this.doBeforeTest(databaseType);

        this.loadBeforeAction();
        createSourceDatabaseDump(databaseType);

        this.requestForge.setMetadata(null);
        this.loadBeforeAction();
        createSourceDatabaseDumpFromExistingService(databaseType);

        ServiceInstanceBinding serviceInstanceBinding = getBinding(Maps.newHashMap());
        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isNotNull();
        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isInstanceOfAny(List.class);

        List<Map<String, Object>> credentials = (List<Map<String, Object>>) serviceInstanceBinding.getCredentials().get("dumps");
        assertThat(credentials).hasSize(2);

        deleteBinding();

        Map<String, Object> parameters = Maps.newHashMap();
        parameters.put(DbDumperServiceInstanceBindingService.FIND_BY_TAGS_KEY, Arrays.asList("mytag"));

        serviceInstanceBinding = getBinding(parameters);

        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isNotNull();
        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isInstanceOfAny(List.class);

        credentials = (List<Map<String, Object>>) serviceInstanceBinding.getCredentials().get("dumps");
        assertThat(credentials).hasSize(1);

        deleteBinding();

        parameters = Maps.newHashMap();
        parameters.put(DbDumperServiceInstanceBindingService.SEE_ALL_DUMPS_KEY, true);

        serviceInstanceBinding = getBinding(parameters);

        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isNotNull();
        assertThat(serviceInstanceBinding.getCredentials().get("dumps")).isInstanceOfAny(List.class);

        credentials = (List<Map<String, Object>>) serviceInstanceBinding.getCredentials().get("dumps");
        assertThat(credentials).hasSize(2);

        deleteBinding();

        this.reportIntegration.setFailed(false);
    }

    @Test
    public void when_dump_and_restore_a_MYSQL_database_it_should_have_the_database_source_equals_to_the_database_target() throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException, ServiceInstanceUpdateNotSupportedException, ServiceBrokerAsyncRequiredException, ServiceBrokerException, ServiceInstanceDoesNotExistException, ServiceKeyException, ServiceInstanceExistsException {
        this.reportIntegration = ReportManager.createReportIntegration(prefixReportName + "MySQL");
        DatabaseType databaseType = DatabaseType.MYSQL;
        this.dumpAndRestoreTest(databaseType);
    }

    @Test
    public void when_dump_and_restore_a_POSTGRES_database_it_should_have_the_database_source_equals_to_the_database_target() throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException, ServiceBrokerException, ServiceInstanceExistsException, ServiceBrokerAsyncRequiredException, ServiceInstanceUpdateNotSupportedException, ServiceInstanceDoesNotExistException, ServiceKeyException {
        this.reportIntegration = ReportManager.createReportIntegration(prefixReportName + "PostgreSQL");
        DatabaseType databaseType = DatabaseType.POSTGRESQL;
        this.dumpAndRestoreTest(databaseType);
    }

    @Test
    public void when_dump_and_restore_a_REDIS_database_it_should_have_the_database_source_equals_to_the_database_target() throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException, ServiceInstanceUpdateNotSupportedException, ServiceBrokerAsyncRequiredException, ServiceBrokerException, ServiceInstanceDoesNotExistException, ServiceKeyException, ServiceInstanceExistsException {
        this.reportIntegration = ReportManager.createReportIntegration(prefixReportName + "Redis");
        DatabaseType databaseType = DatabaseType.REDIS;
        this.dumpAndRestoreTest(databaseType);
    }

    @Test
    public void when_dump_and_restore_a_MONGODB_database_it_should_have_the_database_source_equals_to_the_database_target() throws DatabaseExtractionException, CannotFindDatabaseDumperException, InterruptedException, IOException, ServiceInstanceUpdateNotSupportedException, ServiceBrokerAsyncRequiredException, ServiceBrokerException, ServiceInstanceDoesNotExistException, ServiceKeyException, ServiceInstanceExistsException {
        this.reportIntegration = ReportManager.createReportIntegration(prefixReportName + "MongoDB");
        DatabaseType databaseType = DatabaseType.MONGODB;
        this.dumpAndRestoreTest(databaseType);
    }

    protected InputStream getDatabaseStream(String serviceId) throws DatabaseExtractionException, ServiceKeyException, IOException {
        this.loadBeforeAction();
        DbDumperServiceInstance dbDumperServiceInstance = this.serviceInstanceRepo.findOne(serviceId);
        DatabaseRef databaseRef = dbDumperServiceInstance.getDatabaseRef();

        assertThat(dbDumperServiceInstance.getDatabaseDumpFiles().size() > 0)
                .overridingErrorMessage(String.format("Database '%s' should have least one dump file.", databaseRef.getName()))
                .isTrue();
        assertThat(dbDumperServiceInstance.getDatabaseDumpFiles().get(0)).isNotNull();
        String fileSource = databaseRef.getName() + "/" + dbDumperServiceInstance.getDatabaseDumpFiles().get(dbDumperServiceInstance.getDatabaseDumpFiles().size() - 1).getFileName();

        this.loadBeforeAction();
        this.databaseRefManager.deleteServiceKey(databaseRef);
        return this.filer.retrieveWithStream(fileSource);
    }

    protected InputStream getSourceStream(DatabaseType databaseType) throws DatabaseExtractionException, ServiceKeyException, IOException {
        return this.getDatabaseStream(serviceIdSource);
    }

    protected InputStream getTargetStream(DatabaseType databaseType) throws DatabaseExtractionException, ServiceKeyException, IOException {
        return this.getDatabaseStream(serviceIdTarget);
    }

    public void diffSourceAndTargetDatabase(DatabaseType databaseType) throws DatabaseExtractionException, ServiceKeyException, IOException {
        long currentTime = System.currentTimeMillis();
        InputStream sourceStream = this.getSourceStream(databaseType);
        InputStream targetStream = this.getTargetStream(databaseType);


        int sourceRead = 0;
        int targetRead = 0;
        Long sourceNumberBytesRead = 0L;
        Long targetNumberBytesRead = 0L;
        boolean shouldContinue = true;
        // we create chunk of data to avoid memory heap space
        while (shouldContinue) {
            byte[] sourceBytes = new byte[this.chunkSizeDiff];
            byte[] targetBytes = new byte[this.chunkSizeDiff];
            sourceRead = ByteStreams.read(sourceStream, sourceBytes, 0, sourceBytes.length);
            sourceNumberBytesRead += sourceRead;
            if (sourceRead != sourceBytes.length) {
                shouldContinue = false;
                sourceBytes = Arrays.copyOf(sourceBytes, sourceRead);
                if (sourceBytes.length == 0) {
                    break;
                }
            }
            targetRead = ByteStreams.read(targetStream, targetBytes, 0, targetBytes.length);
            targetNumberBytesRead += targetRead;
            if (targetRead != targetBytes.length) {
                shouldContinue = false;
                targetBytes = Arrays.copyOf(targetBytes, targetRead);
                if (targetBytes.length == 0) {
                    break;
                }
            }
            if (databaseType.equals(DatabaseType.REDIS)) { // Redis rearrange data which make diff files unreliable
                continue;
            }
            if (dbDumpersFactory.getDatabaseDumper(databaseType).isDumpShowable()) {
                assertThat(new String(targetBytes)).isEqualTo(new String(sourceBytes));
            } else {
                assertThat(Arrays.equals(targetBytes, sourceBytes))
                        .overridingErrorMessage(String.format("Dumps files between database source and database target diverged."))
                        .isTrue();
            }
        }
        assertThat(sourceNumberBytesRead).isEqualTo(targetNumberBytesRead);
        this.reportIntegration.setDiffTime((System.currentTimeMillis() - currentTime) / 1000);
        this.logger.info("Diff against source and target database finished after {}", humanize.Humanize.duration(this.reportIntegration.getDiffTime()));
    }

    protected boolean isServerListening(DatabaseType databaseType) throws DatabaseExtractionException {
        DatabaseRef databaseServer = this.databaseAccessMap.get(databaseType).generateDatabaseRef();
        if (databaseServer == null) {
            return false;
        }
        return this.isSocketOpen(databaseServer.getHost(), databaseServer.getPort());
    }

    protected boolean isSocketOpen(String host, Integer port) {
        Socket s = null;
        try {
            s = new Socket(host, port);
            return true;
        } catch (Exception e) {
            return false;
        } finally {
            if (s != null)
                try {
                    s.close();
                } catch (Exception e) {
                }
        }
    }

    public void populateData(DatabaseType databaseType) throws DatabaseExtractionException, CannotFindDatabaseDumperException, IOException, InterruptedException {
        File fakeData = this.databaseAccessMap.get(databaseType).getFakeDataFile();
        if (fakeData == null) {
            fail("Cannot find file for database: " + databaseType);
            return;
        }
        DatabaseRef databaseServer = this.databaseAccessMap.get(databaseType).generateDatabaseRef();
        if (databaseServer == null) {
            fail("Cannot find server for database: " + databaseType);
            return;
        }

        this.createDatabase(databaseType);
        databaseServer.setDatabaseName(DATABASE_SOURCE_NAME);
        this.populateDataToDatabaseRefFromFile(fakeData, databaseServer);
    }

    public void populateDataToDatabaseRefFromFile(File fakeData, DatabaseRef databaseServer) throws CannotFindDatabaseDumperException, IOException, InterruptedException {
        this.reportIntegration.setFakeDataFileSize(fakeData.length());
        long currentTime = System.currentTimeMillis();
        logger.info("Populating fake data on server: {} - database {} will be created with data from file {} which has size of {}", databaseServer.getHost(), DATABASE_SOURCE_NAME, fakeData.getAbsolutePath(), humanize.Humanize.binaryPrefix(fakeData.length()));
        DatabaseDriver databaseDriver = dbDumpersFactory.getDatabaseDumper(databaseServer);
        String[] restoreCommandLine = databaseDriver.getRestoreCommandLine();
        int i = 1;
        while (true) {
            Process process = this.runCommandLine(restoreCommandLine);
            OutputStream outputStream = process.getOutputStream();
            InputStream dumpFileInputStream = Files.asByteSource(fakeData).openStream();
            try {
                ByteStreams.copy(dumpFileInputStream, outputStream);
                outputStream.flush();
                outputStream.close();
            } catch (IOException e) {

            } finally {
                dumpFileInputStream.close();
            }
            process.waitFor();
            if (process.exitValue() == 0) {
                break;
            }
            dumpFileInputStream.close();
            outputStream.close();
            if (i >= populateDataRetry) {
                throw this.generateInterruptedExceptionFromProcess(process);
            }
            logger.warn("Retry {}/{}: fail to populate data.", i, populateDataRetry);
            i++;
        }
        this.reportIntegration.setPopulateToDatabaseTime((System.currentTimeMillis() - currentTime) / 1000);
        logger.info("Finished to populate fake data on server: {} \n Duration: {}", databaseServer.getHost(), humanize.Humanize.duration(this.reportIntegration.getPopulateToDatabaseTime()));
    }

    protected void dropDatabase(DatabaseType databaseType) throws IOException, InterruptedException {
        if (System.getenv("TRAVIS") != null) { //we don't drop databases if we are in travis
            return;
        }

        List<String[]> dropDatabaseCommands = this.databaseAccessMap.get(databaseType).getDropDatabaseCommands();

        if (dropDatabaseCommands.size() == 0) {
            return;
        }
        this.runCommands(dropDatabaseCommands);
    }

    protected void createDatabase(DatabaseType databaseType) throws IOException, InterruptedException {
        if (System.getenv("TRAVIS") != null) { // if in travis we don't try to create database cause they already exists
            return;
        }
        List<String[]> createDatabaseCommands = this.databaseAccessMap.get(databaseType).getCreateDatabaseCommands();

        if (databaseType.equals(DatabaseType.POSTGRESQL)) {
            this.dropDatabase(databaseType);
        }
        if (createDatabaseCommands.size() == 0) {
            return;
        }
        this.runCommands(createDatabaseCommands);

    }

    protected void runCommands(List<String[]> commands) throws IOException, InterruptedException {
        for (String[] command : commands) {
            this.runCommand(command);
        }
    }

    protected InterruptedException generateInterruptedExceptionFromProcess(Process process) throws IOException {
        return new InterruptedException("\nError during process (exit code is " + process.exitValue() + "): \n"
                + this.getInputStreamToStringFromProcess(process.getErrorStream())
                + "\n" + this.getInputStreamToStringFromProcess(process.getInputStream())
        );
    }

    protected void runCommand(String[] command) throws IOException, InterruptedException {
        this.runCommand(command, false);
    }

    protected void runCommand(String[] command, boolean showStdout) throws IOException, InterruptedException {
        Process process = null;
        if (showStdout) {
            process = this.runCommandLineWithStdoutShowed(command);
        } else {
            process = this.runCommandLine(command);
        }
        process.waitFor();
        if (process.exitValue() != 0) {
            throw this.generateInterruptedExceptionFromProcess(process);
        }
    }

    private String getInputStreamToStringFromProcess(InputStream inputStream) throws IOException {
        String outputFromProcess = "";
        String line = "";
        if (inputStream == null) {
            return outputFromProcess;
        }
        BufferedReader brOutput = new BufferedReader(new InputStreamReader(inputStream));
        while ((line = brOutput.readLine()) != null) {
            outputFromProcess += line + "\n";
        }
        return outputFromProcess;
    }

    protected Process runCommandLine(String[] commandLine) throws IOException, InterruptedException {
        logger.info("Running command line: " + String.join(" ", commandLine));

        ProcessBuilder pb = new ProcessBuilder(commandLine);
        pb.redirectOutput(ProcessBuilder.Redirect.PIPE);
        Process process = pb.start();
        return process;
    }

    protected Process runCommandLineWithStdoutShowed(String[] commandLine) throws IOException, InterruptedException {
        logger.info("Running command line: " + String.join(" ", commandLine));

        ProcessBuilder pb = new ProcessBuilder(commandLine);
        pb.inheritIO();
        Process process = pb.start();
        return process;
    }

    public boolean isFinishedAction(String serviceInstanceId) {

        ExecutorService executor = Executors.newCachedThreadPool();
        Callable<Boolean> task = () -> {
            while (true) {
                ServiceInstance serviceInstance = dbDumperServiceInstanceService.getServiceInstance(serviceInstanceId);
                ServiceInstanceLastOperation lastOperation = serviceInstance.getServiceInstanceLastOperation();
                switch (lastOperation.getState()) {
                    case "succeeded":
                        return true;
                    case "in progress":
                        break;
                    case "failed":
                    case "internal error":
                        return false;
                }
                Thread.sleep(5000L);// we yield the task for 5seconds to let the service do is work (actually, Cloud Controller hit getServiceInstance every 30sec)
            }
        };
        Future<Boolean> future = executor.submit(task);
        try {
            Boolean result = future.get(timeoutAction, TimeUnit.MINUTES);
            return result;
        } catch (Exception ex) {
            future.cancel(true);
            fail("Timeout reached.", ex);
        }
        return false;
    }

    public String getDbFromUri(String databaseServerUri, String databaseName) {
        int lastPos = databaseServerUri.lastIndexOf('/');
        return databaseServerUri.substring(0, lastPos) + "/" + databaseName;
    }

    public String getDbSourceFromUri(String databaseServerUri) {
        return this.getDbFromUri(databaseServerUri, DATABASE_SOURCE_NAME);
    }

    public String getDbTargetFromUri(String databaseServerUri) {
        return this.getDbFromUri(databaseServerUri, DATABASE_TARGET_NAME);
    }

    protected void populateDatabaseAccessMap() throws DatabaseExtractionException {
        DatabaseAccess postgresDatabaseAccess = new DatabaseAccess(
                postgresServer,
                Arrays.asList(dbDumpersFactory.getPostgresBinaryDump(), dbDumpersFactory.getPostgresBinaryRestore()),
                postgresFakeData,
                this.getDbSourceFromUri(postgresServer),
                this.getDbTargetFromUri(postgresServer)
        );
        DatabaseRef databaseServerPostgres = postgresDatabaseAccess.generateDatabaseRef();
        postgresDatabaseAccess.addCreateDatabaseCommand(new String[]{
                this.psqlBinary.getAbsolutePath(),
                String.format("--dbname=postgresql://%s:%s@%s:%s/%s",
                        databaseServerPostgres.getUser(),
                        databaseServerPostgres.getPassword(),
                        databaseServerPostgres.getHost(),
                        databaseServerPostgres.getPort(),
                        databaseServerPostgres.getDatabaseName()),
                "-c",
                "CREATE DATABASE " + DATABASE_SOURCE_NAME
        });
        postgresDatabaseAccess.addCreateDatabaseCommand(new String[]{
                this.psqlBinary.getAbsolutePath(),
                String.format("--dbname=postgresql://%s:%s@%s:%s/%s",
                        databaseServerPostgres.getUser(),
                        databaseServerPostgres.getPassword(),
                        databaseServerPostgres.getHost(),
                        databaseServerPostgres.getPort(),
                        databaseServerPostgres.getDatabaseName()),
                "-c",
                "CREATE DATABASE " + DATABASE_TARGET_NAME
        });
        postgresDatabaseAccess.addDropDatabaseCommands(new String[]{
                this.psqlBinary.getAbsolutePath(),
                String.format("--dbname=postgresql://%s:%s@%s:%s/%s",
                        databaseServerPostgres.getUser(),
                        databaseServerPostgres.getPassword(),
                        databaseServerPostgres.getHost(),
                        databaseServerPostgres.getPort(),
                        databaseServerPostgres.getDatabaseName()),
                "-c",
                "DROP DATABASE IF EXISTS " + DATABASE_SOURCE_NAME
        });
        postgresDatabaseAccess.addDropDatabaseCommands(new String[]{
                this.psqlBinary.getAbsolutePath(),
                String.format("--dbname=postgresql://%s:%s@%s:%s/%s",
                        databaseServerPostgres.getUser(),
                        databaseServerPostgres.getPassword(),
                        databaseServerPostgres.getHost(),
                        databaseServerPostgres.getPort(),
                        databaseServerPostgres.getDatabaseName()),
                "-c",
                "DROP DATABASE IF EXISTS " + DATABASE_TARGET_NAME
        });
        DatabaseAccess mysqlDatabaseAccess = new DatabaseAccess(
                mysqlServer,
                Arrays.asList(dbDumpersFactory.getMysqlBinaryDump(), dbDumpersFactory.getMysqlBinaryRestore()),
                mysqlFakeData,
                this.getDbSourceFromUri(mysqlServer),
                this.getDbTargetFromUri(mysqlServer)
        );

        DatabaseRef databaseServerMysql = mysqlDatabaseAccess.generateDatabaseRef();
        mysqlDatabaseAccess.addCreateDatabaseCommand(new String[]{
                this.mysqlBinary.getAbsolutePath(),
                "--host=" + databaseServerMysql.getHost(),
                "--port=" + databaseServerMysql.getPort(),
                "--user=" + databaseServerMysql.getUser(),
                "--password=" + databaseServerMysql.getPassword(),
                "-e",
                "CREATE DATABASE IF NOT EXISTS " + DATABASE_SOURCE_NAME
        });
        mysqlDatabaseAccess.addCreateDatabaseCommand(new String[]{
                this.mysqlBinary.getAbsolutePath(),
                "--host=" + databaseServerMysql.getHost(),
                "--port=" + databaseServerMysql.getPort(),
                "--user=" + databaseServerMysql.getUser(),
                "--password=" + databaseServerMysql.getPassword(),
                "-e",
                "CREATE DATABASE IF NOT EXISTS " + DATABASE_TARGET_NAME
        });
        mysqlDatabaseAccess.addDropDatabaseCommands(new String[]{
                this.mysqlBinary.getAbsolutePath(),
                "--host=" + databaseServerMysql.getHost(),
                "--port=" + databaseServerMysql.getPort(),
                "--user=" + databaseServerMysql.getUser(),
                "--password=" + databaseServerMysql.getPassword(),
                "-e",
                "DROP DATABASE IF EXISTS " + DATABASE_SOURCE_NAME
        });
        mysqlDatabaseAccess.addDropDatabaseCommands(new String[]{
                this.mysqlBinary.getAbsolutePath(),
                "--host=" + databaseServerMysql.getHost(),
                "--port=" + databaseServerMysql.getPort(),
                "--user=" + databaseServerMysql.getUser(),
                "--password=" + databaseServerMysql.getPassword(),
                "-e",
                "DROP DATABASE IF EXISTS " + DATABASE_TARGET_NAME
        });

        this.databaseAccessMap.put(
                DatabaseType.MONGODB,
                new DatabaseAccess(
                        mongoServer,
                        Arrays.asList(dbDumpersFactory.getMongodbBinaryDump(), dbDumpersFactory.getMongodbBinaryRestore()),
                        mongodbFakeData,
                        this.getDbSourceFromUri(mongoServer),
                        this.getDbTargetFromUri(mongoServer)
                )
        );

        this.databaseAccessMap.put(DatabaseType.POSTGRESQL, postgresDatabaseAccess);

        this.databaseAccessMap.put(
                DatabaseType.REDIS,
                new DatabaseAccess(
                        redisServer,
                        Arrays.asList(dbDumpersFactory.getRedisRutilBinary()),
                        redisFakeData,
                        redisServer,
                        redisServer
                )
        );

        this.databaseAccessMap.put(DatabaseType.MYSQL, mysqlDatabaseAccess);
    }
}
