package org.apache.pig.test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.pig.ExecType;
import org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType;

public class SparkMiniCluster extends MiniGenericCluster {
    private static final File CONF_DIR = new File("build/classes");
    private static final File CONF_FILE = new File(CONF_DIR, "hadoop-site.xml");
    private ExecType spark = new SparkExecType();
    SparkMiniCluster() {

    }

    @Override
    public ExecType getExecType() {
        return spark;
    }

    @Override
    protected void setupMiniDfsAndMrClusters() {
        try {
            CONF_DIR.mkdirs();
            if (CONF_FILE.exists()) {
                CONF_FILE.delete();
            }
            m_conf = new Configuration();
            m_conf.set("io.sort.mb", "1");
            m_conf.writeXml(new FileOutputStream(CONF_FILE));
            int dataNodes = 4;
            m_dfs = new MiniDFSCluster(m_conf, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
        } catch (IOException e) {
            throw new RuntimeException(e);

        }

    }

    @Override
    protected void shutdownMiniMrClusters() {
        if (CONF_FILE.exists()) {
            CONF_FILE.delete();
        }
    }

}

package org.apache.pig.test;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.pig.ExecType;
import org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType;

public class SparkMiniCluster extends MiniGenericCluster {
    private static final File CONF_DIR = new File("build/classes");
    private static final File CONF_FILE = new File(CONF_DIR, "hadoop-site.xml");
    private ExecType spark = new SparkExecType();
    SparkMiniCluster() {

    }

    @Override
    public ExecType getExecType() {
        return spark;
    }

    @Override
    protected void setupMiniDfsAndMrClusters() {
        try {
            CONF_DIR.mkdirs();
            if (CONF_FILE.exists()) {
                CONF_FILE.delete();
            }
            m_conf = new Configuration();
            m_conf.set("io.sort.mb", "1");
            m_conf.writeXml(new FileOutputStream(CONF_FILE));
            int dataNodes = 4;
            m_dfs = new MiniDFSCluster(m_conf, dataNodes, true, null);
            m_fileSys = m_dfs.getFileSystem();
            m_fileSys.mkdirs(m_fileSys.getWorkingDirectory());
        } catch (IOException e) {
            throw new RuntimeException(e);

        }

    }

    @Override
    protected void shutdownMiniMrClusters() {
        if (CONF_FILE.exists()) {
            CONF_FILE.delete();
        }
    }

}

Enable unit test "TestStreamingUDF" in spark (liyunzhang via praveen)
PIG-4268: Enable unit test "TestStreamingUDF" in spark (liyunzhang via praveen)

git-svn-id: d317905e1b1233abb7022f5914f79c3119e04b87@1645891 13f79535-47bb-0310-9956-ffa450edef68
