/*******************************************************************************
 * Copyright (c) 2008 Actuate Corporation.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *  Actuate Corporation  - initial API and implementation
 *******************************************************************************/
package org.eclipse.birt.report.engine.dataextraction.csv;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;

import javax.sql.rowset.serial.SerialBlob;
import javax.sql.rowset.serial.SerialException;

import org.eclipse.birt.core.exception.BirtException;
import org.eclipse.birt.report.engine.api.DataExtractionOption;
import org.eclipse.birt.report.engine.api.IDataExtractionOption;
import org.eclipse.birt.report.engine.api.IDataIterator;
import org.eclipse.birt.report.engine.api.IExtractionResults;
import org.eclipse.birt.report.engine.api.IResultMetaData;
import org.eclipse.birt.report.engine.dataextraction.CSVDataExtractionOption;
import org.eclipse.birt.report.engine.dataextraction.ICSVDataExtractionOption;
import org.eclipse.birt.report.engine.dataextraction.csv.mock.MockExtractionResults;

import junit.framework.TestCase;
import org.eclipse.birt.core.data.DataType;

public class CSVDataExtractionImplTest extends TestCase
{
	/**
	 * Test locale used in the test files.
	 */
	private static final String TEST_LOCALE_COUNTRY = "US"; //$NON-NLS-1$
	/**
	 * Test locale used in the test files.
	 */
	private static final String TEST_LOCALE_LANGUAGE = "en"; //$NON-NLS-1$
	/**
	 * Test time zone used in the test files.
	 */
	private static final String TEST_TIME_ZONE = "GMT+1"; //$NON-NLS-1$

	/**
	 * Folder containing the test*.csv files.
	 */
	public static final String ROOT_FOLDER = "test/" //$NON-NLS-1$
			+ CSVDataExtractionImplTest.class.getPackage( ).getName( )
				.replace('.', '/' ) + "/"; //$NON-NLS-1$

	/**
	 * Date format used for the date format test.
	 */
	private static final String TEST_DATE_FORMAT = "dd/MM/yyyy HH:mm:ss"; //$NON-NLS-1$
	private static final String TEST_DATE_FORMAT_WITH_TIMEZONE = "dd/MM/yyyy HH:mm:ss z"; //$NON-NLS-1$
	
	private static final String ENCODING_ISO = "ISO-8859-1"; //$NON-NLS-1$

	private static final String[] TEST_DATA_COLUMNS = new String[] {
		"stringColumn", //$NON-NLS-1$
		"integerColumn", //$NON-NLS-1$
		"dateColumn", //$NON-NLS-1$
		"decimalColumn" //$NON-NLS-1$
		};
	private static final String[] TEST_SELECT_COLUMNS = new String[] {
		"dateColumn", //$NON-NLS-1$
		"stringColumn", //$NON-NLS-1$
		"integerColumn" //$NON-NLS-1$
	};
	
	private static final String[] TEST_INVALID_COLUMNS = new String[] {
		"stringColumn", //$NON-NLS-1$
		"invalidColumn", //$NON-NLS-1$
		"integerColumn"	//$NON-NLS-1$
	};
	
	private static final int[] TEST_DATA_TYPES = new int[] {
		DataType.STRING_TYPE,
		DataType.INTEGER_TYPE,
		DataType.DATE_TYPE,
		DataType.DECIMAL_TYPE
	}; 
	
	// is be initialized in setUp()
	private Object[][] TEST_DATA = null;
	
	private static final String[] TEST_DATA_COLUMNS_QUOTING = new String[] {
		"Column 1", //$NON-NLS-1$
		"Column 2" //$NON-NLS-1$
	};	
	
	private static final int[] TEST_DATA_TYPES_QUOTING = new int[] {
		DataType.STRING_TYPE,
		DataType.STRING_TYPE,
	};	
 
	private static final Object[][] TEST_DATA_QUOTING = new Object[][] {
		// newlines
		new Object[] {
				"A string\nwith\nnewlines", //$NON-NLS-1$
				"A second\r\nstring with\r\nnewlines" //$NON-NLS-1$
		},
		new Object[] {
				" Space at the beginning", //$NON-NLS-1$
				"Space at the end " //$NON-NLS-1$
		},
		new Object[] {
				"\tTab at the beginning", //$NON-NLS-1$
				"Tab at the end\t" //$NON-NLS-1$
		},
		new Object[] {
				"\tTab at both sides\t", //$NON-NLS-1$
				"Tab\tin\tthe\tmiddle" //$NON-NLS-1$
		},
		new Object[] {
				" Space at both sides ", //$NON-NLS-1$
				"String,containing,separator,char" //$NON-NLS-1$
		},
		new Object[] {
				"String \"with\" double-quotes", //$NON-NLS-1$
				"String 'with' single quotes" //$NON-NLS-1$
		},
		new Object[] {
				"String|with|pipes", //$NON-NLS-1$
				"String;with;semicolons" //$NON-NLS-1$
		}
	};

	private ByteArrayOutputStream out;
	private CSVDataExtractionOption option;
	private IExtractionResults results;
	private DateFormat inputDateFormat;
	
	public void setUp()
	{		
		Locale.setDefault( new Locale( TEST_LOCALE_LANGUAGE, TEST_LOCALE_COUNTRY ) );
		TimeZone.setDefault( TimeZone.getTimeZone( TEST_TIME_ZONE ) );
		inputDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); //$NON-NLS-1$
		out = new ByteArrayOutputStream();
		option = createOptions( );
		
		TEST_DATA = new Object[][] {
			new Object[] {
					"myString1", //$NON-NLS-1$
					Integer.valueOf(5),
					makeDate( "2008-08-08 10:30:00" ), //$NON-NLS-1$
					Double.valueOf( 25.689 ) },
			new Object[] {
					null,
					Integer.valueOf(-12),
					null,
					null },
			new Object[] {
					"my,str;in\tg|3", //$NON-NLS-1$
					null,
					makeDate( "2007-01-01 15:30:00" ), //$NON-NLS-1$
					Double.valueOf( -987.654321 )
					},
			// case for testing encoding
			new Object[] {
					"\u00fc\u4f60\u00df\u00e9", //$NON-NLS-1$
					Integer.valueOf(0),
					null,
					Double.valueOf( 0.0 )
					}
		};
		
		results = new MockExtractionResults(
				TEST_DATA_COLUMNS,
				TEST_DATA_TYPES,
				TEST_DATA
		);		
	}
	
	public void tearDown()
	{
		try
		{
			out.close();
		}
		catch ( IOException e )
		{
			e.printStackTrace();
		}
		out = null;
		option = null;
	}

	public void testOutputDefaults( ) throws Exception
	{
		IDataExtractionOption option = new DataExtractionOption();
		option.setOutputFile( "test.csv" ); //$NON-NLS-1$
		option.setOutputFormat( "csv" ); //$NON-NLS-1$		
		subtestRegular( option, "testDefaults.csv"); //$NON-NLS-1$
	}
	
	public void testOutputRegular( ) throws Exception
	{
		option.setLocale( Locale.FRANCE ); // must be ignored because of locale neutral
		option.setLocaleNeutralFormat( true );
		subtestRegular( option, "testRegular.csv"); //$NON-NLS-1$
	}

	public void testOutputLocalized( ) throws Exception
	{
		option.setLocale( Locale.FRANCE );
		option.setLocaleNeutralFormat( false );
		subtestRegular( option, "testLocalized.csv"); //$NON-NLS-1$
	}

	public void testOutputEncoding( ) throws Exception
	{
		option.setLocale( Locale.ENGLISH );
		option.setLocaleNeutralFormat( true );
		option.setEncoding( ENCODING_ISO );
		CSVDataExtractionImpl extract = createExtraction( out, option );
		
		// replace test value with chars available in ISO encoding
		Object[][] data = new Object[][] {
				new Object[] {
					"\u00e9\u00fc", //$NON-NLS-1$
					Integer.valueOf(0),
					null,
					Double.valueOf( 0.0 )
				}
			};
		
		IExtractionResults results = new MockExtractionResults(
				TEST_DATA_COLUMNS,
				TEST_DATA_TYPES,
				data
		); 
		extract.output( results );
		
		String testFile = ROOT_FOLDER + "testEncoding.csv"; //$NON-NLS-1$
		assertFileContent(testFile, out.toByteArray( ));
	}
	
	public void testOutputDateFormatWithLocaleNeutral( ) throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setLocaleNeutralFormat( true );
		option.setTimeZone( TimeZone.getTimeZone( "Asia/Shanghai" ) ); //$NON-NLS-1$ // must be ignored because of locale neutral
		option.setDateFormat( TEST_DATE_FORMAT ); // must be ignored because of locale neutral
		subtestDateFormat("testLocaleNeutralDateFormat.csv"); //$NON-NLS-1$
	}

	public void testOutputDateFormatWithLocale( ) throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setLocaleNeutralFormat( false );
		option.setDateFormat( TEST_DATE_FORMAT );
		
		// replace test value with chars available in ISO encoding
		
		subtestDateFormat( "testDateFormat.csv" ); //$NON-NLS-1$
	}
	
	public void testOutputDefaultDateFormatWithLocale( ) throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setLocaleNeutralFormat( false );
		option.setDateFormat( null ); // use default format
		subtestDateFormat( "testDefaultDateFormat.csv" ); //$NON-NLS-1$
	}

	public void testOutputWithTimeZone( ) throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setTimeZone( TimeZone.getTimeZone( "Asia/Shanghai" ) ); //$NON-NLS-1$
		option.setLocaleNeutralFormat( false );
		option.setDateFormat( null );
		
		// replace test value with chars available in ISO encoding
		
		subtestDateFormat( "testTimeZone.csv" ); //$NON-NLS-1$
	}

	public void testOutputDateFormatWithTimeZone( ) throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setTimeZone( TimeZone.getTimeZone( "Asia/Shanghai" ) ); //$NON-NLS-1$
		option.setLocaleNeutralFormat( false );
		option.setDateFormat( TEST_DATE_FORMAT_WITH_TIMEZONE );
		
		// replace test value with chars available in ISO encoding
		
		subtestDateFormat( "testDateFormatWithTimeZone.csv" ); //$NON-NLS-1$
	}
	
	/**
	 * @param fileName
	 * @throws BirtException
	 */
	private void subtestDateFormat( String fileName ) throws BirtException
	{
		// replace test value with chars available in ISO encoding		
		String[] dataColumns = new String[] {
			"Any Type Column using DateTime", //$NON-NLS-1$
			"Any Type Column using Date", //$NON-NLS-1$
			"Any Type Column using Time", //$NON-NLS-1$
			"DateTime Column", //$NON-NLS-1$
			"SQL Date Column", //$NON-NLS-1$
			"SQL Time Column" //$NON-NLS-1$
		};
		
		int[] dataTypes = new int[] {
			DataType.ANY_TYPE,
			DataType.ANY_TYPE,
			DataType.ANY_TYPE,
			DataType.DATE_TYPE,
			DataType.SQL_DATE_TYPE,
			DataType.SQL_TIME_TYPE
		};
		
		Object[][] data = new Object[][] {
				new Object[] {
						makeDate("2008-08-08 08:08:08"), //$NON-NLS-1$
						new java.sql.Date( makeDate("2008-08-08 08:08:08").getTime( ) ), //$NON-NLS-1$
						new java.sql.Time( makeDate("2008-08-08 08:08:08").getTime( ) ), //$NON-NLS-1$
						makeDate("2008-08-08 08:08:08"), //$NON-NLS-1$
						new java.sql.Date( makeDate("2008-08-08 08:08:08").getTime( ) ), //$NON-NLS-1$
						new java.sql.Time( makeDate("2008-08-08 08:08:08").getTime( ) ) //$NON-NLS-1$
				}
			};
		
		results = new MockExtractionResults(
				dataColumns,
				dataTypes,
				data
		); 		
		
		subtestRegular( option, fileName);
	}
	
	
	public void testOutputWithSelectedColumns() throws Exception
	{
		option.setSelectedColumns( TEST_SELECT_COLUMNS );
		subtestRegular( option, "testSelectColumns.csv"); //$NON-NLS-1$
	}

	/*
	 * Test will ignore the invalid column and output only valid ones without
	 * exception
	 */
	public void testOutputWithInvalidColumn() throws Exception
	{
		option.setSelectedColumns( TEST_INVALID_COLUMNS );
		subtestRegular( option, "testSelectInvalidColumn.csv"); //$NON-NLS-1$
	}

	public void testOutputQuoting() throws Exception
	{
		CSVDataExtractionImpl extract = createExtraction( out, option );		
		IExtractionResults results = new MockExtractionResults(
				TEST_DATA_COLUMNS_QUOTING,
				TEST_DATA_TYPES_QUOTING,
				TEST_DATA_QUOTING
		); 
		extract.output( results );
		
		String testFile = ROOT_FOLDER + "testQuoting.csv"; //$NON-NLS-1$
		assertFileContent(testFile, out.toByteArray( ));
	}

	public void testOutputDataTypes() throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setLocaleNeutralFormat( true );
		subtestDataTypes( "testDataTypes.csv" ); //$NON-NLS-1$
	}

	public void testOutputDataTypesLocalized() throws Exception
	{
		option.setLocale( Locale.FRENCH );
		option.setLocaleNeutralFormat( false );
		subtestDataTypes( "testDataTypesLocalized.csv" ); //$NON-NLS-1$
	}
	
	/**
	 * @throws SerialException
	 * @throws SQLException
	 * @throws BirtException
	 */
	private void subtestDataTypes( String fileName ) throws SerialException, SQLException,
			BirtException
	{
		boolean isLocaleNeutral = option.isLocaleNeutralFormat( );
		final String[] TEST_DATA_TYPES_COLUMNS = new String[] {
			"Any", //$NON-NLS-1$
			"Boolean", //$NON-NLS-1$
			"Integer", //$NON-NLS-1$
			"Double", //$NON-NLS-1$
			"Decimal", //$NON-NLS-1$
			"String", //$NON-NLS-1$
			"Date", //$NON-NLS-1$
			"Blob", //$NON-NLS-1$
			"Binary", //$NON-NLS-1$
			"SQL Date", //$NON-NLS-1$
			"SQL Time" //$NON-NLS-1$
		};	
		
		final int[] TEST_DATA_TYPES_TYPES = new int[] {
			DataType.ANY_TYPE,
			DataType.BOOLEAN_TYPE,
			DataType.INTEGER_TYPE,
			DataType.DOUBLE_TYPE,
			DataType.DECIMAL_TYPE,
			DataType.STRING_TYPE,
			DataType.DATE_TYPE,
			DataType.BLOB_TYPE,
			DataType.BINARY_TYPE,
			DataType.SQL_DATE_TYPE,
			DataType.SQL_TIME_TYPE
		};
		
		final Object[][] data = new Object[][] {
			new Object[] {
					null,
					null,
					null,
					null,
					null,
					null,
					null,
					null,
					null,
					null,
					null
			},
			new Object[] {
					Boolean.TRUE, // any type
					Boolean.TRUE,
					Integer.valueOf(-24),
					Double.valueOf(-123.456789),
					// need to use different values because localized BigDecimal produce huge values (no scientific format)
					isLocaleNeutral?new BigDecimal("123e456789"):new BigDecimal("12345678901234567890123456789"), //$NON-NLS-1$ //$NON-NLS-2$ 
					"Simple String", //$NON-NLS-1$
					makeDate( "2008-08-08 08:08:08" ), //$NON-NLS-1$
					new SerialBlob( new byte[] {1, 2, 3} ), // blob
					new byte[] {4, 5, 6}, // binary
					new java.sql.Date(makeDate( "2008-08-08 08:08:08" ).getTime( )),// sql date //$NON-NLS-1$
					new java.sql.Time(makeDate( "2008-08-08 08:08:08" ).getTime( ))// sql time //$NON-NLS-1$
			},
			new Object[] {
					"Any String", // any type //$NON-NLS-1$
					Boolean.FALSE,
					Integer.valueOf(28),
					Double.valueOf(123.456789),
					isLocaleNeutral?new BigDecimal("123E-456789"):new BigDecimal("123.45678901234567890123456789"), //$NON-NLS-1$ //$NON-NLS-2$
					null,
					null,
					null,
					null,
					null,
					null
			}
		};
		
		results = new MockExtractionResults(
				TEST_DATA_TYPES_COLUMNS,
				TEST_DATA_TYPES_TYPES,
				data
		); 
		
		subtestRegular( option, fileName);
	}
	
	public void testOutputWithoutColumnType() throws Exception
	{
		option.setExportDataType( false );
		subtestRegular( option, "testNoColumnType.csv" ); //$NON-NLS-1$
	}

	public void testOutputWithPipeSeparator() throws Exception
	{
		option.setLocaleNeutralFormat( true );
		option.setSeparator( ICSVDataExtractionOption.SEPARATOR_PIPE );
		subtestRegular( option, "testRegularPipe.csv" ); //$NON-NLS-1$
	}

	public void testOutputWithSemicolonSeparator() throws Exception
	{
		option.setLocaleNeutralFormat( true );
		option.setSeparator( ICSVDataExtractionOption.SEPARATOR_SEMICOLON );
		subtestRegular( option, "testRegularSemicolon.csv" ); //$NON-NLS-1$
	}
	
	public void testOutputWithTabSeparator() throws Exception
	{
		option.setLocaleNeutralFormat( true );
		option.setSeparator( ICSVDataExtractionOption.SEPARATOR_TAB );
		subtestRegular( option, "testRegularTab.csv" ); //$NON-NLS-1$
	}

	/**
	 * @throws BirtException
	 */
	private void subtestRegular( IDataExtractionOption option, String testFile ) throws BirtException
	{
		CSVDataExtractionImpl extract = createExtraction( out, option );
		extract.output( results );		
		assertFileContent( ROOT_FOLDER + testFile, out.toByteArray( ));
	}	
	
	public void testException()
	{
		// use a dummy extraction results class which throws exceptions
		IExtractionResults faultyResults = new IExtractionResults() {

			public void close( ) 			
			{
			}

			public IResultMetaData getResultMetaData( ) throws BirtException
			{
				throw new BirtException("DummyPluginId", "Test exception", null); //$NON-NLS-1$ //$NON-NLS-2$
			}

			public IDataIterator nextResultIterator( ) throws BirtException
			{
				throw new BirtException("DummyPluginId", "Test exception", null); //$NON-NLS-1$ //$NON-NLS-2$
			}			
		};
		
		CSVDataExtractionImpl extract = null;
		// test with null output stream
		try
		{
			extract = createExtraction( null, option );
			fail("Must throw BirtException if passed output stream is null"); //$NON-NLS-1$
		}
		catch (BirtException e)
		{
			assertFalse("Exception message is localized", e.getMessage().startsWith( "exception." ));  //$NON-NLS-1$//$NON-NLS-2$			
		}
		
		try
		{
			extract = createExtraction( out, option );			
		}
		catch (BirtException e )
		{
			fail("Exception occured while creating extraction instance: " + e.getMessage( )); //$NON-NLS-1$			
		}
		
		try
		{
			// test extraction with results throwing exception
			extract.output( faultyResults );
		}
		catch (BirtException e)
		{
			assertEquals("BirtException contains correct plugin id", CSVDataExtractionImpl.PLUGIN_ID, e.getPluginId( )); //$NON-NLS-1$
			assertFalse("Exception message is localized", e.getMessage().startsWith( "exception." ));  //$NON-NLS-1$//$NON-NLS-2$
			
			Throwable cause = e.getCause( );
			assertNotNull("Exception cause is not null", cause); //$NON-NLS-1$
			assertTrue("Exception cause is of type BirtException", cause instanceof BirtException); //$NON-NLS-1$
			BirtException birtException = (BirtException)cause;
			assertEquals("Cause BirtException contains correct error message", "Test exception", birtException.getMessage( )); //$NON-NLS-1$ //$NON-NLS-2$
			assertEquals("Cause BirtException contains correct plugin id", "DummyPluginId", birtException.getPluginId( )); //$NON-NLS-1$ //$NON-NLS-2$
		}
		catch (Exception e)
		{
			fail("output() method must throw BirtException"); //$NON-NLS-1$
		}
	}
	
	/**
	 * @param option
	 * @return
	 * @throws BirtException
	 */
	private CSVDataExtractionImpl createExtraction(
			OutputStream out,
			IDataExtractionOption option ) throws BirtException
	{
		CSVDataExtractionImpl extract = new CSVDataExtractionImpl();
		
		// simulate engine's DataExtractionTaskV1.setupExtractOption() by copying the values to a new object
		// of instance DataExtractionOption		
		Map allOptions = new HashMap();
		allOptions.putAll( option.getOptions( ) );
		
		DataExtractionOption deOptions = new DataExtractionOption(allOptions);
		deOptions.setOutputStream( out );		
		extract.initialize( null, deOptions );
		return extract;
	}

	/**
	 * @return
	 */
	private CSVDataExtractionOption createOptions( )
	{
		CSVDataExtractionOption option = new CSVDataExtractionOption();
		option.setOutputFile( "test.csv" ); //$NON-NLS-1$
		option.setOutputFormat( "csv" ); //$NON-NLS-1$

		//option.setDateFormat( null );
		option.setExportDataType( true );
		option.setLocale( Locale.ENGLISH );
		option.setLocaleNeutralFormat( true );
		
		//option.setEncoding( ENCODING_UTF_8 );
		option.setSelectedColumns( null );
		option.setSeparator( ICSVDataExtractionOption.SEPARATOR_COMMA );
		return option;
	}

	/**
	 * Asserts that the given file contains the given content.
	 * The file will be compared row by row
	 * @param fileName file to load
	 * @param byteArray content to compare
	 */
	protected void assertFileContent( String fileName, byte[] byteArray )
	{
		File file = new File(fileName);
		BufferedReader fileInput = null;
		BufferedReader resultInput = null;
		try
		{		
			int rowIndex = 0;
			fileInput = new BufferedReader( new InputStreamReader( new FileInputStream(file) ) );
			resultInput = new BufferedReader( new InputStreamReader( new ByteArrayInputStream(byteArray) ) );
			
			String fileContent = null;
			while ( ( fileContent = fileInput.readLine( ) ) != null )
			{
				rowIndex++;
				String resultContent = resultInput.readLine( );
				if ( resultContent == null )
				{
					fail("Result content has less lines than test file"); //$NON-NLS-1$
				}
				assertEquals("Row " + rowIndex + " is identical", fileContent, resultContent); //$NON-NLS-1$ //$NON-NLS-2$
			}
		}
		catch ( FileNotFoundException e )
		{
			fail("Test file \"" + file.getAbsolutePath( ) + "\" not found" ); //$NON-NLS-1$ //$NON-NLS-2$
		}
		catch ( IOException e )
		{
			fail("Error occured while reading test file \"" + file.getAbsolutePath( ) + "\"." ); //$NON-NLS-1$ //$NON-NLS-2$
		}
		finally
		{
			if ( fileInput != null )
			{
				try
				{
					fileInput.close();
				}
				catch ( IOException e )
				{
					fail("Exception while closing test file"); //$NON-NLS-1$
				}
			}
			if ( resultInput != null )
			{
				try
				{
					resultInput.close( );
				}
				catch ( IOException e )
				{
				}
			}
		}
	}

	private java.util.Date makeDate(String string)
	{
		try
		{
			return inputDateFormat.parse(string);
		}
		catch (ParseException e)
		{
			fail("Parse exception occurred for date string \"" + string + "\""); //$NON-NLS-1$ //$NON-NLS-2$
			return null;
		}
	}
	
}
