text: /***************************************************************************
                            qgsvectorlayerexporter.cpp
                               -------------------
      begin                : Thu Aug 25 2011
      copyright            : (C) 2011 by Giuseppe Sucameli
      email                : brush.tyler at gmail.com
   ***************************************************************************/
  
  /***************************************************************************
   *                                                                         *
   *   This program is free software; you can redistribute it and/or modify  *
   *   it under the terms of the GNU General Public License as published by  *
   *   the Free Software Foundation; either version 2 of the License, or     *
   *   (at your option) any later version.                                   *
   *                                                                         *
   ***************************************************************************/
  
  #include "qgsfields.h"
  #include "qgsfeature.h"
  #include "qgsfeatureiterator.h"
  #include "qgsgeometry.h"
  #include "qgslogger.h"
  #include "qgsmessagelog.h"
  #include "qgscoordinatereferencesystem.h"
  #include "qgsvectorlayerexporter.h"
  #include "qgsproviderregistry.h"
  #include "qgsdatasourceuri.h"
  #include "qgsexception.h"
  #include "qgsvectordataprovider.h"
  #include "qgsvectorlayer.h"
  
  #include <QProgressDialog>
  
  #define FEATURE_BUFFER_SIZE 200
  
  typedef QgsVectorLayerExporter::ExportError createEmptyLayer_t(
    const QString &uri,
    const QgsFields &fields,
    QgsWkbTypes::Type geometryType,
    const QgsCoordinateReferenceSystem &destCRS,
    bool overwrite,
    QMap<int, int> *oldToNewAttrIdx,
    QString *errorMessage,
    const QMap<QString, QVariant> *options
  );
  
  
  QgsVectorLayerExporter::QgsVectorLayerExporter( const QString &uri,
      const QString &providerKey,
      const QgsFields &fields,
      QgsWkbTypes::Type geometryType,
      const QgsCoordinateReferenceSystem &crs,
      bool overwrite,
      const QMap<QString, QVariant> &options,
      QgsFeatureSink::SinkFlags sinkFlags )
    : mErrorCount( 0 )
    , mAttributeCount( -1 )
  
  {
    mProvider = nullptr;
  
    QgsProviderRegistry *pReg = QgsProviderRegistry::instance();
  
    std::unique_ptr< QLibrary > myLib( pReg->createProviderLibrary( providerKey ) );
    if ( !myLib )
    {
      mError = ErrInvalidProvider;
      mErrorMessage = QObject::tr( "Unable to load %1 provider" ).arg( providerKey );
      return;
    }
  
    createEmptyLayer_t *pCreateEmpty = reinterpret_cast< createEmptyLayer_t * >( cast_to_fptr( myLib->resolve( "createEmptyLayer" ) ) );
    if ( !pCreateEmpty )
    {
      mError = ErrProviderUnsupportedFeature;
      mErrorMessage = QObject::tr( "Provider %1 has no %2 method" ).arg( providerKey, QStringLiteral( "createEmptyLayer" ) );
      return;
    }
  
    // create an empty layer
    QString errMsg;
    mError = pCreateEmpty( uri, fields, geometryType, crs, overwrite, &mOldToNewAttrIdx, &errMsg, !options.isEmpty() ? &options : nullptr );
    if ( errorCode() )
    {
      mErrorMessage = errMsg;
      return;
    }
  
    const auto constMOldToNewAttrIdx = mOldToNewAttrIdx;
    for ( int idx : constMOldToNewAttrIdx )
    {
      if ( idx > mAttributeCount )
        mAttributeCount = idx;
    }
  
    mAttributeCount++;
  
    QgsDebugMsg( QStringLiteral( "Created empty layer" ) );
  
    QString uriUpdated( uri );
    // HACK sorry...
    if ( providerKey == QLatin1String( "ogr" ) )
    {
      QString layerName;
      if ( options.contains( QStringLiteral( "layerName" ) ) )
        layerName = options.value( QStringLiteral( "layerName" ) ).toString();
      if ( !layerName.isEmpty() )
      {
        uriUpdated += QLatin1String( "|layername=" );
        uriUpdated += layerName;
      }
    }
  
    QgsDataProvider::ProviderOptions providerOptions;
    QgsVectorDataProvider *vectorProvider = dynamic_cast< QgsVectorDataProvider * >( pReg->createProvider( providerKey, uriUpdated, providerOptions ) );
    if ( !vectorProvider || !vectorProvider->isValid() || ( vectorProvider->capabilities() & QgsVectorDataProvider::AddFeatures ) == 0 )
    {
      mError = ErrInvalidLayer;
      mErrorMessage = QObject::tr( "Loading of layer failed" );
  
      delete vectorProvider;
      return;
    }
  
    // If the result is a geopackage layer and there is already a field name FID requested which
    // might contain duplicates, make sure to generate a new field with a unique name instead
    // that will be filled by ogr with unique values.
  
    // HACK sorry
    const QString path = QgsProviderRegistry::instance()->decodeUri( QStringLiteral( "ogr" ), uri ).value( QStringLiteral( "path" ) ).toString();
    if ( sinkFlags.testFlag( QgsFeatureSink::SinkFlag::RegeneratePrimaryKey ) && path.endsWith( QLatin1String( ".gpkg" ), Qt::CaseInsensitive ) )
    {
      QString fidName = options.value( QStringLiteral( "FID" ), QStringLiteral( "FID" ) ).toString();
      int fidIdx = vectorProvider->fields().lookupField( fidName );
      if ( fidIdx != -1 )
      {
        mOldToNewAttrIdx.remove( fidIdx );
      }
    }
  
    mProvider = vectorProvider;
    mError = NoError;
  }
  
  QgsVectorLayerExporter::~QgsVectorLayerExporter()
  {
    flushBuffer();
    delete mProvider;
  }
  
  QgsVectorLayerExporter::ExportError QgsVectorLayerExporter::errorCode() const
  {
    return mError;
  }
  
  QString QgsVectorLayerExporter::errorMessage() const
  {
    return mErrorMessage;
  }
  
  bool QgsVectorLayerExporter::addFeatures( QgsFeatureList &features, Flags flags )
  {
    QgsFeatureList::iterator fIt = features.begin();
    bool result = true;
    for ( ; fIt != features.end(); ++fIt )
    {
      result = result && addFeature( *fIt, flags );
    }
    return result;
  }
  
  bool QgsVectorLayerExporter::addFeature( QgsFeature &feat, Flags )
  {
    QgsAttributes attrs = feat.attributes();
  
    QgsFeature newFeat;
    if ( feat.hasGeometry() )
      newFeat.setGeometry( feat.geometry() );
  
    newFeat.initAttributes( mAttributeCount );
  
    for ( int i = 0; i < attrs.count(); ++i )
    {
      // add only mapped attributes (un-mapped ones will not be present in the
      // destination layer)
      int dstIdx = mOldToNewAttrIdx.value( i, -1 );
      if ( dstIdx < 0 )
        continue;
  
      QgsDebugMsgLevel( QStringLiteral( "moving field from pos %1 to %2" ).arg( i ).arg( dstIdx ), 3 );
      newFeat.setAttribute( dstIdx, attrs.at( i ) );
    }
  
    mFeatureBuffer.append( newFeat );
  
    if ( mFeatureBuffer.count() >= FEATURE_BUFFER_SIZE )
    {
      return flushBuffer();
    }
  
    return true;
  }
  
  bool QgsVectorLayerExporter::flushBuffer()
  {
    if ( mFeatureBuffer.count() <= 0 )
      return true;
  
    if ( !mProvider->addFeatures( mFeatureBuffer, QgsFeatureSink::FastInsert ) )
    {
      QStringList errors = mProvider->errors();
      mProvider->clearErrors();
  
      mErrorMessage = QObject::tr( "Creation error for features from #%1 to #%2. Provider errors was: \n%3" )
                      .arg( mFeatureBuffer.first().id() )
                      .arg( mFeatureBuffer.last().id() )
                      .arg( errors.join( QStringLiteral( "\n" ) ) );
  
      mError = ErrFeatureWriteFailed;
      mErrorCount += mFeatureBuffer.count();
  
      mFeatureBuffer.clear();
      QgsDebugMsg( mErrorMessage );
      return false;
    }
  
    mFeatureBuffer.clear();
    return true;
  }
  
  bool QgsVectorLayerExporter::createSpatialIndex()
  {
    if ( mProvider && ( mProvider->capabilities() & QgsVectorDataProvider::CreateSpatialIndex ) != 0 )
    {
      return mProvider->createSpatialIndex();
    }
    else
    {
      return true;
    }
  }
  
  QgsVectorLayerExporter::ExportError
  QgsVectorLayerExporter::exportLayer( QgsVectorLayer *layer,
                                       const QString &uri,
                                       const QString &providerKey,
                                       const QgsCoordinateReferenceSystem &destCRS,
                                       bool onlySelected,
                                       QString *errorMessage,
                                       const QMap<QString, QVariant> &options,
                                       QgsFeedback *feedback )
  {
    QgsCoordinateReferenceSystem outputCRS;
    QgsCoordinateTransform ct;
    bool shallTransform = false;
  
    if ( !layer )
      return ErrInvalidLayer;
  
    if ( destCRS.isValid() )
    {
      // This means we should transform
      outputCRS = destCRS;
      shallTransform = true;
    }
    else
    {
      // This means we shouldn't transform, use source CRS as output (if defined)
      outputCRS = layer->crs();
    }
  
  
    bool overwrite = false;
    bool forceSinglePartGeom = false;
    QMap<QString, QVariant> providerOptions = options;
    if ( !options.isEmpty() )
    {
      overwrite = providerOptions.take( QStringLiteral( "overwrite" ) ).toBool();
      forceSinglePartGeom = providerOptions.take( QStringLiteral( "forceSinglePartGeometryType" ) ).toBool();
    }
  
    QgsFields fields = layer->fields();
    QgsWkbTypes::Type wkbType = layer->wkbType();
  
    // Special handling for Shapefiles
    if ( layer->providerType() == QLatin1String( "ogr" ) && layer->storageType() == QLatin1String( "ESRI Shapefile" ) )
    {
      // convert field names to lowercase
      for ( int fldIdx = 0; fldIdx < fields.count(); ++fldIdx )
      {
        fields[fldIdx].setName( fields.at( fldIdx ).name().toLower() );
      }
  
      if ( !forceSinglePartGeom )
      {
        // convert wkbtype to multipart (see #5547)
        switch ( wkbType )
        {
          case QgsWkbTypes::Point:
            wkbType = QgsWkbTypes::MultiPoint;
            break;
          case QgsWkbTypes::LineString:
            wkbType = QgsWkbTypes::MultiLineString;
            break;
          case QgsWkbTypes::Polygon:
            wkbType = QgsWkbTypes::MultiPolygon;
            break;
          case QgsWkbTypes::Point25D:
            wkbType = QgsWkbTypes::MultiPoint25D;
            break;
          case QgsWkbTypes::LineString25D:
            wkbType = QgsWkbTypes::MultiLineString25D;
            break;
          case QgsWkbTypes::Polygon25D:
            wkbType = QgsWkbTypes::MultiPolygon25D;
            break;
          default:
            break;
        }
      }
    }
  
    QgsVectorLayerExporter *writer =
      new QgsVectorLayerExporter( uri, providerKey, fields, wkbType, outputCRS, overwrite, providerOptions );
  
    // check whether file creation was successful
    ExportError err = writer->errorCode();
    if ( err != NoError )
    {
      if ( errorMessage )
        *errorMessage = writer->errorMessage();
      delete writer;
      return err;
    }
  
    if ( errorMessage )
    {
      errorMessage->clear();
    }
  
    QgsFeature fet;
  
    QgsFeatureRequest req;
    if ( wkbType == QgsWkbTypes::NoGeometry )
      req.setFlags( QgsFeatureRequest::NoGeometry );
    if ( onlySelected )
      req.setFilterFids( layer->selectedFeatureIds() );
  
    QgsFeatureIterator fit = layer->getFeatures( req );
  
    // Create our transform
    if ( destCRS.isValid() )
    {
      ct = QgsCoordinateTransform( layer->crs(), destCRS, layer->transformContext() );
    }
  
    // Check for failure
    if ( !ct.isValid() )
      shallTransform = false;
  
    long n = 0;
    long approxTotal = onlySelected ? layer->selectedFeatureCount() : layer->featureCount();
  
    if ( errorMessage )
    {
      *errorMessage = QObject::tr( "Feature write errors:" );
    }
  
    bool canceled = false;
  
    // write all features
    while ( fit.nextFeature( fet ) )
    {
      if ( feedback && feedback->isCanceled() )
      {
        canceled = true;
        if ( errorMessage )
        {
          *errorMessage += '\n' + QObject::tr( "Import was canceled at %1 of %2" ).arg( n ).arg( approxTotal );
        }
        break;
      }
  
      if ( writer->errorCount() > 1000 )
      {
        if ( errorMessage )
        {
          *errorMessage += '\n' + QObject::tr( "Stopping after %1 errors" ).arg( writer->errorCount() );
        }
        break;
      }
  
      if ( shallTransform )
      {
        try
        {
          if ( fet.hasGeometry() )
          {
            QgsGeometry g = fet.geometry();
            g.transform( ct );
            fet.setGeometry( g );
          }
        }
        catch ( QgsCsException &e )
        {
          delete writer;
  
          QString msg = QObject::tr( "Failed to transform a point while drawing a feature with ID '%1'. Writing stopped. (Exception: %2)" )
                        .arg( fet.id() ).arg( e.what() );
          QgsMessageLog::logMessage( msg, QObject::tr( "Vector import" ) );
          if ( errorMessage )
            *errorMessage += '\n' + msg;
  
          return ErrProjection;
        }
      }
      if ( !writer->addFeature( fet ) )
      {
        if ( writer->errorCode() && errorMessage )
        {
          *errorMessage += '\n' + writer->errorMessage();
        }
      }
      n++;
  
      if ( feedback )
      {
        feedback->setProgress( 100.0 * static_cast< double >( n ) / approxTotal );
      }
  
    }
  
    // flush the buffer to be sure that all features are written
    if ( !writer->flushBuffer() )
    {
      if ( writer->errorCode() && errorMessage )
      {
        *errorMessage += '\n' + writer->errorMessage();
      }
    }
    int errors = writer->errorCount();
  
    if ( !writer->createSpatialIndex() )
    {
      if ( writer->errorCode() && errorMessage )
      {
        *errorMessage += '\n' + writer->errorMessage();
      }
    }
  
    delete writer;
  
    if ( errorMessage )
    {
      if ( errors > 0 )
      {
        *errorMessage += '\n' + QObject::tr( "Only %1 of %2 features written." ).arg( n - errors ).arg( n );
      }
      else
      {
        errorMessage->clear();
      }
    }
  
    if ( canceled )
      return ErrUserCanceled;
    else if ( errors > 0 )
      return ErrFeatureWriteFailed;
  
    return NoError;
  }
  
  
  //
  // QgsVectorLayerExporterTask
  //
  
  QgsVectorLayerExporterTask::QgsVectorLayerExporterTask( QgsVectorLayer *layer, const QString &uri, const QString &providerKey, const QgsCoordinateReferenceSystem &destinationCrs, const QMap<QString, QVariant> &options, bool ownsLayer )
    : QgsTask( tr( "Exporting %1" ).arg( layer->name() ), QgsTask::CanCancel )
    , mLayer( layer )
    , mOwnsLayer( ownsLayer )
    , mDestUri( uri )
    , mDestProviderKey( providerKey )
    , mDestCrs( destinationCrs )
    , mOptions( options )
    , mOwnedFeedback( new QgsFeedback() )
  {
    if ( mLayer )
      setDependentLayers( QList< QgsMapLayer * >() << mLayer );
  }
  
  QgsVectorLayerExporterTask *QgsVectorLayerExporterTask::withLayerOwnership( QgsVectorLayer *layer, const QString &uri, const QString &providerKey, const QgsCoordinateReferenceSystem &destinationCrs, const QMap<QString, QVariant> &options )
  {
    std::unique_ptr< QgsVectorLayerExporterTask > newTask( new QgsVectorLayerExporterTask( layer, uri, providerKey, destinationCrs, options ) );
    newTask->mOwnsLayer = true;
    return newTask.release();
  }
  
  void QgsVectorLayerExporterTask::cancel()
  {
    mOwnedFeedback->cancel();
    QgsTask::cancel();
  }
  
  bool QgsVectorLayerExporterTask::run()
  {
    if ( !mLayer )
      return false;
  
    connect( mOwnedFeedback.get(), &QgsFeedback::progressChanged, this, &QgsVectorLayerExporterTask::setProgress );
  
  
    mError = QgsVectorLayerExporter::exportLayer(
               mLayer.data(), mDestUri, mDestProviderKey, mDestCrs, false, &mErrorMessage,
               mOptions, mOwnedFeedback.get() );
  
    return mError == QgsVectorLayerExporter::NoError;
  }
  
  void QgsVectorLayerExporterTask::finished( bool result )
  {
    // QgsMapLayer has QTimer member, which must not be destroyed from another thread
    if ( mOwnsLayer )
      delete mLayer;
  
    if ( result )
      emit exportComplete();
    else
      emit errorOccurred( mError, mErrorMessage );
  }
  